| File: | jdk/src/hotspot/share/opto/loopnode.cpp |
| Warning: | line 3586, column 32 Division by zero |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | ||||||||||||
| 2 | * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. | ||||||||||||
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | ||||||||||||
| 4 | * | ||||||||||||
| 5 | * This code is free software; you can redistribute it and/or modify it | ||||||||||||
| 6 | * under the terms of the GNU General Public License version 2 only, as | ||||||||||||
| 7 | * published by the Free Software Foundation. | ||||||||||||
| 8 | * | ||||||||||||
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT | ||||||||||||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | ||||||||||||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | ||||||||||||
| 12 | * version 2 for more details (a copy is included in the LICENSE file that | ||||||||||||
| 13 | * accompanied this code). | ||||||||||||
| 14 | * | ||||||||||||
| 15 | * You should have received a copy of the GNU General Public License version | ||||||||||||
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, | ||||||||||||
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | ||||||||||||
| 18 | * | ||||||||||||
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | ||||||||||||
| 20 | * or visit www.oracle.com if you need additional information or have any | ||||||||||||
| 21 | * questions. | ||||||||||||
| 22 | * | ||||||||||||
| 23 | */ | ||||||||||||
| 24 | |||||||||||||
| 25 | #include "precompiled.hpp" | ||||||||||||
| 26 | #include "ci/ciMethodData.hpp" | ||||||||||||
| 27 | #include "compiler/compileLog.hpp" | ||||||||||||
| 28 | #include "gc/shared/barrierSet.hpp" | ||||||||||||
| 29 | #include "gc/shared/c2/barrierSetC2.hpp" | ||||||||||||
| 30 | #include "libadt/vectset.hpp" | ||||||||||||
| 31 | #include "memory/allocation.inline.hpp" | ||||||||||||
| 32 | #include "memory/resourceArea.hpp" | ||||||||||||
| 33 | #include "opto/addnode.hpp" | ||||||||||||
| 34 | #include "opto/arraycopynode.hpp" | ||||||||||||
| 35 | #include "opto/callnode.hpp" | ||||||||||||
| 36 | #include "opto/castnode.hpp" | ||||||||||||
| 37 | #include "opto/connode.hpp" | ||||||||||||
| 38 | #include "opto/convertnode.hpp" | ||||||||||||
| 39 | #include "opto/divnode.hpp" | ||||||||||||
| 40 | #include "opto/idealGraphPrinter.hpp" | ||||||||||||
| 41 | #include "opto/loopnode.hpp" | ||||||||||||
| 42 | #include "opto/movenode.hpp" | ||||||||||||
| 43 | #include "opto/mulnode.hpp" | ||||||||||||
| 44 | #include "opto/opaquenode.hpp" | ||||||||||||
| 45 | #include "opto/rootnode.hpp" | ||||||||||||
| 46 | #include "opto/runtime.hpp" | ||||||||||||
| 47 | #include "opto/superword.hpp" | ||||||||||||
| 48 | #include "runtime/sharedRuntime.hpp" | ||||||||||||
| 49 | #include "utilities/powerOfTwo.hpp" | ||||||||||||
| 50 | |||||||||||||
| 51 | //============================================================================= | ||||||||||||
| 52 | //--------------------------is_cloop_ind_var----------------------------------- | ||||||||||||
| 53 | // Determine if a node is a counted loop induction variable. | ||||||||||||
| 54 | // NOTE: The method is declared in "node.hpp". | ||||||||||||
| 55 | bool Node::is_cloop_ind_var() const { | ||||||||||||
| 56 | return (is_Phi() && | ||||||||||||
| 57 | as_Phi()->region()->is_CountedLoop() && | ||||||||||||
| 58 | as_Phi()->region()->as_CountedLoop()->phi() == this); | ||||||||||||
| 59 | } | ||||||||||||
| 60 | |||||||||||||
| 61 | //============================================================================= | ||||||||||||
| 62 | //------------------------------dump_spec-------------------------------------- | ||||||||||||
| 63 | // Dump special per-node info | ||||||||||||
| 64 | #ifndef PRODUCT | ||||||||||||
| 65 | void LoopNode::dump_spec(outputStream *st) const { | ||||||||||||
| 66 | if (is_inner_loop()) st->print( "inner " ); | ||||||||||||
| 67 | if (is_partial_peel_loop()) st->print( "partial_peel " ); | ||||||||||||
| 68 | if (partial_peel_has_failed()) st->print( "partial_peel_failed " ); | ||||||||||||
| 69 | } | ||||||||||||
| 70 | #endif | ||||||||||||
| 71 | |||||||||||||
| 72 | //------------------------------is_valid_counted_loop------------------------- | ||||||||||||
| 73 | bool LoopNode::is_valid_counted_loop(BasicType bt) const { | ||||||||||||
| 74 | if (is_BaseCountedLoop() && as_BaseCountedLoop()->bt() == bt) { | ||||||||||||
| 75 | BaseCountedLoopNode* l = as_BaseCountedLoop(); | ||||||||||||
| 76 | BaseCountedLoopEndNode* le = l->loopexit_or_null(); | ||||||||||||
| 77 | if (le != NULL__null && | ||||||||||||
| 78 | le->proj_out_or_null(1 /* true */) == l->in(LoopNode::LoopBackControl)) { | ||||||||||||
| 79 | Node* phi = l->phi(); | ||||||||||||
| 80 | Node* exit = le->proj_out_or_null(0 /* false */); | ||||||||||||
| 81 | if (exit != NULL__null && exit->Opcode() == Op_IfFalse && | ||||||||||||
| 82 | phi != NULL__null && phi->is_Phi() && | ||||||||||||
| 83 | phi->in(LoopNode::LoopBackControl) == l->incr() && | ||||||||||||
| 84 | le->loopnode() == l && le->stride_is_con()) { | ||||||||||||
| 85 | return true; | ||||||||||||
| 86 | } | ||||||||||||
| 87 | } | ||||||||||||
| 88 | } | ||||||||||||
| 89 | return false; | ||||||||||||
| 90 | } | ||||||||||||
| 91 | |||||||||||||
| 92 | //------------------------------get_early_ctrl--------------------------------- | ||||||||||||
| 93 | // Compute earliest legal control | ||||||||||||
| 94 | Node *PhaseIdealLoop::get_early_ctrl( Node *n ) { | ||||||||||||
| 95 | assert( !n->is_Phi() && !n->is_CFG(), "this code only handles data nodes" )do { if (!(!n->is_Phi() && !n->is_CFG())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 95, "assert(" "!n->is_Phi() && !n->is_CFG()" ") failed" , "this code only handles data nodes"); ::breakpoint(); } } while (0); | ||||||||||||
| 96 | uint i; | ||||||||||||
| 97 | Node *early; | ||||||||||||
| 98 | if (n->in(0) && !n->is_expensive()) { | ||||||||||||
| 99 | early = n->in(0); | ||||||||||||
| 100 | if (!early->is_CFG()) // Might be a non-CFG multi-def | ||||||||||||
| 101 | early = get_ctrl(early); // So treat input as a straight data input | ||||||||||||
| 102 | i = 1; | ||||||||||||
| 103 | } else { | ||||||||||||
| 104 | early = get_ctrl(n->in(1)); | ||||||||||||
| 105 | i = 2; | ||||||||||||
| 106 | } | ||||||||||||
| 107 | uint e_d = dom_depth(early); | ||||||||||||
| 108 | assert( early, "" )do { if (!(early)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 108, "assert(" "early" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 109 | for (; i < n->req(); i++) { | ||||||||||||
| 110 | Node *cin = get_ctrl(n->in(i)); | ||||||||||||
| 111 | assert( cin, "" )do { if (!(cin)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 111, "assert(" "cin" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 112 | // Keep deepest dominator depth | ||||||||||||
| 113 | uint c_d = dom_depth(cin); | ||||||||||||
| 114 | if (c_d > e_d) { // Deeper guy? | ||||||||||||
| 115 | early = cin; // Keep deepest found so far | ||||||||||||
| 116 | e_d = c_d; | ||||||||||||
| 117 | } else if (c_d == e_d && // Same depth? | ||||||||||||
| 118 | early != cin) { // If not equal, must use slower algorithm | ||||||||||||
| 119 | // If same depth but not equal, one _must_ dominate the other | ||||||||||||
| 120 | // and we want the deeper (i.e., dominated) guy. | ||||||||||||
| 121 | Node *n1 = early; | ||||||||||||
| 122 | Node *n2 = cin; | ||||||||||||
| 123 | while (1) { | ||||||||||||
| 124 | n1 = idom(n1); // Walk up until break cycle | ||||||||||||
| 125 | n2 = idom(n2); | ||||||||||||
| 126 | if (n1 == cin || // Walked early up to cin | ||||||||||||
| 127 | dom_depth(n2) < c_d) | ||||||||||||
| 128 | break; // early is deeper; keep him | ||||||||||||
| 129 | if (n2 == early || // Walked cin up to early | ||||||||||||
| 130 | dom_depth(n1) < c_d) { | ||||||||||||
| 131 | early = cin; // cin is deeper; keep him | ||||||||||||
| 132 | break; | ||||||||||||
| 133 | } | ||||||||||||
| 134 | } | ||||||||||||
| 135 | e_d = dom_depth(early); // Reset depth register cache | ||||||||||||
| 136 | } | ||||||||||||
| 137 | } | ||||||||||||
| 138 | |||||||||||||
| 139 | // Return earliest legal location | ||||||||||||
| 140 | assert(early == find_non_split_ctrl(early), "unexpected early control")do { if (!(early == find_non_split_ctrl(early))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 140, "assert(" "early == find_non_split_ctrl(early)" ") failed" , "unexpected early control"); ::breakpoint(); } } while (0); | ||||||||||||
| 141 | |||||||||||||
| 142 | if (n->is_expensive() && !_verify_only && !_verify_me) { | ||||||||||||
| 143 | assert(n->in(0), "should have control input")do { if (!(n->in(0))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 143, "assert(" "n->in(0)" ") failed", "should have control input" ); ::breakpoint(); } } while (0); | ||||||||||||
| 144 | early = get_early_ctrl_for_expensive(n, early); | ||||||||||||
| 145 | } | ||||||||||||
| 146 | |||||||||||||
| 147 | return early; | ||||||||||||
| 148 | } | ||||||||||||
| 149 | |||||||||||||
| 150 | //------------------------------get_early_ctrl_for_expensive--------------------------------- | ||||||||||||
| 151 | // Move node up the dominator tree as high as legal while still beneficial | ||||||||||||
| 152 | Node *PhaseIdealLoop::get_early_ctrl_for_expensive(Node *n, Node* earliest) { | ||||||||||||
| 153 | assert(n->in(0) && n->is_expensive(), "expensive node with control input here")do { if (!(n->in(0) && n->is_expensive())) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 153, "assert(" "n->in(0) && n->is_expensive()" ") failed", "expensive node with control input here"); ::breakpoint (); } } while (0); | ||||||||||||
| 154 | assert(OptimizeExpensiveOps, "optimization off?")do { if (!(OptimizeExpensiveOps)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 154, "assert(" "OptimizeExpensiveOps" ") failed", "optimization off?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 155 | |||||||||||||
| 156 | Node* ctl = n->in(0); | ||||||||||||
| 157 | assert(ctl->is_CFG(), "expensive input 0 must be cfg")do { if (!(ctl->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 157, "assert(" "ctl->is_CFG()" ") failed", "expensive input 0 must be cfg" ); ::breakpoint(); } } while (0); | ||||||||||||
| 158 | uint min_dom_depth = dom_depth(earliest); | ||||||||||||
| 159 | #ifdef ASSERT1 | ||||||||||||
| 160 | if (!is_dominator(ctl, earliest) && !is_dominator(earliest, ctl)) { | ||||||||||||
| 161 | dump_bad_graph("Bad graph detected in get_early_ctrl_for_expensive", n, earliest, ctl); | ||||||||||||
| 162 | assert(false, "Bad graph detected in get_early_ctrl_for_expensive")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 162, "assert(" "false" ") failed", "Bad graph detected in get_early_ctrl_for_expensive" ); ::breakpoint(); } } while (0); | ||||||||||||
| 163 | } | ||||||||||||
| 164 | #endif | ||||||||||||
| 165 | if (dom_depth(ctl) < min_dom_depth) { | ||||||||||||
| 166 | return earliest; | ||||||||||||
| 167 | } | ||||||||||||
| 168 | |||||||||||||
| 169 | while (1) { | ||||||||||||
| 170 | Node *next = ctl; | ||||||||||||
| 171 | // Moving the node out of a loop on the projection of a If | ||||||||||||
| 172 | // confuses loop predication. So once we hit a Loop in a If branch | ||||||||||||
| 173 | // that doesn't branch to an UNC, we stop. The code that process | ||||||||||||
| 174 | // expensive nodes will notice the loop and skip over it to try to | ||||||||||||
| 175 | // move the node further up. | ||||||||||||
| 176 | if (ctl->is_CountedLoop() && ctl->in(1) != NULL__null && ctl->in(1)->in(0) != NULL__null && ctl->in(1)->in(0)->is_If()) { | ||||||||||||
| 177 | if (!ctl->in(1)->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { | ||||||||||||
| 178 | break; | ||||||||||||
| 179 | } | ||||||||||||
| 180 | next = idom(ctl->in(1)->in(0)); | ||||||||||||
| 181 | } else if (ctl->is_Proj()) { | ||||||||||||
| 182 | // We only move it up along a projection if the projection is | ||||||||||||
| 183 | // the single control projection for its parent: same code path, | ||||||||||||
| 184 | // if it's a If with UNC or fallthrough of a call. | ||||||||||||
| 185 | Node* parent_ctl = ctl->in(0); | ||||||||||||
| 186 | if (parent_ctl == NULL__null) { | ||||||||||||
| 187 | break; | ||||||||||||
| 188 | } else if (parent_ctl->is_CountedLoopEnd() && parent_ctl->as_CountedLoopEnd()->loopnode() != NULL__null) { | ||||||||||||
| 189 | next = parent_ctl->as_CountedLoopEnd()->loopnode()->init_control(); | ||||||||||||
| 190 | } else if (parent_ctl->is_If()) { | ||||||||||||
| 191 | if (!ctl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none)) { | ||||||||||||
| 192 | break; | ||||||||||||
| 193 | } | ||||||||||||
| 194 | assert(idom(ctl) == parent_ctl, "strange")do { if (!(idom(ctl) == parent_ctl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 194, "assert(" "idom(ctl) == parent_ctl" ") failed", "strange" ); ::breakpoint(); } } while (0); | ||||||||||||
| 195 | next = idom(parent_ctl); | ||||||||||||
| 196 | } else if (ctl->is_CatchProj()) { | ||||||||||||
| 197 | if (ctl->as_Proj()->_con != CatchProjNode::fall_through_index) { | ||||||||||||
| 198 | break; | ||||||||||||
| 199 | } | ||||||||||||
| 200 | assert(parent_ctl->in(0)->in(0)->is_Call(), "strange graph")do { if (!(parent_ctl->in(0)->in(0)->is_Call())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 200, "assert(" "parent_ctl->in(0)->in(0)->is_Call()" ") failed", "strange graph"); ::breakpoint(); } } while (0); | ||||||||||||
| 201 | next = parent_ctl->in(0)->in(0)->in(0); | ||||||||||||
| 202 | } else { | ||||||||||||
| 203 | // Check if parent control has a single projection (this | ||||||||||||
| 204 | // control is the only possible successor of the parent | ||||||||||||
| 205 | // control). If so, we can try to move the node above the | ||||||||||||
| 206 | // parent control. | ||||||||||||
| 207 | int nb_ctl_proj = 0; | ||||||||||||
| 208 | for (DUIterator_Fast imax, i = parent_ctl->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 209 | Node *p = parent_ctl->fast_out(i); | ||||||||||||
| 210 | if (p->is_Proj() && p->is_CFG()) { | ||||||||||||
| 211 | nb_ctl_proj++; | ||||||||||||
| 212 | if (nb_ctl_proj > 1) { | ||||||||||||
| 213 | break; | ||||||||||||
| 214 | } | ||||||||||||
| 215 | } | ||||||||||||
| 216 | } | ||||||||||||
| 217 | |||||||||||||
| 218 | if (nb_ctl_proj > 1) { | ||||||||||||
| 219 | break; | ||||||||||||
| 220 | } | ||||||||||||
| 221 | assert(parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() ||do { if (!(parent_ctl->is_Start() || parent_ctl->is_MemBar () || parent_ctl->is_Call() || BarrierSet::barrier_set()-> barrier_set_c2()->is_gc_barrier_node(parent_ctl))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 222, "assert(" "parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl)" ") failed", "unexpected node"); ::breakpoint(); } } while (0 ) | ||||||||||||
| 222 | BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl), "unexpected node")do { if (!(parent_ctl->is_Start() || parent_ctl->is_MemBar () || parent_ctl->is_Call() || BarrierSet::barrier_set()-> barrier_set_c2()->is_gc_barrier_node(parent_ctl))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 222, "assert(" "parent_ctl->is_Start() || parent_ctl->is_MemBar() || parent_ctl->is_Call() || BarrierSet::barrier_set()->barrier_set_c2()->is_gc_barrier_node(parent_ctl)" ") failed", "unexpected node"); ::breakpoint(); } } while (0 ); | ||||||||||||
| 223 | assert(idom(ctl) == parent_ctl, "strange")do { if (!(idom(ctl) == parent_ctl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 223, "assert(" "idom(ctl) == parent_ctl" ") failed", "strange" ); ::breakpoint(); } } while (0); | ||||||||||||
| 224 | next = idom(parent_ctl); | ||||||||||||
| 225 | } | ||||||||||||
| 226 | } else { | ||||||||||||
| 227 | next = idom(ctl); | ||||||||||||
| 228 | } | ||||||||||||
| 229 | if (next->is_Root() || next->is_Start() || dom_depth(next) < min_dom_depth) { | ||||||||||||
| 230 | break; | ||||||||||||
| 231 | } | ||||||||||||
| 232 | ctl = next; | ||||||||||||
| 233 | } | ||||||||||||
| 234 | |||||||||||||
| 235 | if (ctl != n->in(0)) { | ||||||||||||
| 236 | _igvn.replace_input_of(n, 0, ctl); | ||||||||||||
| 237 | _igvn.hash_insert(n); | ||||||||||||
| 238 | } | ||||||||||||
| 239 | |||||||||||||
| 240 | return ctl; | ||||||||||||
| 241 | } | ||||||||||||
| 242 | |||||||||||||
| 243 | |||||||||||||
| 244 | //------------------------------set_early_ctrl--------------------------------- | ||||||||||||
| 245 | // Set earliest legal control | ||||||||||||
| 246 | void PhaseIdealLoop::set_early_ctrl(Node* n, bool update_body) { | ||||||||||||
| 247 | Node *early = get_early_ctrl(n); | ||||||||||||
| 248 | |||||||||||||
| 249 | // Record earliest legal location | ||||||||||||
| 250 | set_ctrl(n, early); | ||||||||||||
| 251 | IdealLoopTree *loop = get_loop(early); | ||||||||||||
| 252 | if (update_body && loop->_child == NULL__null) { | ||||||||||||
| 253 | loop->_body.push(n); | ||||||||||||
| 254 | } | ||||||||||||
| 255 | } | ||||||||||||
| 256 | |||||||||||||
| 257 | //------------------------------set_subtree_ctrl------------------------------- | ||||||||||||
| 258 | // set missing _ctrl entries on new nodes | ||||||||||||
| 259 | void PhaseIdealLoop::set_subtree_ctrl(Node* n, bool update_body) { | ||||||||||||
| 260 | // Already set? Get out. | ||||||||||||
| 261 | if (_nodes[n->_idx]) return; | ||||||||||||
| 262 | // Recursively set _nodes array to indicate where the Node goes | ||||||||||||
| 263 | uint i; | ||||||||||||
| 264 | for (i = 0; i < n->req(); ++i) { | ||||||||||||
| 265 | Node *m = n->in(i); | ||||||||||||
| 266 | if (m && m != C->root()) { | ||||||||||||
| 267 | set_subtree_ctrl(m, update_body); | ||||||||||||
| 268 | } | ||||||||||||
| 269 | } | ||||||||||||
| 270 | |||||||||||||
| 271 | // Fixup self | ||||||||||||
| 272 | set_early_ctrl(n, update_body); | ||||||||||||
| 273 | } | ||||||||||||
| 274 | |||||||||||||
| 275 | IdealLoopTree* PhaseIdealLoop::insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift) { | ||||||||||||
| 276 | IdealLoopTree* outer_ilt = new IdealLoopTree(this, outer_l, outer_ift); | ||||||||||||
| 277 | IdealLoopTree* parent = loop->_parent; | ||||||||||||
| 278 | IdealLoopTree* sibling = parent->_child; | ||||||||||||
| 279 | if (sibling == loop) { | ||||||||||||
| 280 | parent->_child = outer_ilt; | ||||||||||||
| 281 | } else { | ||||||||||||
| 282 | while (sibling->_next != loop) { | ||||||||||||
| 283 | sibling = sibling->_next; | ||||||||||||
| 284 | } | ||||||||||||
| 285 | sibling->_next = outer_ilt; | ||||||||||||
| 286 | } | ||||||||||||
| 287 | outer_ilt->_next = loop->_next; | ||||||||||||
| 288 | outer_ilt->_parent = parent; | ||||||||||||
| 289 | outer_ilt->_child = loop; | ||||||||||||
| 290 | outer_ilt->_nest = loop->_nest; | ||||||||||||
| 291 | loop->_parent = outer_ilt; | ||||||||||||
| 292 | loop->_next = NULL__null; | ||||||||||||
| 293 | loop->_nest++; | ||||||||||||
| 294 | assert(loop->_nest <= SHRT_MAX, "sanity")do { if (!(loop->_nest <= 32767)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 294, "assert(" "loop->_nest <= 32767" ") failed", "sanity" ); ::breakpoint(); } } while (0); | ||||||||||||
| 295 | return outer_ilt; | ||||||||||||
| 296 | } | ||||||||||||
| 297 | |||||||||||||
| 298 | // Create a skeleton strip mined outer loop: a Loop head before the | ||||||||||||
| 299 | // inner strip mined loop, a safepoint and an exit condition guarded | ||||||||||||
| 300 | // by an opaque node after the inner strip mined loop with a backedge | ||||||||||||
| 301 | // to the loop head. The inner strip mined loop is left as it is. Only | ||||||||||||
| 302 | // once loop optimizations are over, do we adjust the inner loop exit | ||||||||||||
| 303 | // condition to limit its number of iterations, set the outer loop | ||||||||||||
| 304 | // exit condition and add Phis to the outer loop head. Some loop | ||||||||||||
| 305 | // optimizations that operate on the inner strip mined loop need to be | ||||||||||||
| 306 | // aware of the outer strip mined loop: loop unswitching needs to | ||||||||||||
| 307 | // clone the outer loop as well as the inner, unrolling needs to only | ||||||||||||
| 308 | // clone the inner loop etc. No optimizations need to change the outer | ||||||||||||
| 309 | // strip mined loop as it is only a skeleton. | ||||||||||||
| 310 | IdealLoopTree* PhaseIdealLoop::create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control, | ||||||||||||
| 311 | IdealLoopTree* loop, float cl_prob, float le_fcnt, | ||||||||||||
| 312 | Node*& entry_control, Node*& iffalse) { | ||||||||||||
| 313 | Node* outer_test = _igvn.intcon(0); | ||||||||||||
| 314 | set_ctrl(outer_test, C->root()); | ||||||||||||
| 315 | Node *orig = iffalse; | ||||||||||||
| 316 | iffalse = iffalse->clone(); | ||||||||||||
| 317 | _igvn.register_new_node_with_optimizer(iffalse); | ||||||||||||
| 318 | set_idom(iffalse, idom(orig), dom_depth(orig)); | ||||||||||||
| 319 | |||||||||||||
| 320 | IfNode *outer_le = new OuterStripMinedLoopEndNode(iffalse, outer_test, cl_prob, le_fcnt); | ||||||||||||
| 321 | Node *outer_ift = new IfTrueNode (outer_le); | ||||||||||||
| 322 | Node* outer_iff = orig; | ||||||||||||
| 323 | _igvn.replace_input_of(outer_iff, 0, outer_le); | ||||||||||||
| 324 | |||||||||||||
| 325 | LoopNode *outer_l = new OuterStripMinedLoopNode(C, init_control, outer_ift); | ||||||||||||
| 326 | entry_control = outer_l; | ||||||||||||
| 327 | |||||||||||||
| 328 | IdealLoopTree* outer_ilt = insert_outer_loop(loop, outer_l, outer_ift); | ||||||||||||
| 329 | |||||||||||||
| 330 | set_loop(iffalse, outer_ilt); | ||||||||||||
| 331 | // When this code runs, loop bodies have not yet been populated. | ||||||||||||
| 332 | const bool body_populated = false; | ||||||||||||
| 333 | register_control(outer_le, outer_ilt, iffalse, body_populated); | ||||||||||||
| 334 | register_control(outer_ift, outer_ilt, outer_le, body_populated); | ||||||||||||
| 335 | set_idom(outer_iff, outer_le, dom_depth(outer_le)); | ||||||||||||
| 336 | _igvn.register_new_node_with_optimizer(outer_l); | ||||||||||||
| 337 | set_loop(outer_l, outer_ilt); | ||||||||||||
| 338 | set_idom(outer_l, init_control, dom_depth(init_control)+1); | ||||||||||||
| 339 | |||||||||||||
| 340 | return outer_ilt; | ||||||||||||
| 341 | } | ||||||||||||
| 342 | |||||||||||||
| 343 | void PhaseIdealLoop::insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol) { | ||||||||||||
| 344 | Node* new_predicate_proj = create_new_if_for_predicate(limit_check_proj, NULL__null, | ||||||||||||
| 345 | Deoptimization::Reason_loop_limit_check, | ||||||||||||
| 346 | Op_If); | ||||||||||||
| 347 | Node* iff = new_predicate_proj->in(0); | ||||||||||||
| 348 | assert(iff->Opcode() == Op_If, "bad graph shape")do { if (!(iff->Opcode() == Op_If)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 348, "assert(" "iff->Opcode() == Op_If" ") failed", "bad graph shape" ); ::breakpoint(); } } while (0); | ||||||||||||
| 349 | Node* conv = iff->in(1); | ||||||||||||
| 350 | assert(conv->Opcode() == Op_Conv2B, "bad graph shape")do { if (!(conv->Opcode() == Op_Conv2B)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 350, "assert(" "conv->Opcode() == Op_Conv2B" ") failed", "bad graph shape"); ::breakpoint(); } } while (0); | ||||||||||||
| 351 | Node* opaq = conv->in(1); | ||||||||||||
| 352 | assert(opaq->Opcode() == Op_Opaque1, "bad graph shape")do { if (!(opaq->Opcode() == Op_Opaque1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 352, "assert(" "opaq->Opcode() == Op_Opaque1" ") failed" , "bad graph shape"); ::breakpoint(); } } while (0); | ||||||||||||
| 353 | cmp_limit = _igvn.register_new_node_with_optimizer(cmp_limit); | ||||||||||||
| 354 | bol = _igvn.register_new_node_with_optimizer(bol); | ||||||||||||
| 355 | set_subtree_ctrl(bol, false); | ||||||||||||
| 356 | _igvn.replace_input_of(iff, 1, bol); | ||||||||||||
| 357 | |||||||||||||
| 358 | #ifndef PRODUCT | ||||||||||||
| 359 | // report that the loop predication has been actually performed | ||||||||||||
| 360 | // for this loop | ||||||||||||
| 361 | if (TraceLoopLimitCheck) { | ||||||||||||
| 362 | tty->print_cr("Counted Loop Limit Check generated:"); | ||||||||||||
| 363 | debug_only( bol->dump(2); )bol->dump(2); | ||||||||||||
| 364 | } | ||||||||||||
| 365 | #endif | ||||||||||||
| 366 | } | ||||||||||||
| 367 | |||||||||||||
| 368 | Node* PhaseIdealLoop::loop_exit_control(Node* x, IdealLoopTree* loop) { | ||||||||||||
| 369 | // Counted loop head must be a good RegionNode with only 3 not NULL | ||||||||||||
| 370 | // control input edges: Self, Entry, LoopBack. | ||||||||||||
| 371 | if (x->in(LoopNode::Self) == NULL__null || x->req() != 3 || loop->_irreducible) { | ||||||||||||
| 372 | return NULL__null; | ||||||||||||
| 373 | } | ||||||||||||
| 374 | Node *init_control = x->in(LoopNode::EntryControl); | ||||||||||||
| 375 | Node *back_control = x->in(LoopNode::LoopBackControl); | ||||||||||||
| 376 | if (init_control == NULL__null || back_control == NULL__null) { // Partially dead | ||||||||||||
| 377 | return NULL__null; | ||||||||||||
| 378 | } | ||||||||||||
| 379 | // Must also check for TOP when looking for a dead loop | ||||||||||||
| 380 | if (init_control->is_top() || back_control->is_top()) { | ||||||||||||
| 381 | return NULL__null; | ||||||||||||
| 382 | } | ||||||||||||
| 383 | |||||||||||||
| 384 | // Allow funny placement of Safepoint | ||||||||||||
| 385 | if (back_control->Opcode() == Op_SafePoint) { | ||||||||||||
| 386 | back_control = back_control->in(TypeFunc::Control); | ||||||||||||
| 387 | } | ||||||||||||
| 388 | |||||||||||||
| 389 | // Controlling test for loop | ||||||||||||
| 390 | Node *iftrue = back_control; | ||||||||||||
| 391 | uint iftrue_op = iftrue->Opcode(); | ||||||||||||
| 392 | if (iftrue_op != Op_IfTrue && | ||||||||||||
| 393 | iftrue_op != Op_IfFalse) { | ||||||||||||
| 394 | // I have a weird back-control. Probably the loop-exit test is in | ||||||||||||
| 395 | // the middle of the loop and I am looking at some trailing control-flow | ||||||||||||
| 396 | // merge point. To fix this I would have to partially peel the loop. | ||||||||||||
| 397 | return NULL__null; // Obscure back-control | ||||||||||||
| 398 | } | ||||||||||||
| 399 | |||||||||||||
| 400 | // Get boolean guarding loop-back test | ||||||||||||
| 401 | Node *iff = iftrue->in(0); | ||||||||||||
| 402 | if (get_loop(iff) != loop || !iff->in(1)->is_Bool()) { | ||||||||||||
| 403 | return NULL__null; | ||||||||||||
| 404 | } | ||||||||||||
| 405 | return iftrue; | ||||||||||||
| 406 | } | ||||||||||||
| 407 | |||||||||||||
| 408 | Node* PhaseIdealLoop::loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob) { | ||||||||||||
| 409 | Node* iftrue = back_control; | ||||||||||||
| 410 | uint iftrue_op = iftrue->Opcode(); | ||||||||||||
| 411 | Node* iff = iftrue->in(0); | ||||||||||||
| 412 | BoolNode* test = iff->in(1)->as_Bool(); | ||||||||||||
| 413 | bt = test->_test._test; | ||||||||||||
| 414 | cl_prob = iff->as_If()->_prob; | ||||||||||||
| 415 | if (iftrue_op == Op_IfFalse) { | ||||||||||||
| 416 | bt = BoolTest(bt).negate(); | ||||||||||||
| 417 | cl_prob = 1.0 - cl_prob; | ||||||||||||
| 418 | } | ||||||||||||
| 419 | // Get backedge compare | ||||||||||||
| 420 | Node* cmp = test->in(1); | ||||||||||||
| 421 | if (!cmp->is_Cmp()) { | ||||||||||||
| 422 | return NULL__null; | ||||||||||||
| 423 | } | ||||||||||||
| 424 | |||||||||||||
| 425 | // Find the trip-counter increment & limit. Limit must be loop invariant. | ||||||||||||
| 426 | incr = cmp->in(1); | ||||||||||||
| 427 | limit = cmp->in(2); | ||||||||||||
| 428 | |||||||||||||
| 429 | // --------- | ||||||||||||
| 430 | // need 'loop()' test to tell if limit is loop invariant | ||||||||||||
| 431 | // --------- | ||||||||||||
| 432 | |||||||||||||
| 433 | if (!is_member(loop, get_ctrl(incr))) { // Swapped trip counter and limit? | ||||||||||||
| 434 | Node* tmp = incr; // Then reverse order into the CmpI | ||||||||||||
| 435 | incr = limit; | ||||||||||||
| 436 | limit = tmp; | ||||||||||||
| 437 | bt = BoolTest(bt).commute(); // And commute the exit test | ||||||||||||
| 438 | } | ||||||||||||
| 439 | if (is_member(loop, get_ctrl(limit))) { // Limit must be loop-invariant | ||||||||||||
| 440 | return NULL__null; | ||||||||||||
| 441 | } | ||||||||||||
| 442 | if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant | ||||||||||||
| 443 | return NULL__null; | ||||||||||||
| 444 | } | ||||||||||||
| 445 | return cmp; | ||||||||||||
| 446 | } | ||||||||||||
| 447 | |||||||||||||
| 448 | Node* PhaseIdealLoop::loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr) { | ||||||||||||
| 449 | if (incr->is_Phi()) { | ||||||||||||
| 450 | if (incr->as_Phi()->region() != x || incr->req() != 3) { | ||||||||||||
| 451 | return NULL__null; // Not simple trip counter expression | ||||||||||||
| 452 | } | ||||||||||||
| 453 | phi_incr = incr; | ||||||||||||
| 454 | incr = phi_incr->in(LoopNode::LoopBackControl); // Assume incr is on backedge of Phi | ||||||||||||
| 455 | if (!is_member(loop, get_ctrl(incr))) { // Trip counter must be loop-variant | ||||||||||||
| 456 | return NULL__null; | ||||||||||||
| 457 | } | ||||||||||||
| 458 | } | ||||||||||||
| 459 | return incr; | ||||||||||||
| 460 | } | ||||||||||||
| 461 | |||||||||||||
| 462 | Node* PhaseIdealLoop::loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi) { | ||||||||||||
| 463 | assert(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL, "caller resp.")do { if (!(incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 463, "assert(" "incr->Opcode() == Op_AddI || incr->Opcode() == Op_AddL" ") failed", "caller resp."); ::breakpoint(); } } while (0); | ||||||||||||
| 464 | // Get merge point | ||||||||||||
| 465 | xphi = incr->in(1); | ||||||||||||
| 466 | Node *stride = incr->in(2); | ||||||||||||
| 467 | if (!stride->is_Con()) { // Oops, swap these | ||||||||||||
| 468 | if (!xphi->is_Con()) { // Is the other guy a constant? | ||||||||||||
| 469 | return NULL__null; // Nope, unknown stride, bail out | ||||||||||||
| 470 | } | ||||||||||||
| 471 | Node *tmp = xphi; // 'incr' is commutative, so ok to swap | ||||||||||||
| 472 | xphi = stride; | ||||||||||||
| 473 | stride = tmp; | ||||||||||||
| 474 | } | ||||||||||||
| 475 | return stride; | ||||||||||||
| 476 | } | ||||||||||||
| 477 | |||||||||||||
| 478 | PhiNode* PhaseIdealLoop::loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop) { | ||||||||||||
| 479 | if (!xphi->is_Phi()) { | ||||||||||||
| 480 | return NULL__null; // Too much math on the trip counter | ||||||||||||
| 481 | } | ||||||||||||
| 482 | if (phi_incr != NULL__null && phi_incr != xphi) { | ||||||||||||
| 483 | return NULL__null; | ||||||||||||
| 484 | } | ||||||||||||
| 485 | PhiNode *phi = xphi->as_Phi(); | ||||||||||||
| 486 | |||||||||||||
| 487 | // Phi must be of loop header; backedge must wrap to increment | ||||||||||||
| 488 | if (phi->region() != x) { | ||||||||||||
| 489 | return NULL__null; | ||||||||||||
| 490 | } | ||||||||||||
| 491 | return phi; | ||||||||||||
| 492 | } | ||||||||||||
| 493 | |||||||||||||
| 494 | static int check_stride_overflow(jlong stride_con, const TypeInteger* limit_t, BasicType bt) { | ||||||||||||
| 495 | if (stride_con > 0) { | ||||||||||||
| 496 | if (limit_t->lo_as_long() > (max_signed_integer(bt) - stride_con)) { | ||||||||||||
| 497 | return -1; | ||||||||||||
| 498 | } | ||||||||||||
| 499 | if (limit_t->hi_as_long() > (max_signed_integer(bt) - stride_con)) { | ||||||||||||
| 500 | return 1; | ||||||||||||
| 501 | } | ||||||||||||
| 502 | } else { | ||||||||||||
| 503 | if (limit_t->hi_as_long() < (min_signed_integer(bt) - stride_con)) { | ||||||||||||
| 504 | return -1; | ||||||||||||
| 505 | } | ||||||||||||
| 506 | if (limit_t->lo_as_long() < (min_signed_integer(bt) - stride_con)) { | ||||||||||||
| 507 | return 1; | ||||||||||||
| 508 | } | ||||||||||||
| 509 | } | ||||||||||||
| 510 | return 0; | ||||||||||||
| 511 | } | ||||||||||||
| 512 | |||||||||||||
| 513 | static bool condition_stride_ok(BoolTest::mask bt, jlong stride_con) { | ||||||||||||
| 514 | // If the condition is inverted and we will be rolling | ||||||||||||
| 515 | // through MININT to MAXINT, then bail out. | ||||||||||||
| 516 | if (bt == BoolTest::eq || // Bail out, but this loop trips at most twice! | ||||||||||||
| 517 | // Odd stride | ||||||||||||
| 518 | (bt == BoolTest::ne && stride_con != 1 && stride_con != -1) || | ||||||||||||
| 519 | // Count down loop rolls through MAXINT | ||||||||||||
| 520 | ((bt == BoolTest::le || bt == BoolTest::lt) && stride_con < 0) || | ||||||||||||
| 521 | // Count up loop rolls through MININT | ||||||||||||
| 522 | ((bt == BoolTest::ge || bt == BoolTest::gt) && stride_con > 0)) { | ||||||||||||
| 523 | return false; // Bail out | ||||||||||||
| 524 | } | ||||||||||||
| 525 | return true; | ||||||||||||
| 526 | } | ||||||||||||
| 527 | |||||||||||||
| 528 | Node* PhaseIdealLoop::loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, | ||||||||||||
| 529 | BasicType bt) { | ||||||||||||
| 530 | Node* iv_as_long; | ||||||||||||
| 531 | if (bt == T_LONG) { | ||||||||||||
| 532 | iv_as_long = new ConvI2LNode(inner_iv, TypeLong::INT); | ||||||||||||
| 533 | register_new_node(iv_as_long, inner_head); | ||||||||||||
| 534 | } else { | ||||||||||||
| 535 | iv_as_long = inner_iv; | ||||||||||||
| 536 | } | ||||||||||||
| 537 | Node* iv_replacement = AddNode::make(outer_phi, iv_as_long, bt); | ||||||||||||
| 538 | register_new_node(iv_replacement, inner_head); | ||||||||||||
| 539 | for (DUIterator_Last imin, i = iv_to_replace->last_outs(imin); i >= imin;) { | ||||||||||||
| 540 | Node* u = iv_to_replace->last_out(i); | ||||||||||||
| 541 | #ifdef ASSERT1 | ||||||||||||
| 542 | if (!is_dominator(inner_head, ctrl_or_self(u))) { | ||||||||||||
| 543 | assert(u->is_Phi(), "should be a Phi")do { if (!(u->is_Phi())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 543, "assert(" "u->is_Phi()" ") failed", "should be a Phi" ); ::breakpoint(); } } while (0); | ||||||||||||
| 544 | for (uint j = 1; j < u->req(); j++) { | ||||||||||||
| 545 | if (u->in(j) == iv_to_replace) { | ||||||||||||
| 546 | assert(is_dominator(inner_head, u->in(0)->in(j)), "iv use above loop?")do { if (!(is_dominator(inner_head, u->in(0)->in(j)))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 546, "assert(" "is_dominator(inner_head, u->in(0)->in(j))" ") failed", "iv use above loop?"); ::breakpoint(); } } while (0); | ||||||||||||
| 547 | } | ||||||||||||
| 548 | } | ||||||||||||
| 549 | } | ||||||||||||
| 550 | #endif | ||||||||||||
| 551 | _igvn.rehash_node_delayed(u); | ||||||||||||
| 552 | int nb = u->replace_edge(iv_to_replace, iv_replacement, &_igvn); | ||||||||||||
| 553 | i -= nb; | ||||||||||||
| 554 | } | ||||||||||||
| 555 | return iv_replacement; | ||||||||||||
| 556 | } | ||||||||||||
| 557 | |||||||||||||
| 558 | void PhaseIdealLoop::add_empty_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt) { | ||||||||||||
| 559 | if (!C->too_many_traps(reason)) { | ||||||||||||
| 560 | Node *cont = _igvn.intcon(1); | ||||||||||||
| 561 | Node* opq = new Opaque1Node(C, cont); | ||||||||||||
| 562 | _igvn.register_new_node_with_optimizer(opq); | ||||||||||||
| 563 | Node *bol = new Conv2BNode(opq); | ||||||||||||
| 564 | _igvn.register_new_node_with_optimizer(bol); | ||||||||||||
| 565 | set_subtree_ctrl(bol, false); | ||||||||||||
| 566 | IfNode* iff = new IfNode(inner_head->in(LoopNode::EntryControl), bol, PROB_MAX(1.0f-(1e-6f)), COUNT_UNKNOWN(-1.0f)); | ||||||||||||
| 567 | register_control(iff, loop, inner_head->in(LoopNode::EntryControl)); | ||||||||||||
| 568 | Node* iffalse = new IfFalseNode(iff); | ||||||||||||
| 569 | register_control(iffalse, _ltree_root, iff); | ||||||||||||
| 570 | Node* iftrue = new IfTrueNode(iff); | ||||||||||||
| 571 | register_control(iftrue, loop, iff); | ||||||||||||
| 572 | C->add_predicate_opaq(opq); | ||||||||||||
| 573 | |||||||||||||
| 574 | int trap_request = Deoptimization::make_trap_request(reason, Deoptimization::Action_maybe_recompile); | ||||||||||||
| 575 | address call_addr = SharedRuntime::uncommon_trap_blob()->entry_point(); | ||||||||||||
| 576 | const TypePtr* no_memory_effects = NULL__null; | ||||||||||||
| 577 | JVMState* jvms = sfpt->jvms(); | ||||||||||||
| 578 | CallNode* unc = new CallStaticJavaNode(OptoRuntime::uncommon_trap_Type(), call_addr, "uncommon_trap", | ||||||||||||
| 579 | no_memory_effects); | ||||||||||||
| 580 | |||||||||||||
| 581 | Node* mem = NULL__null; | ||||||||||||
| 582 | Node* i_o = NULL__null; | ||||||||||||
| 583 | if (sfpt->is_Call()) { | ||||||||||||
| 584 | mem = sfpt->proj_out(TypeFunc::Memory); | ||||||||||||
| 585 | i_o = sfpt->proj_out(TypeFunc::I_O); | ||||||||||||
| 586 | } else { | ||||||||||||
| 587 | mem = sfpt->memory(); | ||||||||||||
| 588 | i_o = sfpt->i_o(); | ||||||||||||
| 589 | } | ||||||||||||
| 590 | |||||||||||||
| 591 | Node *frame = new ParmNode(C->start(), TypeFunc::FramePtr); | ||||||||||||
| 592 | register_new_node(frame, C->start()); | ||||||||||||
| 593 | Node *ret = new ParmNode(C->start(), TypeFunc::ReturnAdr); | ||||||||||||
| 594 | register_new_node(ret, C->start()); | ||||||||||||
| 595 | |||||||||||||
| 596 | unc->init_req(TypeFunc::Control, iffalse); | ||||||||||||
| 597 | unc->init_req(TypeFunc::I_O, i_o); | ||||||||||||
| 598 | unc->init_req(TypeFunc::Memory, mem); // may gc ptrs | ||||||||||||
| 599 | unc->init_req(TypeFunc::FramePtr, frame); | ||||||||||||
| 600 | unc->init_req(TypeFunc::ReturnAdr, ret); | ||||||||||||
| 601 | unc->init_req(TypeFunc::Parms+0, _igvn.intcon(trap_request)); | ||||||||||||
| 602 | unc->set_cnt(PROB_UNLIKELY_MAG(4)(1e-4f)); | ||||||||||||
| 603 | unc->copy_call_debug_info(&_igvn, sfpt); | ||||||||||||
| 604 | |||||||||||||
| 605 | for (uint i = TypeFunc::Parms; i < unc->req(); i++) { | ||||||||||||
| 606 | set_subtree_ctrl(unc->in(i), false); | ||||||||||||
| 607 | } | ||||||||||||
| 608 | register_control(unc, _ltree_root, iffalse); | ||||||||||||
| 609 | |||||||||||||
| 610 | Node* ctrl = new ProjNode(unc, TypeFunc::Control); | ||||||||||||
| 611 | register_control(ctrl, _ltree_root, unc); | ||||||||||||
| 612 | Node* halt = new HaltNode(ctrl, frame, "uncommon trap returned which should never happen" PRODUCT_ONLY(COMMA /*reachable*/false)); | ||||||||||||
| 613 | register_control(halt, _ltree_root, ctrl); | ||||||||||||
| 614 | C->root()->add_req(halt); | ||||||||||||
| 615 | |||||||||||||
| 616 | _igvn.replace_input_of(inner_head, LoopNode::EntryControl, iftrue); | ||||||||||||
| 617 | set_idom(inner_head, iftrue, dom_depth(inner_head)); | ||||||||||||
| 618 | } | ||||||||||||
| 619 | } | ||||||||||||
| 620 | |||||||||||||
| 621 | // Find a safepoint node that dominates the back edge. We need a | ||||||||||||
| 622 | // SafePointNode so we can use its jvm state to create empty | ||||||||||||
| 623 | // predicates. | ||||||||||||
| 624 | static bool no_side_effect_since_safepoint(Compile* C, Node* x, Node* mem, MergeMemNode* mm) { | ||||||||||||
| 625 | SafePointNode* safepoint = NULL__null; | ||||||||||||
| 626 | for (DUIterator_Fast imax, i = x->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 627 | Node* u = x->fast_out(i); | ||||||||||||
| 628 | if (u->is_Phi() && u->bottom_type() == Type::MEMORY) { | ||||||||||||
| 629 | Node* m = u->in(LoopNode::LoopBackControl); | ||||||||||||
| 630 | if (u->adr_type() == TypePtr::BOTTOM) { | ||||||||||||
| 631 | if (m->is_MergeMem() && mem->is_MergeMem()) { | ||||||||||||
| 632 | if (m != mem DEBUG_ONLY(|| true)|| true) { | ||||||||||||
| 633 | for (MergeMemStream mms(m->as_MergeMem(), mem->as_MergeMem()); mms.next_non_empty2(); ) { | ||||||||||||
| 634 | if (!mms.is_empty()) { | ||||||||||||
| 635 | if (mms.memory() != mms.memory2()) { | ||||||||||||
| 636 | return false; | ||||||||||||
| 637 | } | ||||||||||||
| 638 | #ifdef ASSERT1 | ||||||||||||
| 639 | if (mms.alias_idx() != Compile::AliasIdxBot) { | ||||||||||||
| 640 | mm->set_memory_at(mms.alias_idx(), mem->as_MergeMem()->base_memory()); | ||||||||||||
| 641 | } | ||||||||||||
| 642 | #endif | ||||||||||||
| 643 | } | ||||||||||||
| 644 | } | ||||||||||||
| 645 | } | ||||||||||||
| 646 | } else if (mem->is_MergeMem()) { | ||||||||||||
| 647 | if (m != mem->as_MergeMem()->base_memory()) { | ||||||||||||
| 648 | return false; | ||||||||||||
| 649 | } | ||||||||||||
| 650 | } else { | ||||||||||||
| 651 | return false; | ||||||||||||
| 652 | } | ||||||||||||
| 653 | } else { | ||||||||||||
| 654 | if (mem->is_MergeMem()) { | ||||||||||||
| 655 | if (m != mem->as_MergeMem()->memory_at(C->get_alias_index(u->adr_type()))) { | ||||||||||||
| 656 | return false; | ||||||||||||
| 657 | } | ||||||||||||
| 658 | #ifdef ASSERT1 | ||||||||||||
| 659 | mm->set_memory_at(C->get_alias_index(u->adr_type()), mem->as_MergeMem()->base_memory()); | ||||||||||||
| 660 | #endif | ||||||||||||
| 661 | } else { | ||||||||||||
| 662 | if (m != mem) { | ||||||||||||
| 663 | return false; | ||||||||||||
| 664 | } | ||||||||||||
| 665 | } | ||||||||||||
| 666 | } | ||||||||||||
| 667 | } | ||||||||||||
| 668 | } | ||||||||||||
| 669 | return true; | ||||||||||||
| 670 | } | ||||||||||||
| 671 | |||||||||||||
| 672 | SafePointNode* PhaseIdealLoop::find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop) { | ||||||||||||
| 673 | IfNode* exit_test = back_control->in(0)->as_If(); | ||||||||||||
| 674 | SafePointNode* safepoint = NULL__null; | ||||||||||||
| 675 | if (exit_test->in(0)->is_SafePoint() && exit_test->in(0)->outcnt() == 1) { | ||||||||||||
| 676 | safepoint = exit_test->in(0)->as_SafePoint(); | ||||||||||||
| 677 | } else { | ||||||||||||
| 678 | Node* c = back_control; | ||||||||||||
| 679 | while (c != x && c->Opcode() != Op_SafePoint) { | ||||||||||||
| 680 | c = idom(c); | ||||||||||||
| 681 | } | ||||||||||||
| 682 | |||||||||||||
| 683 | if (c->Opcode() == Op_SafePoint) { | ||||||||||||
| 684 | safepoint = c->as_SafePoint(); | ||||||||||||
| 685 | } | ||||||||||||
| 686 | |||||||||||||
| 687 | if (safepoint == NULL__null) { | ||||||||||||
| 688 | return NULL__null; | ||||||||||||
| 689 | } | ||||||||||||
| 690 | |||||||||||||
| 691 | Node* mem = safepoint->in(TypeFunc::Memory); | ||||||||||||
| 692 | |||||||||||||
| 693 | // We can only use that safepoint if there's not side effect | ||||||||||||
| 694 | // between the backedge and the safepoint. | ||||||||||||
| 695 | |||||||||||||
| 696 | // mm is used for book keeping | ||||||||||||
| 697 | MergeMemNode* mm = NULL__null; | ||||||||||||
| 698 | #ifdef ASSERT1 | ||||||||||||
| 699 | if (mem->is_MergeMem()) { | ||||||||||||
| 700 | mm = mem->clone()->as_MergeMem(); | ||||||||||||
| 701 | _igvn._worklist.push(mm); | ||||||||||||
| 702 | for (MergeMemStream mms(mem->as_MergeMem()); mms.next_non_empty(); ) { | ||||||||||||
| 703 | if (mms.alias_idx() != Compile::AliasIdxBot && loop != get_loop(ctrl_or_self(mms.memory()))) { | ||||||||||||
| 704 | mm->set_memory_at(mms.alias_idx(), mem->as_MergeMem()->base_memory()); | ||||||||||||
| 705 | } | ||||||||||||
| 706 | } | ||||||||||||
| 707 | } | ||||||||||||
| 708 | #endif | ||||||||||||
| 709 | if (!no_side_effect_since_safepoint(C, x, mem, mm)) { | ||||||||||||
| 710 | safepoint = NULL__null; | ||||||||||||
| 711 | } else { | ||||||||||||
| 712 | assert(mm == NULL|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory(), "all memory state should have been processed")do { if (!(mm == __null|| _igvn.transform(mm) == mem->as_MergeMem ()->base_memory())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 712, "assert(" "mm == __null|| _igvn.transform(mm) == mem->as_MergeMem()->base_memory()" ") failed", "all memory state should have been processed"); :: breakpoint(); } } while (0); | ||||||||||||
| 713 | } | ||||||||||||
| 714 | #ifdef ASSERT1 | ||||||||||||
| 715 | if (mm != NULL__null) { | ||||||||||||
| 716 | _igvn.remove_dead_node(mm); | ||||||||||||
| 717 | } | ||||||||||||
| 718 | #endif | ||||||||||||
| 719 | } | ||||||||||||
| 720 | return safepoint; | ||||||||||||
| 721 | } | ||||||||||||
| 722 | |||||||||||||
| 723 | // If the loop has the shape of a counted loop but with a long | ||||||||||||
| 724 | // induction variable, transform the loop in a loop nest: an inner | ||||||||||||
| 725 | // loop that iterates for at most max int iterations with an integer | ||||||||||||
| 726 | // induction variable and an outer loop that iterates over the full | ||||||||||||
| 727 | // range of long values from the initial loop in (at most) max int | ||||||||||||
| 728 | // steps. That is: | ||||||||||||
| 729 | // | ||||||||||||
| 730 | // x: for (long phi = init; phi < limit; phi += stride) { | ||||||||||||
| 731 | // // phi := Phi(L, init, incr) | ||||||||||||
| 732 | // // incr := AddL(phi, longcon(stride)) | ||||||||||||
| 733 | // long incr = phi + stride; | ||||||||||||
| 734 | // ... use phi and incr ... | ||||||||||||
| 735 | // } | ||||||||||||
| 736 | // | ||||||||||||
| 737 | // OR: | ||||||||||||
| 738 | // | ||||||||||||
| 739 | // x: for (long phi = init; (phi += stride) < limit; ) { | ||||||||||||
| 740 | // // phi := Phi(L, AddL(init, stride), incr) | ||||||||||||
| 741 | // // incr := AddL(phi, longcon(stride)) | ||||||||||||
| 742 | // long incr = phi + stride; | ||||||||||||
| 743 | // ... use phi and (phi + stride) ... | ||||||||||||
| 744 | // } | ||||||||||||
| 745 | // | ||||||||||||
| 746 | // ==transform=> | ||||||||||||
| 747 | // | ||||||||||||
| 748 | // const ulong inner_iters_limit = INT_MAX - stride - 1; //near 0x7FFFFFF0 | ||||||||||||
| 749 | // assert(stride <= inner_iters_limit); // else abort transform | ||||||||||||
| 750 | // assert((extralong)limit + stride <= LONG_MAX); // else deopt | ||||||||||||
| 751 | // outer_head: for (long outer_phi = init;;) { | ||||||||||||
| 752 | // // outer_phi := Phi(outer_head, init, AddL(outer_phi, I2L(inner_phi))) | ||||||||||||
| 753 | // ulong inner_iters_max = (ulong) MAX(0, ((extralong)limit + stride - outer_phi)); | ||||||||||||
| 754 | // long inner_iters_actual = MIN(inner_iters_limit, inner_iters_max); | ||||||||||||
| 755 | // assert(inner_iters_actual == (int)inner_iters_actual); | ||||||||||||
| 756 | // int inner_phi, inner_incr; | ||||||||||||
| 757 | // x: for (inner_phi = 0;; inner_phi = inner_incr) { | ||||||||||||
| 758 | // // inner_phi := Phi(x, intcon(0), inner_incr) | ||||||||||||
| 759 | // // inner_incr := AddI(inner_phi, intcon(stride)) | ||||||||||||
| 760 | // inner_incr = inner_phi + stride; | ||||||||||||
| 761 | // if (inner_incr < inner_iters_actual) { | ||||||||||||
| 762 | // ... use phi=>(outer_phi+inner_phi) and incr=>(outer_phi+inner_incr) ... | ||||||||||||
| 763 | // continue; | ||||||||||||
| 764 | // } | ||||||||||||
| 765 | // else break; | ||||||||||||
| 766 | // } | ||||||||||||
| 767 | // if ((outer_phi+inner_phi) < limit) //OR (outer_phi+inner_incr) < limit | ||||||||||||
| 768 | // continue; | ||||||||||||
| 769 | // else break; | ||||||||||||
| 770 | // } | ||||||||||||
| 771 | // | ||||||||||||
| 772 | // The same logic is used to transform an int counted loop that contains long range checks into a loop nest of 2 int | ||||||||||||
| 773 | // loops with long range checks transformed to int range checks in the inner loop. | ||||||||||||
| 774 | bool PhaseIdealLoop::create_loop_nest(IdealLoopTree* loop, Node_List &old_new) { | ||||||||||||
| 775 | Node* x = loop->_head; | ||||||||||||
| 776 | // Only for inner loops | ||||||||||||
| 777 | if (loop->_child != NULL__null || !x->is_BaseCountedLoop() || x->as_Loop()->is_loop_nest_outer_loop()) { | ||||||||||||
| 778 | return false; | ||||||||||||
| 779 | } | ||||||||||||
| 780 | |||||||||||||
| 781 | if (x->is_CountedLoop() && !x->as_CountedLoop()->is_main_loop() && !x->as_CountedLoop()->is_normal_loop()) { | ||||||||||||
| 782 | return false; | ||||||||||||
| 783 | } | ||||||||||||
| 784 | |||||||||||||
| 785 | BaseCountedLoopNode* head = x->as_BaseCountedLoop(); | ||||||||||||
| 786 | BasicType bt = x->as_BaseCountedLoop()->bt(); | ||||||||||||
| 787 | |||||||||||||
| 788 | check_counted_loop_shape(loop, x, bt); | ||||||||||||
| 789 | |||||||||||||
| 790 | #ifndef PRODUCT | ||||||||||||
| 791 | if (bt == T_LONG) { | ||||||||||||
| 792 | Atomic::inc(&_long_loop_candidates); | ||||||||||||
| 793 | } | ||||||||||||
| 794 | #endif | ||||||||||||
| 795 | |||||||||||||
| 796 | jlong stride_con = head->stride_con(); | ||||||||||||
| 797 | assert(stride_con != 0, "missed some peephole opt")do { if (!(stride_con != 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 797, "assert(" "stride_con != 0" ") failed", "missed some peephole opt" ); ::breakpoint(); } } while (0); | ||||||||||||
| 798 | // We can't iterate for more than max int at a time. | ||||||||||||
| 799 | if (stride_con != (jint)stride_con) { | ||||||||||||
| 800 | assert(bt == T_LONG, "only for long loops")do { if (!(bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 800, "assert(" "bt == T_LONG" ") failed", "only for long loops" ); ::breakpoint(); } } while (0); | ||||||||||||
| 801 | return false; | ||||||||||||
| 802 | } | ||||||||||||
| 803 | // The number of iterations for the integer count loop: guarantee no | ||||||||||||
| 804 | // overflow: max_jint - stride_con max. -1 so there's no need for a | ||||||||||||
| 805 | // loop limit check if the exit test is <= or >=. | ||||||||||||
| 806 | int iters_limit = max_jint - ABS(stride_con) - 1; | ||||||||||||
| 807 | #ifdef ASSERT1 | ||||||||||||
| 808 | if (bt == T_LONG && StressLongCountedLoop > 0) { | ||||||||||||
| 809 | iters_limit = iters_limit / StressLongCountedLoop; | ||||||||||||
| 810 | } | ||||||||||||
| 811 | #endif | ||||||||||||
| 812 | // At least 2 iterations so counted loop construction doesn't fail | ||||||||||||
| 813 | if (iters_limit/ABS(stride_con) < 2) { | ||||||||||||
| 814 | return false; | ||||||||||||
| 815 | } | ||||||||||||
| 816 | |||||||||||||
| 817 | PhiNode* phi = head->phi()->as_Phi(); | ||||||||||||
| 818 | Node* incr = head->incr(); | ||||||||||||
| 819 | |||||||||||||
| 820 | Node* back_control = head->in(LoopNode::LoopBackControl); | ||||||||||||
| 821 | |||||||||||||
| 822 | // data nodes on back branch not supported | ||||||||||||
| 823 | if (back_control->outcnt() > 1) { | ||||||||||||
| 824 | return false; | ||||||||||||
| 825 | } | ||||||||||||
| 826 | |||||||||||||
| 827 | Node* limit = head->limit(); | ||||||||||||
| 828 | // We'll need to use the loop limit before the inner loop is entered | ||||||||||||
| 829 | if (!is_dominator(get_ctrl(limit), x)) { | ||||||||||||
| 830 | return false; | ||||||||||||
| 831 | } | ||||||||||||
| 832 | |||||||||||||
| 833 | // May not have gone thru igvn yet so don't use _igvn.type(phi) (PhaseIdealLoop::is_counted_loop() sets the iv phi's type) | ||||||||||||
| 834 | const TypeInteger* phi_t = phi->bottom_type()->is_integer(bt); | ||||||||||||
| 835 | assert(phi_t->hi_as_long() >= phi_t->lo_as_long(), "dead phi?")do { if (!(phi_t->hi_as_long() >= phi_t->lo_as_long( ))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 835, "assert(" "phi_t->hi_as_long() >= phi_t->lo_as_long()" ") failed", "dead phi?"); ::breakpoint(); } } while (0); | ||||||||||||
| 836 | iters_limit = checked_cast<int>(MIN2((julong)iters_limit, (julong)(phi_t->hi_as_long() - phi_t->lo_as_long()))); | ||||||||||||
| 837 | |||||||||||||
| 838 | IfNode* exit_test = head->loopexit(); | ||||||||||||
| 839 | BoolTest::mask mask = exit_test->as_BaseCountedLoopEnd()->test_trip(); | ||||||||||||
| 840 | Node* cmp = exit_test->as_BaseCountedLoopEnd()->cmp_node(); | ||||||||||||
| 841 | |||||||||||||
| 842 | assert(back_control->Opcode() == Op_IfTrue, "wrong projection for back edge")do { if (!(back_control->Opcode() == Op_IfTrue)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 842, "assert(" "back_control->Opcode() == Op_IfTrue" ") failed" , "wrong projection for back edge"); ::breakpoint(); } } while (0); | ||||||||||||
| 843 | |||||||||||||
| 844 | Node_List range_checks; | ||||||||||||
| 845 | iters_limit = extract_long_range_checks(loop, stride_con, iters_limit, phi, range_checks); | ||||||||||||
| 846 | |||||||||||||
| 847 | if (bt == T_INT) { | ||||||||||||
| 848 | // The only purpose of creating a loop nest is to handle long range checks. If there are none, do not proceed further. | ||||||||||||
| 849 | if (range_checks.size() == 0) { | ||||||||||||
| 850 | return false; | ||||||||||||
| 851 | } | ||||||||||||
| 852 | } | ||||||||||||
| 853 | |||||||||||||
| 854 | // We need a safepoint to insert empty predicates for the inner loop. | ||||||||||||
| 855 | SafePointNode* safepoint; | ||||||||||||
| 856 | if (bt == T_INT && head->as_CountedLoop()->is_strip_mined()) { | ||||||||||||
| 857 | // Loop is strip mined: use the safepoint of the outer strip mined loop | ||||||||||||
| 858 | strip_mined_nest_back_to_counted_loop(loop, head, back_control, exit_test, safepoint); | ||||||||||||
| 859 | } else { | ||||||||||||
| 860 | safepoint = find_safepoint(back_control, x, loop); | ||||||||||||
| 861 | } | ||||||||||||
| 862 | |||||||||||||
| 863 | Node* exit_branch = exit_test->proj_out(false); | ||||||||||||
| 864 | Node* entry_control = head->in(LoopNode::EntryControl); | ||||||||||||
| 865 | |||||||||||||
| 866 | // Clone the control flow of the loop to build an outer loop | ||||||||||||
| 867 | Node* outer_back_branch = back_control->clone(); | ||||||||||||
| 868 | Node* outer_exit_test = new IfNode(exit_test->in(0), exit_test->in(1), exit_test->_prob, exit_test->_fcnt); | ||||||||||||
| 869 | Node* inner_exit_branch = exit_branch->clone(); | ||||||||||||
| 870 | |||||||||||||
| 871 | LoopNode* outer_head = new LoopNode(entry_control, outer_back_branch); | ||||||||||||
| 872 | IdealLoopTree* outer_ilt = insert_outer_loop(loop, outer_head, outer_back_branch); | ||||||||||||
| 873 | |||||||||||||
| 874 | const bool body_populated = true; | ||||||||||||
| 875 | register_control(outer_head, outer_ilt, entry_control, body_populated); | ||||||||||||
| 876 | |||||||||||||
| 877 | _igvn.register_new_node_with_optimizer(inner_exit_branch); | ||||||||||||
| 878 | set_loop(inner_exit_branch, outer_ilt); | ||||||||||||
| 879 | set_idom(inner_exit_branch, exit_test, dom_depth(exit_branch)); | ||||||||||||
| 880 | |||||||||||||
| 881 | outer_exit_test->set_req(0, inner_exit_branch); | ||||||||||||
| 882 | register_control(outer_exit_test, outer_ilt, inner_exit_branch, body_populated); | ||||||||||||
| 883 | |||||||||||||
| 884 | _igvn.replace_input_of(exit_branch, 0, outer_exit_test); | ||||||||||||
| 885 | set_idom(exit_branch, outer_exit_test, dom_depth(exit_branch)); | ||||||||||||
| 886 | |||||||||||||
| 887 | outer_back_branch->set_req(0, outer_exit_test); | ||||||||||||
| 888 | register_control(outer_back_branch, outer_ilt, outer_exit_test, body_populated); | ||||||||||||
| 889 | |||||||||||||
| 890 | _igvn.replace_input_of(x, LoopNode::EntryControl, outer_head); | ||||||||||||
| 891 | set_idom(x, outer_head, dom_depth(x)); | ||||||||||||
| 892 | |||||||||||||
| 893 | // add an iv phi to the outer loop and use it to compute the inner | ||||||||||||
| 894 | // loop iteration limit | ||||||||||||
| 895 | Node* outer_phi = phi->clone(); | ||||||||||||
| 896 | outer_phi->set_req(0, outer_head); | ||||||||||||
| 897 | register_new_node(outer_phi, outer_head); | ||||||||||||
| 898 | |||||||||||||
| 899 | Node* inner_iters_max = NULL__null; | ||||||||||||
| 900 | if (stride_con > 0) { | ||||||||||||
| 901 | inner_iters_max = MaxNode::max_diff_with_zero(limit, outer_phi, TypeInteger::bottom(bt), _igvn); | ||||||||||||
| 902 | } else { | ||||||||||||
| 903 | inner_iters_max = MaxNode::max_diff_with_zero(outer_phi, limit, TypeInteger::bottom(bt), _igvn); | ||||||||||||
| 904 | } | ||||||||||||
| 905 | |||||||||||||
| 906 | Node* inner_iters_limit = _igvn.integercon(iters_limit, bt); | ||||||||||||
| 907 | // inner_iters_max may not fit in a signed integer (iterating from | ||||||||||||
| 908 | // Long.MIN_VALUE to Long.MAX_VALUE for instance). Use an unsigned | ||||||||||||
| 909 | // min. | ||||||||||||
| 910 | Node* inner_iters_actual = MaxNode::unsigned_min(inner_iters_max, inner_iters_limit, TypeInteger::make(0, iters_limit, Type::WidenMin, bt), _igvn); | ||||||||||||
| 911 | |||||||||||||
| 912 | Node* inner_iters_actual_int; | ||||||||||||
| 913 | if (bt == T_LONG) { | ||||||||||||
| 914 | inner_iters_actual_int = new ConvL2INode(inner_iters_actual); | ||||||||||||
| 915 | _igvn.register_new_node_with_optimizer(inner_iters_actual_int); | ||||||||||||
| 916 | } else { | ||||||||||||
| 917 | inner_iters_actual_int = inner_iters_actual; | ||||||||||||
| 918 | } | ||||||||||||
| 919 | |||||||||||||
| 920 | Node* int_zero = _igvn.intcon(0); | ||||||||||||
| 921 | set_ctrl(int_zero, C->root()); | ||||||||||||
| 922 | if (stride_con < 0) { | ||||||||||||
| 923 | inner_iters_actual_int = new SubINode(int_zero, inner_iters_actual_int); | ||||||||||||
| 924 | _igvn.register_new_node_with_optimizer(inner_iters_actual_int); | ||||||||||||
| 925 | } | ||||||||||||
| 926 | |||||||||||||
| 927 | // Clone the iv data nodes as an integer iv | ||||||||||||
| 928 | Node* int_stride = _igvn.intcon(checked_cast<int>(stride_con)); | ||||||||||||
| 929 | set_ctrl(int_stride, C->root()); | ||||||||||||
| 930 | Node* inner_phi = new PhiNode(x->in(0), TypeInt::INT); | ||||||||||||
| 931 | Node* inner_incr = new AddINode(inner_phi, int_stride); | ||||||||||||
| 932 | Node* inner_cmp = NULL__null; | ||||||||||||
| 933 | inner_cmp = new CmpINode(inner_incr, inner_iters_actual_int); | ||||||||||||
| 934 | Node* inner_bol = new BoolNode(inner_cmp, exit_test->in(1)->as_Bool()->_test._test); | ||||||||||||
| 935 | inner_phi->set_req(LoopNode::EntryControl, int_zero); | ||||||||||||
| 936 | inner_phi->set_req(LoopNode::LoopBackControl, inner_incr); | ||||||||||||
| 937 | register_new_node(inner_phi, x); | ||||||||||||
| 938 | register_new_node(inner_incr, x); | ||||||||||||
| 939 | register_new_node(inner_cmp, x); | ||||||||||||
| 940 | register_new_node(inner_bol, x); | ||||||||||||
| 941 | |||||||||||||
| 942 | _igvn.replace_input_of(exit_test, 1, inner_bol); | ||||||||||||
| 943 | |||||||||||||
| 944 | // Clone inner loop phis to outer loop | ||||||||||||
| 945 | for (uint i = 0; i < head->outcnt(); i++) { | ||||||||||||
| 946 | Node* u = head->raw_out(i); | ||||||||||||
| 947 | if (u->is_Phi() && u != inner_phi && u != phi) { | ||||||||||||
| 948 | assert(u->in(0) == head, "inconsistent")do { if (!(u->in(0) == head)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 948, "assert(" "u->in(0) == head" ") failed", "inconsistent" ); ::breakpoint(); } } while (0); | ||||||||||||
| 949 | Node* clone = u->clone(); | ||||||||||||
| 950 | clone->set_req(0, outer_head); | ||||||||||||
| 951 | register_new_node(clone, outer_head); | ||||||||||||
| 952 | _igvn.replace_input_of(u, LoopNode::EntryControl, clone); | ||||||||||||
| 953 | } | ||||||||||||
| 954 | } | ||||||||||||
| 955 | |||||||||||||
| 956 | // Replace inner loop long iv phi as inner loop int iv phi + outer | ||||||||||||
| 957 | // loop iv phi | ||||||||||||
| 958 | Node* iv_add = loop_nest_replace_iv(phi, inner_phi, outer_phi, head, bt); | ||||||||||||
| 959 | |||||||||||||
| 960 | // Replace inner loop long iv incr with inner loop int incr + outer | ||||||||||||
| 961 | // loop iv phi | ||||||||||||
| 962 | loop_nest_replace_iv(incr, inner_incr, outer_phi, head, bt); | ||||||||||||
| 963 | |||||||||||||
| 964 | set_subtree_ctrl(inner_iters_actual_int, body_populated); | ||||||||||||
| 965 | |||||||||||||
| 966 | LoopNode* inner_head = create_inner_head(loop, head, exit_test); | ||||||||||||
| 967 | |||||||||||||
| 968 | // Summary of steps from inital loop to loop nest: | ||||||||||||
| 969 | // | ||||||||||||
| 970 | // == old IR nodes => | ||||||||||||
| 971 | // | ||||||||||||
| 972 | // entry_control: {...} | ||||||||||||
| 973 | // x: | ||||||||||||
| 974 | // for (long phi = init;;) { | ||||||||||||
| 975 | // // phi := Phi(x, init, incr) | ||||||||||||
| 976 | // // incr := AddL(phi, longcon(stride)) | ||||||||||||
| 977 | // exit_test: | ||||||||||||
| 978 | // if (phi < limit) | ||||||||||||
| 979 | // back_control: fallthrough; | ||||||||||||
| 980 | // else | ||||||||||||
| 981 | // exit_branch: break; | ||||||||||||
| 982 | // long incr = phi + stride; | ||||||||||||
| 983 | // ... use phi and incr ... | ||||||||||||
| 984 | // phi = incr; | ||||||||||||
| 985 | // } | ||||||||||||
| 986 | // | ||||||||||||
| 987 | // == new IR nodes (just before final peel) => | ||||||||||||
| 988 | // | ||||||||||||
| 989 | // entry_control: {...} | ||||||||||||
| 990 | // long adjusted_limit = limit + stride; //because phi_incr != NULL | ||||||||||||
| 991 | // assert(!limit_check_required || (extralong)limit + stride == adjusted_limit); // else deopt | ||||||||||||
| 992 | // ulong inner_iters_limit = max_jint - ABS(stride) - 1; //near 0x7FFFFFF0 | ||||||||||||
| 993 | // outer_head: | ||||||||||||
| 994 | // for (long outer_phi = init;;) { | ||||||||||||
| 995 | // // outer_phi := phi->clone(), in(0):=outer_head, => Phi(outer_head, init, incr) | ||||||||||||
| 996 | // // REPLACE phi => AddL(outer_phi, I2L(inner_phi)) | ||||||||||||
| 997 | // // REPLACE incr => AddL(outer_phi, I2L(inner_incr)) | ||||||||||||
| 998 | // // SO THAT outer_phi := Phi(outer_head, init, AddL(outer_phi, I2L(inner_incr))) | ||||||||||||
| 999 | // ulong inner_iters_max = (ulong) MAX(0, ((extralong)adjusted_limit - outer_phi) * SGN(stride)); | ||||||||||||
| 1000 | // int inner_iters_actual_int = (int) MIN(inner_iters_limit, inner_iters_max) * SGN(stride); | ||||||||||||
| 1001 | // inner_head: x: //in(1) := outer_head | ||||||||||||
| 1002 | // int inner_phi; | ||||||||||||
| 1003 | // for (inner_phi = 0;;) { | ||||||||||||
| 1004 | // // inner_phi := Phi(x, intcon(0), inner_phi + stride) | ||||||||||||
| 1005 | // int inner_incr = inner_phi + stride; | ||||||||||||
| 1006 | // bool inner_bol = (inner_incr < inner_iters_actual_int); | ||||||||||||
| 1007 | // exit_test: //exit_test->in(1) := inner_bol; | ||||||||||||
| 1008 | // if (inner_bol) // WAS (phi < limit) | ||||||||||||
| 1009 | // back_control: fallthrough; | ||||||||||||
| 1010 | // else | ||||||||||||
| 1011 | // inner_exit_branch: break; //exit_branch->clone() | ||||||||||||
| 1012 | // ... use phi=>(outer_phi+inner_phi) and incr=>(outer_phi+inner_incr) ... | ||||||||||||
| 1013 | // inner_phi = inner_phi + stride; // inner_incr | ||||||||||||
| 1014 | // } | ||||||||||||
| 1015 | // outer_exit_test: //exit_test->clone(), in(0):=inner_exit_branch | ||||||||||||
| 1016 | // if ((outer_phi+inner_phi) < limit) // WAS (phi < limit) | ||||||||||||
| 1017 | // outer_back_branch: fallthrough; //back_control->clone(), in(0):=outer_exit_test | ||||||||||||
| 1018 | // else | ||||||||||||
| 1019 | // exit_branch: break; //in(0) := outer_exit_test | ||||||||||||
| 1020 | // } | ||||||||||||
| 1021 | |||||||||||||
| 1022 | if (bt == T_INT) { | ||||||||||||
| 1023 | outer_phi = new ConvI2LNode(outer_phi); | ||||||||||||
| 1024 | register_new_node(outer_phi, outer_head); | ||||||||||||
| 1025 | } | ||||||||||||
| 1026 | |||||||||||||
| 1027 | transform_long_range_checks(checked_cast<int>(stride_con), range_checks, outer_phi, inner_iters_actual_int, | ||||||||||||
| 1028 | inner_phi, iv_add, inner_head); | ||||||||||||
| 1029 | // Peel one iteration of the loop and use the safepoint at the end | ||||||||||||
| 1030 | // of the peeled iteration to insert empty predicates. If no well | ||||||||||||
| 1031 | // positioned safepoint peel to guarantee a safepoint in the outer | ||||||||||||
| 1032 | // loop. | ||||||||||||
| 1033 | if (safepoint != NULL__null || !loop->_has_call) { | ||||||||||||
| 1034 | old_new.clear(); | ||||||||||||
| 1035 | do_peeling(loop, old_new); | ||||||||||||
| 1036 | } else { | ||||||||||||
| 1037 | C->set_major_progress(); | ||||||||||||
| 1038 | } | ||||||||||||
| 1039 | |||||||||||||
| 1040 | if (safepoint != NULL__null) { | ||||||||||||
| 1041 | SafePointNode* cloned_sfpt = old_new[safepoint->_idx]->as_SafePoint(); | ||||||||||||
| 1042 | |||||||||||||
| 1043 | if (UseLoopPredicate) { | ||||||||||||
| 1044 | add_empty_predicate(Deoptimization::Reason_predicate, inner_head, outer_ilt, cloned_sfpt); | ||||||||||||
| 1045 | } | ||||||||||||
| 1046 | if (UseProfiledLoopPredicate) { | ||||||||||||
| 1047 | add_empty_predicate(Deoptimization::Reason_profile_predicate, inner_head, outer_ilt, cloned_sfpt); | ||||||||||||
| 1048 | } | ||||||||||||
| 1049 | add_empty_predicate(Deoptimization::Reason_loop_limit_check, inner_head, outer_ilt, cloned_sfpt); | ||||||||||||
| 1050 | } | ||||||||||||
| 1051 | |||||||||||||
| 1052 | #ifndef PRODUCT | ||||||||||||
| 1053 | if (bt == T_LONG) { | ||||||||||||
| 1054 | Atomic::inc(&_long_loop_nests); | ||||||||||||
| 1055 | } | ||||||||||||
| 1056 | #endif | ||||||||||||
| 1057 | |||||||||||||
| 1058 | inner_head->mark_loop_nest_inner_loop(); | ||||||||||||
| 1059 | outer_head->mark_loop_nest_outer_loop(); | ||||||||||||
| 1060 | |||||||||||||
| 1061 | return true; | ||||||||||||
| 1062 | } | ||||||||||||
| 1063 | |||||||||||||
| 1064 | // Convert the strip mined loop nest back to a single loop with the safepoint right before the loop exit test | ||||||||||||
| 1065 | void PhaseIdealLoop::strip_mined_nest_back_to_counted_loop(IdealLoopTree* loop, const BaseCountedLoopNode* head, | ||||||||||||
| 1066 | Node* back_control, IfNode*& exit_test, | ||||||||||||
| 1067 | SafePointNode*& safepoint) { | ||||||||||||
| 1068 | CountedLoopNode* cl = head->as_CountedLoop(); | ||||||||||||
| 1069 | cl->verify_strip_mined(1); | ||||||||||||
| 1070 | safepoint = cl->outer_safepoint(); | ||||||||||||
| 1071 | CountedLoopEndNode* cle = cl->loopexit(); | ||||||||||||
| 1072 | OuterStripMinedLoopNode* outer_head = cl->outer_loop(); | ||||||||||||
| 1073 | OuterStripMinedLoopEndNode* outer_end = cl->outer_loop_end(); | ||||||||||||
| 1074 | |||||||||||||
| 1075 | cl->clear_strip_mined(); | ||||||||||||
| 1076 | |||||||||||||
| 1077 | _igvn.replace_input_of(cl, LoopNode::EntryControl, outer_head->in(LoopNode::EntryControl)); | ||||||||||||
| 1078 | _igvn.replace_input_of(outer_head, LoopNode::EntryControl, C->top()); | ||||||||||||
| 1079 | set_idom(cl, cl->in(LoopNode::EntryControl), dom_depth(cl)); | ||||||||||||
| 1080 | |||||||||||||
| 1081 | Node* exit_bol = cle->in(1); | ||||||||||||
| 1082 | Node *zero = _igvn.intcon(0); | ||||||||||||
| 1083 | set_ctrl(zero, C->root()); | ||||||||||||
| 1084 | _igvn.replace_input_of(cle, 1, zero); | ||||||||||||
| 1085 | |||||||||||||
| 1086 | _igvn.replace_input_of(outer_end, 1, exit_bol); | ||||||||||||
| 1087 | |||||||||||||
| 1088 | assert(outer_head->in(LoopNode::LoopBackControl)->in(0) == outer_end, "")do { if (!(outer_head->in(LoopNode::LoopBackControl)->in (0) == outer_end)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1088, "assert(" "outer_head->in(LoopNode::LoopBackControl)->in(0) == outer_end" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 1089 | _igvn.replace_input_of(outer_head->in(LoopNode::LoopBackControl), 0, C->top()); | ||||||||||||
| 1090 | _igvn.replace_input_of(back_control, 0, outer_end); | ||||||||||||
| 1091 | set_idom(back_control, outer_end, dom_depth(outer_end) + 1); | ||||||||||||
| 1092 | |||||||||||||
| 1093 | Unique_Node_List wq; | ||||||||||||
| 1094 | wq.push(safepoint); | ||||||||||||
| 1095 | |||||||||||||
| 1096 | IdealLoopTree* outer_loop_ilt = get_loop(outer_head); | ||||||||||||
| 1097 | |||||||||||||
| 1098 | for (uint i = 0; i < wq.size(); i++) { | ||||||||||||
| 1099 | Node* n = wq.at(i); | ||||||||||||
| 1100 | for (uint j = 0; j < n->req(); ++j) { | ||||||||||||
| 1101 | Node* in = n->in(j); | ||||||||||||
| 1102 | if (in == NULL__null || in->is_CFG()) { | ||||||||||||
| 1103 | continue; | ||||||||||||
| 1104 | } | ||||||||||||
| 1105 | if (get_loop(get_ctrl(in)) != outer_loop_ilt) { | ||||||||||||
| 1106 | continue; | ||||||||||||
| 1107 | } | ||||||||||||
| 1108 | assert(!loop->_body.contains(in), "")do { if (!(!loop->_body.contains(in))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1108, "assert(" "!loop->_body.contains(in)" ") failed", "" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1109 | loop->_body.push(in); | ||||||||||||
| 1110 | wq.push(in); | ||||||||||||
| 1111 | } | ||||||||||||
| 1112 | } | ||||||||||||
| 1113 | |||||||||||||
| 1114 | set_loop(outer_end, loop); | ||||||||||||
| 1115 | loop->_body.push(outer_end); | ||||||||||||
| 1116 | set_loop(safepoint, loop); | ||||||||||||
| 1117 | loop->_body.push(safepoint); | ||||||||||||
| 1118 | set_loop(safepoint->in(0), loop); | ||||||||||||
| 1119 | loop->_body.push(safepoint->in(0)); | ||||||||||||
| 1120 | |||||||||||||
| 1121 | exit_test = outer_end; | ||||||||||||
| 1122 | |||||||||||||
| 1123 | outer_loop_ilt->_tail = C->top(); | ||||||||||||
| 1124 | } | ||||||||||||
| 1125 | |||||||||||||
| 1126 | int PhaseIdealLoop::extract_long_range_checks(const IdealLoopTree* loop, jlong stride_con, int iters_limit, PhiNode* phi, | ||||||||||||
| 1127 | Node_List& range_checks) { | ||||||||||||
| 1128 | if (stride_con < 0) { // only for stride_con > 0 && scale > 0 for now | ||||||||||||
| 1129 | return iters_limit; | ||||||||||||
| 1130 | } | ||||||||||||
| 1131 | const jlong min_iters = 2; | ||||||||||||
| 1132 | jlong reduced_iters_limit = iters_limit; | ||||||||||||
| 1133 | jlong original_iters_limit = iters_limit; | ||||||||||||
| 1134 | for (uint i = 0; i < loop->_body.size(); i++) { | ||||||||||||
| 1135 | Node* c = loop->_body.at(i); | ||||||||||||
| 1136 | if (c->is_IfProj() && c->in(0)->is_RangeCheck()) { | ||||||||||||
| 1137 | CallStaticJavaNode* call = c->as_IfProj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); | ||||||||||||
| 1138 | if (call != NULL__null) { | ||||||||||||
| 1139 | Node* range = NULL__null; | ||||||||||||
| 1140 | Node* offset = NULL__null; | ||||||||||||
| 1141 | jlong scale = 0; | ||||||||||||
| 1142 | RangeCheckNode* rc = c->in(0)->as_RangeCheck(); | ||||||||||||
| 1143 | if (loop->is_range_check_if(rc, this, T_LONG, phi, range, offset, scale) && | ||||||||||||
| 1144 | loop->is_invariant(range) && loop->is_invariant(offset) && | ||||||||||||
| 1145 | scale > 0 && // only for stride_con > 0 && scale > 0 for now | ||||||||||||
| 1146 | original_iters_limit / ABS(scale * stride_con) >= min_iters) { | ||||||||||||
| 1147 | reduced_iters_limit = MIN2(reduced_iters_limit, original_iters_limit/ABS(scale)); | ||||||||||||
| 1148 | range_checks.push(c); | ||||||||||||
| 1149 | } | ||||||||||||
| 1150 | } | ||||||||||||
| 1151 | } | ||||||||||||
| 1152 | } | ||||||||||||
| 1153 | |||||||||||||
| 1154 | return checked_cast<int>(reduced_iters_limit); | ||||||||||||
| 1155 | } | ||||||||||||
| 1156 | |||||||||||||
| 1157 | // One execution of the inner loop covers a sub-range of the entire iteration range of the loop: [A,Z), aka [A=init, | ||||||||||||
| 1158 | // Z=limit). If the loop has at least one trip (which is the case here), the iteration variable i always takes A as its | ||||||||||||
| 1159 | // first value, followed by A+S (S is the stride), next A+2S, etc. The limit is exclusive, so that the final value B of | ||||||||||||
| 1160 | // i is never Z. It will be B=Z-1 if S=1, or B=Z+1 if S=-1. If |S|>1 the formula for the last value requires a floor | ||||||||||||
| 1161 | // operation, specifically B=floor((Z-sgn(S)-A)/S)*S+A. Thus i ranges as i:[A,B] or i:[A,Z) or i:[A,Z-U) for some U<S. | ||||||||||||
| 1162 | |||||||||||||
| 1163 | // N.B. We handle only the case of positive S currently, so comments about S<0 are not operative at present. Also, | ||||||||||||
| 1164 | // we only support positive index scale value (K > 0) to simplify the logic for clamping 32-bit bounds (L_2, R_2). | ||||||||||||
| 1165 | // For restrictions on S and K, see the guards in extract_long_range_checks. | ||||||||||||
| 1166 | |||||||||||||
| 1167 | // Within the loop there may be many range checks. Each such range check (R.C.) is of the form 0 <= i*K+L < R, where K | ||||||||||||
| 1168 | // is a scale factor applied to the loop iteration variable i, and L is some offset; K, L, and R are loop-invariant. | ||||||||||||
| 1169 | // Because R is never negative, this check can always be simplified to an unsigned check i*K+L <u R. | ||||||||||||
| 1170 | |||||||||||||
| 1171 | // When a long loop over a 64-bit variable i (outer_iv) is decomposed into a series of shorter sub-loops over a 32-bit | ||||||||||||
| 1172 | // variable j (inner_iv), j ranges over a shorter interval j:[0,Z_2), where the limit is chosen to prevent various cases | ||||||||||||
| 1173 | // of 32-bit overflow (including multiplications j*K below). In the sub-loop the logical value i is offset from j by a | ||||||||||||
| 1174 | // 64-bit constant C, so i ranges in i:C+[0,Z_2). | ||||||||||||
| 1175 | |||||||||||||
| 1176 | // The union of all the C+[0,Z_2) ranges from the sub-loops must be identical to the whole range [A,B]. Assuming S>0, | ||||||||||||
| 1177 | // the first C must be A itself, and the next C value is the previous C+Z_2. In each sub-loop, j counts up from zero | ||||||||||||
| 1178 | // and exits just before i=C+Z_2. | ||||||||||||
| 1179 | |||||||||||||
| 1180 | // (N.B. If S<0 the formulas are different, because all the loops count downward.) | ||||||||||||
| 1181 | |||||||||||||
| 1182 | // Returning to range checks, we see that each i*K+L <u R expands to (C+j)*K+L <u R, or j*K+Q <u R, where Q=(C*K+L). | ||||||||||||
| 1183 | // (Recall that K and L and R are loop-invariant scale, offset and range values for a particular R.C.) This is still a | ||||||||||||
| 1184 | // 64-bit comparison, so the range check elimination logic will not apply to it. (The R.C.E. transforms operate only on | ||||||||||||
| 1185 | // 32-bit indexes and comparisons, because they use 64-bit temporary values to avoid overflow; see | ||||||||||||
| 1186 | // PhaseIdealLoop::add_constraint.) | ||||||||||||
| 1187 | |||||||||||||
| 1188 | // We must transform this comparison so that it gets the same answer, but by means of a 32-bit R.C. (using j not i) of | ||||||||||||
| 1189 | // the form j*K+L_2 <u32 R_2. Note that L_2 and R_2 must be loop-invariant, but only with respect to the sub-loop. Thus, the | ||||||||||||
| 1190 | // problem reduces to computing values for L_2 and R_2 (for each R.C. in the loop) in the loop header for the sub-loop. | ||||||||||||
| 1191 | // Then the standard R.C.E. transforms can take those as inputs and further compute the necessary minimum and maximum | ||||||||||||
| 1192 | // values for the 32-bit counter j within which the range checks can be eliminated. | ||||||||||||
| 1193 | |||||||||||||
| 1194 | // So, given j*K+Q <u R, we need to find some j*K+L_2 <u32 R_2, where L_2 and R_2 fit in 32 bits, and the 32-bit operations do | ||||||||||||
| 1195 | // not overflow. We also need to cover the cases where i*K+L (= j*K+Q) overflows to a 64-bit negative, since that is | ||||||||||||
| 1196 | // allowed as an input to the R.C., as long as the R.C. as a whole fails. | ||||||||||||
| 1197 | |||||||||||||
| 1198 | // If 32-bit multiplication j*K might overflow, we adjust the sub-loop limit Z_2 closer to zero to reduce j's range. | ||||||||||||
| 1199 | |||||||||||||
| 1200 | // For each R.C. j*K+Q <u32 R, the range of mathematical values of j*K+Q in the sub-loop is [Q_min, Q_max), where | ||||||||||||
| 1201 | // Q_min=Q and Q_max=Z_2*K+Q. Making the upper limit Q_max be exclusive helps it integrate correctly with the strict | ||||||||||||
| 1202 | // comparisons against R and R_2. Sometimes a very high R will be replaced by an R_2 derived from the more moderate | ||||||||||||
| 1203 | // Q_max, and replacing one exclusive limit by another exclusive limit avoids off-by-one complexities. | ||||||||||||
| 1204 | |||||||||||||
| 1205 | // N.B. If (S*K)<0 then the formulas for Q_min and Q_max may differ; the values may need to be swapped and adjusted to | ||||||||||||
| 1206 | // the correct type of bound (inclusive or exclusive). | ||||||||||||
| 1207 | |||||||||||||
| 1208 | // Case A: Some Negatives (but no overflow). | ||||||||||||
| 1209 | // Number line: | ||||||||||||
| 1210 | // |s64_min . . . 0 . . . s64_max| | ||||||||||||
| 1211 | // | . Q_min..Q_max . 0 . . . . | s64 negative | ||||||||||||
| 1212 | // | . . . Q_min..0..Q_max . . . | small mixed | ||||||||||||
| 1213 | // | ||||||||||||
| 1214 | // if Q_min <s64 0, then use this test: | ||||||||||||
| 1215 | // j*K + s32_trunc(Q_min) <u32 clamp(R, 0, Q_max) | ||||||||||||
| 1216 | |||||||||||||
| 1217 | // If the 32-bit truncation loses information, no harm is done, since certainly the clamp also returns R_2=zero. | ||||||||||||
| 1218 | |||||||||||||
| 1219 | // Case B: No Negatives. | ||||||||||||
| 1220 | // Number line: | ||||||||||||
| 1221 | // |s64_min . . . 0 . . . s64_max| | ||||||||||||
| 1222 | // | . . . . 0 Q_min..Q_max . . | small positive | ||||||||||||
| 1223 | // | . . . . 0 . Q_min..Q_max . | s64 positive | ||||||||||||
| 1224 | // | ||||||||||||
| 1225 | // if both Q_min, Q_max >=s64 0, then use this test: | ||||||||||||
| 1226 | // j*K + 0 <u32 clamp(R, Q_min, Q_max) - Q_min | ||||||||||||
| 1227 | // or equivalently: | ||||||||||||
| 1228 | // j*K + 0 <u32 clamp(R - Q_min, 0, Q_max - Q_min) | ||||||||||||
| 1229 | |||||||||||||
| 1230 | // Case C: Overflow in the 64-bit domain | ||||||||||||
| 1231 | // Number line: | ||||||||||||
| 1232 | // |..Q_max-2^64 . . 0 . . . Q_min..| s64 overflow | ||||||||||||
| 1233 | // | ||||||||||||
| 1234 | // if Q_min >=s64 0 but Q_max <s64 0, then use this test: | ||||||||||||
| 1235 | // j*K + 0 <u32 clamp(R, Q_min, R) - Q_min | ||||||||||||
| 1236 | // or equivalently: | ||||||||||||
| 1237 | // j*K + 0 <u32 clamp(R - Q_min, 0, R - Q_min) | ||||||||||||
| 1238 | // or also equivalently: | ||||||||||||
| 1239 | // j*K + 0 <u32 max(0, R - Q_min) | ||||||||||||
| 1240 | // | ||||||||||||
| 1241 | // Here the clamp function is a simple 64-bit min/max: | ||||||||||||
| 1242 | // clamp(X, L, H) := max(L, min(X, H)) | ||||||||||||
| 1243 | // When degenerately L > H, it returns L not H. | ||||||||||||
| 1244 | // | ||||||||||||
| 1245 | // Tests above can be merged into a single one: | ||||||||||||
| 1246 | // L_clamp = Q_min < 0 ? 0 : Q_min | ||||||||||||
| 1247 | // H_clamp = Q_max < Q_min ? R : Q_max | ||||||||||||
| 1248 | // j*K + Q_min - L_clamp <u32 clamp(R, L_clamp, H_clamp) - L_clamp | ||||||||||||
| 1249 | // or equivalently: | ||||||||||||
| 1250 | // j*K + Q_min - L_clamp <u32 clamp(R - L_clamp, 0, H_clamp - L_clamp) | ||||||||||||
| 1251 | // | ||||||||||||
| 1252 | // Readers may find the equivalent forms easier to reason about, but the forms given first generate better code. | ||||||||||||
| 1253 | |||||||||||||
| 1254 | void PhaseIdealLoop::transform_long_range_checks(int stride_con, const Node_List &range_checks, Node* outer_phi, | ||||||||||||
| 1255 | Node* inner_iters_actual_int, Node* inner_phi, | ||||||||||||
| 1256 | Node* iv_add, LoopNode* inner_head) { | ||||||||||||
| 1257 | Node* long_zero = _igvn.longcon(0); | ||||||||||||
| 1258 | set_ctrl(long_zero, C->root()); | ||||||||||||
| 1259 | |||||||||||||
| 1260 | for (uint i = 0; i < range_checks.size(); i++) { | ||||||||||||
| 1261 | ProjNode* proj = range_checks.at(i)->as_Proj(); | ||||||||||||
| 1262 | ProjNode* unc_proj = proj->other_if_proj(); | ||||||||||||
| 1263 | RangeCheckNode* rc = proj->in(0)->as_RangeCheck(); | ||||||||||||
| 1264 | jlong scale = 0; | ||||||||||||
| 1265 | Node* offset = NULL__null; | ||||||||||||
| 1266 | Node* rc_bol = rc->in(1); | ||||||||||||
| 1267 | Node* rc_cmp = rc_bol->in(1); | ||||||||||||
| 1268 | if (rc_cmp->Opcode() == Op_CmpU) { | ||||||||||||
| 1269 | // could be shared and have already been taken care of | ||||||||||||
| 1270 | continue; | ||||||||||||
| 1271 | } | ||||||||||||
| 1272 | bool converted = false; | ||||||||||||
| 1273 | bool ok = is_scaled_iv_plus_offset(rc_cmp->in(1), iv_add, &scale, &offset, T_LONG, &converted); | ||||||||||||
| 1274 | assert(ok, "inconsistent: was tested before")do { if (!(ok)) { (*g_assert_poison) = 'X';; report_vm_error( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1274, "assert(" "ok" ") failed", "inconsistent: was tested before" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1275 | Node* range = rc_cmp->in(2); | ||||||||||||
| 1276 | Node* c = rc->in(0); | ||||||||||||
| 1277 | Node* entry_control = inner_head->in(LoopNode::EntryControl); | ||||||||||||
| 1278 | |||||||||||||
| 1279 | Node* R = range; | ||||||||||||
| 1280 | Node* K = _igvn.longcon(scale); | ||||||||||||
| 1281 | set_ctrl(K, this->C->root()); | ||||||||||||
| 1282 | |||||||||||||
| 1283 | Node* L = offset; | ||||||||||||
| 1284 | |||||||||||||
| 1285 | if (converted) { | ||||||||||||
| 1286 | // This converts: | ||||||||||||
| 1287 | // i*K + L <u64 R | ||||||||||||
| 1288 | // with K an int into: | ||||||||||||
| 1289 | // i*(long)K + L <u64 unsigned_min((long)max_jint + L + 1, R) | ||||||||||||
| 1290 | // to protect against an overflow of i*K | ||||||||||||
| 1291 | // | ||||||||||||
| 1292 | // Because if i*K overflows, there are K,L where: | ||||||||||||
| 1293 | // i*K + L <u64 R is false | ||||||||||||
| 1294 | // when | ||||||||||||
| 1295 | // i*(long)K is > (long)max_jint and < R | ||||||||||||
| 1296 | // and so i*(long)K + L <u64 R is true | ||||||||||||
| 1297 | // As a consequence simply converting: | ||||||||||||
| 1298 | // i*K + L <u64 R to i*(long)K + L <u64 R could cause incorrect execution | ||||||||||||
| 1299 | // | ||||||||||||
| 1300 | // It's always true that: | ||||||||||||
| 1301 | // i*K <u64 (long)max_jint + 1 | ||||||||||||
| 1302 | // which implies i*K + L <u64 (long)max_jint + 1 + L | ||||||||||||
| 1303 | // As a consequence: | ||||||||||||
| 1304 | // i*(long)K + L <u64 unsigned_min((long)max_jint + L + 1, R) | ||||||||||||
| 1305 | // is always false in case of overflow of i*K | ||||||||||||
| 1306 | // | ||||||||||||
| 1307 | // Note, there are K,L where i*K overflows and | ||||||||||||
| 1308 | // i*K + L <u64 R is true, but | ||||||||||||
| 1309 | // i*(long)K + L <u64 unsigned_min((long)max_jint + L + 1, R) is false | ||||||||||||
| 1310 | // So this transformation could cause spurious deoptimizations and failed range check elimination | ||||||||||||
| 1311 | // (but not incorrect execution) for unlikely corner cases with overflow | ||||||||||||
| 1312 | Node* max_jint_plus_one_long = _igvn.longcon((jlong)max_jint + 1); | ||||||||||||
| 1313 | set_ctrl(max_jint_plus_one_long, C->root()); | ||||||||||||
| 1314 | Node* max_range = new AddLNode(max_jint_plus_one_long, L); | ||||||||||||
| 1315 | register_new_node(max_range, entry_control); | ||||||||||||
| 1316 | R = MaxNode::unsigned_min(R, max_range, TypeLong::POS, _igvn); | ||||||||||||
| 1317 | set_subtree_ctrl(R, true); | ||||||||||||
| 1318 | } | ||||||||||||
| 1319 | |||||||||||||
| 1320 | Node* C = outer_phi; | ||||||||||||
| 1321 | Node* Z_2 = new ConvI2LNode(inner_iters_actual_int, TypeLong::LONG); | ||||||||||||
| 1322 | register_new_node(Z_2, entry_control); | ||||||||||||
| 1323 | |||||||||||||
| 1324 | // Start with 64-bit values: | ||||||||||||
| 1325 | // i*K + L <u64 R | ||||||||||||
| 1326 | // (C+j)*K + L <u64 R | ||||||||||||
| 1327 | // j*K + L_2 <u64 R where L_2 = C*K+L | ||||||||||||
| 1328 | Node* L_2 = new MulLNode(C, K); | ||||||||||||
| 1329 | register_new_node(L_2, entry_control); | ||||||||||||
| 1330 | L_2 = new AddLNode(L_2, L); | ||||||||||||
| 1331 | register_new_node(L_2, entry_control); | ||||||||||||
| 1332 | |||||||||||||
| 1333 | // Compute endpoints of the range of values j*K. | ||||||||||||
| 1334 | // Q_min = (j=0)*K + L_2; Q_max = (j=Z_2)*K + L_2 | ||||||||||||
| 1335 | Node* Q_min = L_2; | ||||||||||||
| 1336 | Node* Q_max = new MulLNode(Z_2, K); | ||||||||||||
| 1337 | register_new_node(Q_max, entry_control); | ||||||||||||
| 1338 | Q_max = new AddLNode(Q_max, L_2); | ||||||||||||
| 1339 | register_new_node(Q_max, entry_control); | ||||||||||||
| 1340 | |||||||||||||
| 1341 | // L_clamp = Q_min < 0 ? 0 : Q_min | ||||||||||||
| 1342 | Node* Q_min_cmp = new CmpLNode(Q_min, long_zero); | ||||||||||||
| 1343 | register_new_node(Q_min_cmp, entry_control); | ||||||||||||
| 1344 | Node* Q_min_bool = new BoolNode(Q_min_cmp, BoolTest::lt); | ||||||||||||
| 1345 | register_new_node(Q_min_bool, entry_control); | ||||||||||||
| 1346 | Node* L_clamp = new CMoveLNode(Q_min_bool, Q_min, long_zero, TypeLong::LONG); | ||||||||||||
| 1347 | register_new_node(L_clamp, entry_control); | ||||||||||||
| 1348 | |||||||||||||
| 1349 | // H_clamp = Q_max < Q_min ? R : Q_max | ||||||||||||
| 1350 | Node* Q_max_cmp = new CmpLNode(Q_max, Q_min); | ||||||||||||
| 1351 | register_new_node(Q_max_cmp, entry_control); | ||||||||||||
| 1352 | Node* Q_max_bool = new BoolNode(Q_max_cmp, BoolTest::lt); | ||||||||||||
| 1353 | register_new_node(Q_max_bool, entry_control); | ||||||||||||
| 1354 | Node* H_clamp = new CMoveLNode(Q_max_bool, Q_max, R, TypeLong::LONG); | ||||||||||||
| 1355 | register_new_node(H_clamp, entry_control); | ||||||||||||
| 1356 | |||||||||||||
| 1357 | // R_2 = clamp(R, L_clamp, H_clamp) - L_clamp | ||||||||||||
| 1358 | // that is: R_2 = clamp(R, L_clamp, H_clamp) if Q_min < 0 | ||||||||||||
| 1359 | // or: R_2 = clamp(R, L_clamp, H_clamp) - Q_min if Q_min > 0 | ||||||||||||
| 1360 | Node* R_2 = clamp(R, L_clamp, H_clamp); | ||||||||||||
| 1361 | R_2 = new SubLNode(R_2, L_clamp); | ||||||||||||
| 1362 | register_new_node(R_2, entry_control); | ||||||||||||
| 1363 | R_2 = new ConvL2INode(R_2, TypeInt::POS); | ||||||||||||
| 1364 | register_new_node(R_2, entry_control); | ||||||||||||
| 1365 | |||||||||||||
| 1366 | // Q = Q_min - L_clamp | ||||||||||||
| 1367 | // that is: Q = Q_min - 0 if Q_min < 0 | ||||||||||||
| 1368 | // or: Q = Q_min - Q_min = 0 if Q_min > 0 | ||||||||||||
| 1369 | Node* Q = new SubLNode(Q_min, L_clamp); | ||||||||||||
| 1370 | register_new_node(Q, entry_control); | ||||||||||||
| 1371 | Q = new ConvL2INode(Q, TypeInt::INT); | ||||||||||||
| 1372 | register_new_node(Q, entry_control); | ||||||||||||
| 1373 | |||||||||||||
| 1374 | // Transform the range check | ||||||||||||
| 1375 | K = _igvn.intcon(checked_cast<int>(scale)); | ||||||||||||
| 1376 | set_ctrl(K, this->C->root()); | ||||||||||||
| 1377 | Node* scaled_iv = new MulINode(inner_phi, K); | ||||||||||||
| 1378 | register_new_node(scaled_iv, c); | ||||||||||||
| 1379 | Node* scaled_iv_plus_offset = scaled_iv_plus_offset = new AddINode(scaled_iv, Q); | ||||||||||||
| 1380 | register_new_node(scaled_iv_plus_offset, c); | ||||||||||||
| 1381 | |||||||||||||
| 1382 | Node* new_rc_cmp = new CmpUNode(scaled_iv_plus_offset, R_2); | ||||||||||||
| 1383 | register_new_node(new_rc_cmp, c); | ||||||||||||
| 1384 | |||||||||||||
| 1385 | _igvn.replace_input_of(rc_bol, 1, new_rc_cmp); | ||||||||||||
| 1386 | } | ||||||||||||
| 1387 | } | ||||||||||||
| 1388 | |||||||||||||
| 1389 | Node* PhaseIdealLoop::clamp(Node* R, Node* L, Node* H) { | ||||||||||||
| 1390 | Node* min = MaxNode::signed_min(R, H, TypeLong::LONG, _igvn); | ||||||||||||
| 1391 | set_subtree_ctrl(min, true); | ||||||||||||
| 1392 | Node* max = MaxNode::signed_max(L, min, TypeLong::LONG, _igvn); | ||||||||||||
| 1393 | set_subtree_ctrl(max, true); | ||||||||||||
| 1394 | return max; | ||||||||||||
| 1395 | } | ||||||||||||
| 1396 | |||||||||||||
| 1397 | LoopNode* PhaseIdealLoop::create_inner_head(IdealLoopTree* loop, BaseCountedLoopNode* head, | ||||||||||||
| 1398 | IfNode* exit_test) { | ||||||||||||
| 1399 | LoopNode* new_inner_head = new LoopNode(head->in(1), head->in(2)); | ||||||||||||
| 1400 | IfNode* new_inner_exit = new IfNode(exit_test->in(0), exit_test->in(1), exit_test->_prob, exit_test->_fcnt); | ||||||||||||
| 1401 | _igvn.register_new_node_with_optimizer(new_inner_head); | ||||||||||||
| 1402 | _igvn.register_new_node_with_optimizer(new_inner_exit); | ||||||||||||
| 1403 | loop->_body.push(new_inner_head); | ||||||||||||
| 1404 | loop->_body.push(new_inner_exit); | ||||||||||||
| 1405 | loop->_body.yank(head); | ||||||||||||
| 1406 | loop->_body.yank(exit_test); | ||||||||||||
| 1407 | set_loop(new_inner_head, loop); | ||||||||||||
| 1408 | set_loop(new_inner_exit, loop); | ||||||||||||
| 1409 | set_idom(new_inner_head, idom(head), dom_depth(head)); | ||||||||||||
| 1410 | set_idom(new_inner_exit, idom(exit_test), dom_depth(exit_test)); | ||||||||||||
| 1411 | lazy_replace(head, new_inner_head); | ||||||||||||
| 1412 | lazy_replace(exit_test, new_inner_exit); | ||||||||||||
| 1413 | loop->_head = new_inner_head; | ||||||||||||
| 1414 | return new_inner_head; | ||||||||||||
| 1415 | } | ||||||||||||
| 1416 | |||||||||||||
| 1417 | #ifdef ASSERT1 | ||||||||||||
| 1418 | void PhaseIdealLoop::check_counted_loop_shape(IdealLoopTree* loop, Node* x, BasicType bt) { | ||||||||||||
| 1419 | Node* back_control = loop_exit_control(x, loop); | ||||||||||||
| 1420 | assert(back_control != NULL, "no back control")do { if (!(back_control != __null)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1420, "assert(" "back_control != __null" ") failed", "no back control" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1421 | |||||||||||||
| 1422 | BoolTest::mask mask = BoolTest::illegal; | ||||||||||||
| 1423 | float cl_prob = 0; | ||||||||||||
| 1424 | Node* incr = NULL__null; | ||||||||||||
| 1425 | Node* limit = NULL__null; | ||||||||||||
| 1426 | |||||||||||||
| 1427 | Node* cmp = loop_exit_test(back_control, loop, incr, limit, mask, cl_prob); | ||||||||||||
| 1428 | assert(cmp != NULL && cmp->Opcode() == Op_Cmp(bt), "no exit test")do { if (!(cmp != __null && cmp->Opcode() == Op_Cmp (bt))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1428, "assert(" "cmp != __null && cmp->Opcode() == Op_Cmp(bt)" ") failed", "no exit test"); ::breakpoint(); } } while (0); | ||||||||||||
| 1429 | |||||||||||||
| 1430 | Node* phi_incr = NULL__null; | ||||||||||||
| 1431 | incr = loop_iv_incr(incr, x, loop, phi_incr); | ||||||||||||
| 1432 | assert(incr != NULL && incr->Opcode() == Op_Add(bt), "no incr")do { if (!(incr != __null && incr->Opcode() == Op_Add (bt))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1432, "assert(" "incr != __null && incr->Opcode() == Op_Add(bt)" ") failed", "no incr"); ::breakpoint(); } } while (0); | ||||||||||||
| 1433 | |||||||||||||
| 1434 | Node* xphi = NULL__null; | ||||||||||||
| 1435 | Node* stride = loop_iv_stride(incr, loop, xphi); | ||||||||||||
| 1436 | |||||||||||||
| 1437 | assert(stride != NULL, "no stride")do { if (!(stride != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1437, "assert(" "stride != __null" ") failed", "no stride") ; ::breakpoint(); } } while (0); | ||||||||||||
| 1438 | |||||||||||||
| 1439 | PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); | ||||||||||||
| 1440 | |||||||||||||
| 1441 | assert(phi != NULL && phi->in(LoopNode::LoopBackControl) == incr, "No phi")do { if (!(phi != __null && phi->in(LoopNode::LoopBackControl ) == incr)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1441, "assert(" "phi != __null && phi->in(LoopNode::LoopBackControl) == incr" ") failed", "No phi"); ::breakpoint(); } } while (0); | ||||||||||||
| 1442 | |||||||||||||
| 1443 | jlong stride_con = stride->get_integer_as_long(bt); | ||||||||||||
| 1444 | |||||||||||||
| 1445 | assert(condition_stride_ok(mask, stride_con), "illegal condition")do { if (!(condition_stride_ok(mask, stride_con))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1445, "assert(" "condition_stride_ok(mask, stride_con)" ") failed" , "illegal condition"); ::breakpoint(); } } while (0); | ||||||||||||
| 1446 | |||||||||||||
| 1447 | assert(mask != BoolTest::ne, "unexpected condition")do { if (!(mask != BoolTest::ne)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1447, "assert(" "mask != BoolTest::ne" ") failed", "unexpected condition" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1448 | assert(phi_incr == NULL, "bad loop shape")do { if (!(phi_incr == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1448, "assert(" "phi_incr == __null" ") failed", "bad loop shape" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1449 | assert(cmp->in(1) == incr, "bad exit test shape")do { if (!(cmp->in(1) == incr)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1449, "assert(" "cmp->in(1) == incr" ") failed", "bad exit test shape" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1450 | |||||||||||||
| 1451 | // Safepoint on backedge not supported | ||||||||||||
| 1452 | assert(x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint, "no safepoint on backedge")do { if (!(x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint)) { (*g_assert_poison) = 'X';; report_vm_error( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1452, "assert(" "x->in(LoopNode::LoopBackControl)->Opcode() != Op_SafePoint" ") failed", "no safepoint on backedge"); ::breakpoint(); } } while (0); | ||||||||||||
| 1453 | } | ||||||||||||
| 1454 | #endif | ||||||||||||
| 1455 | |||||||||||||
| 1456 | #ifdef ASSERT1 | ||||||||||||
| 1457 | // convert an int counted loop to a long counted to stress handling of | ||||||||||||
| 1458 | // long counted loops | ||||||||||||
| 1459 | bool PhaseIdealLoop::convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop) { | ||||||||||||
| 1460 | Unique_Node_List iv_nodes; | ||||||||||||
| 1461 | Node_List old_new; | ||||||||||||
| 1462 | iv_nodes.push(cmp); | ||||||||||||
| 1463 | bool failed = false; | ||||||||||||
| 1464 | |||||||||||||
| 1465 | for (uint i = 0; i < iv_nodes.size() && !failed; i++) { | ||||||||||||
| 1466 | Node* n = iv_nodes.at(i); | ||||||||||||
| 1467 | switch(n->Opcode()) { | ||||||||||||
| 1468 | case Op_Phi: { | ||||||||||||
| 1469 | Node* clone = new PhiNode(n->in(0), TypeLong::LONG); | ||||||||||||
| 1470 | old_new.map(n->_idx, clone); | ||||||||||||
| 1471 | break; | ||||||||||||
| 1472 | } | ||||||||||||
| 1473 | case Op_CmpI: { | ||||||||||||
| 1474 | Node* clone = new CmpLNode(NULL__null, NULL__null); | ||||||||||||
| 1475 | old_new.map(n->_idx, clone); | ||||||||||||
| 1476 | break; | ||||||||||||
| 1477 | } | ||||||||||||
| 1478 | case Op_AddI: { | ||||||||||||
| 1479 | Node* clone = new AddLNode(NULL__null, NULL__null); | ||||||||||||
| 1480 | old_new.map(n->_idx, clone); | ||||||||||||
| 1481 | break; | ||||||||||||
| 1482 | } | ||||||||||||
| 1483 | case Op_CastII: { | ||||||||||||
| 1484 | failed = true; | ||||||||||||
| 1485 | break; | ||||||||||||
| 1486 | } | ||||||||||||
| 1487 | default: | ||||||||||||
| 1488 | DEBUG_ONLY(n->dump())n->dump(); | ||||||||||||
| 1489 | fatal("unexpected")do { (*g_assert_poison) = 'X';; report_fatal(INTERNAL_ERROR, "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1489, "unexpected"); ::breakpoint(); } while (0); | ||||||||||||
| 1490 | } | ||||||||||||
| 1491 | |||||||||||||
| 1492 | for (uint i = 1; i < n->req(); i++) { | ||||||||||||
| 1493 | Node* in = n->in(i); | ||||||||||||
| 1494 | if (in == NULL__null) { | ||||||||||||
| 1495 | continue; | ||||||||||||
| 1496 | } | ||||||||||||
| 1497 | if (loop->is_member(get_loop(get_ctrl(in)))) { | ||||||||||||
| 1498 | iv_nodes.push(in); | ||||||||||||
| 1499 | } | ||||||||||||
| 1500 | } | ||||||||||||
| 1501 | } | ||||||||||||
| 1502 | |||||||||||||
| 1503 | if (failed) { | ||||||||||||
| 1504 | for (uint i = 0; i < iv_nodes.size(); i++) { | ||||||||||||
| 1505 | Node* n = iv_nodes.at(i); | ||||||||||||
| 1506 | Node* clone = old_new[n->_idx]; | ||||||||||||
| 1507 | if (clone != NULL__null) { | ||||||||||||
| 1508 | _igvn.remove_dead_node(clone); | ||||||||||||
| 1509 | } | ||||||||||||
| 1510 | } | ||||||||||||
| 1511 | return false; | ||||||||||||
| 1512 | } | ||||||||||||
| 1513 | |||||||||||||
| 1514 | for (uint i = 0; i < iv_nodes.size(); i++) { | ||||||||||||
| 1515 | Node* n = iv_nodes.at(i); | ||||||||||||
| 1516 | Node* clone = old_new[n->_idx]; | ||||||||||||
| 1517 | for (uint i = 1; i < n->req(); i++) { | ||||||||||||
| 1518 | Node* in = n->in(i); | ||||||||||||
| 1519 | if (in == NULL__null) { | ||||||||||||
| 1520 | continue; | ||||||||||||
| 1521 | } | ||||||||||||
| 1522 | Node* in_clone = old_new[in->_idx]; | ||||||||||||
| 1523 | if (in_clone == NULL__null) { | ||||||||||||
| 1524 | assert(_igvn.type(in)->isa_int(), "")do { if (!(_igvn.type(in)->isa_int())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1524, "assert(" "_igvn.type(in)->isa_int()" ") failed", "" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1525 | in_clone = new ConvI2LNode(in); | ||||||||||||
| 1526 | _igvn.register_new_node_with_optimizer(in_clone); | ||||||||||||
| 1527 | set_subtree_ctrl(in_clone, false); | ||||||||||||
| 1528 | } | ||||||||||||
| 1529 | if (in_clone->in(0) == NULL__null) { | ||||||||||||
| 1530 | in_clone->set_req(0, C->top()); | ||||||||||||
| 1531 | clone->set_req(i, in_clone); | ||||||||||||
| 1532 | in_clone->set_req(0, NULL__null); | ||||||||||||
| 1533 | } else { | ||||||||||||
| 1534 | clone->set_req(i, in_clone); | ||||||||||||
| 1535 | } | ||||||||||||
| 1536 | } | ||||||||||||
| 1537 | _igvn.register_new_node_with_optimizer(clone); | ||||||||||||
| 1538 | } | ||||||||||||
| 1539 | set_ctrl(old_new[phi->_idx], phi->in(0)); | ||||||||||||
| 1540 | |||||||||||||
| 1541 | for (uint i = 0; i < iv_nodes.size(); i++) { | ||||||||||||
| 1542 | Node* n = iv_nodes.at(i); | ||||||||||||
| 1543 | Node* clone = old_new[n->_idx]; | ||||||||||||
| 1544 | set_subtree_ctrl(clone, false); | ||||||||||||
| 1545 | Node* m = n->Opcode() == Op_CmpI ? clone : NULL__null; | ||||||||||||
| 1546 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 1547 | Node* u = n->fast_out(i); | ||||||||||||
| 1548 | if (iv_nodes.member(u)) { | ||||||||||||
| 1549 | continue; | ||||||||||||
| 1550 | } | ||||||||||||
| 1551 | if (m == NULL__null) { | ||||||||||||
| 1552 | m = new ConvL2INode(clone); | ||||||||||||
| 1553 | _igvn.register_new_node_with_optimizer(m); | ||||||||||||
| 1554 | set_subtree_ctrl(m, false); | ||||||||||||
| 1555 | } | ||||||||||||
| 1556 | _igvn.rehash_node_delayed(u); | ||||||||||||
| 1557 | int nb = u->replace_edge(n, m, &_igvn); | ||||||||||||
| 1558 | --i, imax -= nb; | ||||||||||||
| 1559 | } | ||||||||||||
| 1560 | } | ||||||||||||
| 1561 | return true; | ||||||||||||
| 1562 | } | ||||||||||||
| 1563 | #endif | ||||||||||||
| 1564 | |||||||||||||
| 1565 | //------------------------------is_counted_loop-------------------------------- | ||||||||||||
| 1566 | bool PhaseIdealLoop::is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt) { | ||||||||||||
| 1567 | PhaseGVN *gvn = &_igvn; | ||||||||||||
| 1568 | |||||||||||||
| 1569 | Node* back_control = loop_exit_control(x, loop); | ||||||||||||
| 1570 | if (back_control == NULL__null) { | ||||||||||||
| 1571 | return false; | ||||||||||||
| 1572 | } | ||||||||||||
| 1573 | |||||||||||||
| 1574 | BoolTest::mask bt = BoolTest::illegal; | ||||||||||||
| 1575 | float cl_prob = 0; | ||||||||||||
| 1576 | Node* incr = NULL__null; | ||||||||||||
| 1577 | Node* limit = NULL__null; | ||||||||||||
| 1578 | Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob); | ||||||||||||
| 1579 | if (cmp == NULL__null || cmp->Opcode() != Op_Cmp(iv_bt)) { | ||||||||||||
| 1580 | return false; // Avoid pointer & float & 64-bit compares | ||||||||||||
| 1581 | } | ||||||||||||
| 1582 | |||||||||||||
| 1583 | // Trip-counter increment must be commutative & associative. | ||||||||||||
| 1584 | if (incr->Opcode() == Op_Cast(iv_bt)) { | ||||||||||||
| 1585 | incr = incr->in(1); | ||||||||||||
| 1586 | } | ||||||||||||
| 1587 | |||||||||||||
| 1588 | Node* phi_incr = NULL__null; | ||||||||||||
| 1589 | incr = loop_iv_incr(incr, x, loop, phi_incr); | ||||||||||||
| 1590 | if (incr == NULL__null) { | ||||||||||||
| 1591 | return false; | ||||||||||||
| 1592 | } | ||||||||||||
| 1593 | |||||||||||||
| 1594 | Node* trunc1 = NULL__null; | ||||||||||||
| 1595 | Node* trunc2 = NULL__null; | ||||||||||||
| 1596 | const TypeInteger* iv_trunc_t = NULL__null; | ||||||||||||
| 1597 | Node* orig_incr = incr; | ||||||||||||
| 1598 | if (!(incr = CountedLoopNode::match_incr_with_optional_truncation(incr, &trunc1, &trunc2, &iv_trunc_t, iv_bt))) { | ||||||||||||
| 1599 | return false; // Funny increment opcode | ||||||||||||
| 1600 | } | ||||||||||||
| 1601 | assert(incr->Opcode() == Op_Add(iv_bt), "wrong increment code")do { if (!(incr->Opcode() == Op_Add(iv_bt))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1601, "assert(" "incr->Opcode() == Op_Add(iv_bt)" ") failed" , "wrong increment code"); ::breakpoint(); } } while (0); | ||||||||||||
| 1602 | |||||||||||||
| 1603 | Node* xphi = NULL__null; | ||||||||||||
| 1604 | Node* stride = loop_iv_stride(incr, loop, xphi); | ||||||||||||
| 1605 | |||||||||||||
| 1606 | if (stride == NULL__null) { | ||||||||||||
| 1607 | return false; | ||||||||||||
| 1608 | } | ||||||||||||
| 1609 | |||||||||||||
| 1610 | if (xphi->Opcode() == Op_Cast(iv_bt)) { | ||||||||||||
| 1611 | xphi = xphi->in(1); | ||||||||||||
| 1612 | } | ||||||||||||
| 1613 | |||||||||||||
| 1614 | // Stride must be constant | ||||||||||||
| 1615 | jlong stride_con = stride->get_integer_as_long(iv_bt); | ||||||||||||
| 1616 | assert(stride_con != 0, "missed some peephole opt")do { if (!(stride_con != 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1616, "assert(" "stride_con != 0" ") failed", "missed some peephole opt" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1617 | |||||||||||||
| 1618 | PhiNode* phi = loop_iv_phi(xphi, phi_incr, x, loop); | ||||||||||||
| 1619 | |||||||||||||
| 1620 | if (phi == NULL__null || | ||||||||||||
| 1621 | (trunc1 == NULL__null && phi->in(LoopNode::LoopBackControl) != incr) || | ||||||||||||
| 1622 | (trunc1 != NULL__null && phi->in(LoopNode::LoopBackControl) != trunc1)) { | ||||||||||||
| 1623 | return false; | ||||||||||||
| 1624 | } | ||||||||||||
| 1625 | |||||||||||||
| 1626 | if (x->in(LoopNode::LoopBackControl)->Opcode() == Op_SafePoint && | ||||||||||||
| 1627 | ((iv_bt == T_INT && LoopStripMiningIter != 0) || | ||||||||||||
| 1628 | iv_bt == T_LONG)) { | ||||||||||||
| 1629 | // Leaving the safepoint on the backedge and creating a | ||||||||||||
| 1630 | // CountedLoop will confuse optimizations. We can't move the | ||||||||||||
| 1631 | // safepoint around because its jvm state wouldn't match a new | ||||||||||||
| 1632 | // location. Give up on that loop. | ||||||||||||
| 1633 | return false; | ||||||||||||
| 1634 | } | ||||||||||||
| 1635 | |||||||||||||
| 1636 | Node* iftrue = back_control; | ||||||||||||
| 1637 | uint iftrue_op = iftrue->Opcode(); | ||||||||||||
| 1638 | Node* iff = iftrue->in(0); | ||||||||||||
| 1639 | BoolNode* test = iff->in(1)->as_Bool(); | ||||||||||||
| 1640 | |||||||||||||
| 1641 | const TypeInteger* limit_t = gvn->type(limit)->is_integer(iv_bt); | ||||||||||||
| 1642 | if (trunc1 != NULL__null) { | ||||||||||||
| 1643 | // When there is a truncation, we must be sure that after the truncation | ||||||||||||
| 1644 | // the trip counter will end up higher than the limit, otherwise we are looking | ||||||||||||
| 1645 | // at an endless loop. Can happen with range checks. | ||||||||||||
| 1646 | |||||||||||||
| 1647 | // Example: | ||||||||||||
| 1648 | // int i = 0; | ||||||||||||
| 1649 | // while (true) | ||||||||||||
| 1650 | // sum + = array[i]; | ||||||||||||
| 1651 | // i++; | ||||||||||||
| 1652 | // i = i && 0x7fff; | ||||||||||||
| 1653 | // } | ||||||||||||
| 1654 | // | ||||||||||||
| 1655 | // If the array is shorter than 0x8000 this exits through a AIOOB | ||||||||||||
| 1656 | // - Counted loop transformation is ok | ||||||||||||
| 1657 | // If the array is longer then this is an endless loop | ||||||||||||
| 1658 | // - No transformation can be done. | ||||||||||||
| 1659 | |||||||||||||
| 1660 | const TypeInteger* incr_t = gvn->type(orig_incr)->is_integer(iv_bt); | ||||||||||||
| 1661 | if (limit_t->hi_as_long() > incr_t->hi_as_long()) { | ||||||||||||
| 1662 | // if the limit can have a higher value than the increment (before the phi) | ||||||||||||
| 1663 | return false; | ||||||||||||
| 1664 | } | ||||||||||||
| 1665 | } | ||||||||||||
| 1666 | |||||||||||||
| 1667 | Node *init_trip = phi->in(LoopNode::EntryControl); | ||||||||||||
| 1668 | |||||||||||||
| 1669 | // If iv trunc type is smaller than int, check for possible wrap. | ||||||||||||
| 1670 | if (!TypeInteger::bottom(iv_bt)->higher_equal(iv_trunc_t)) { | ||||||||||||
| 1671 | assert(trunc1 != NULL, "must have found some truncation")do { if (!(trunc1 != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1671, "assert(" "trunc1 != __null" ") failed", "must have found some truncation" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1672 | |||||||||||||
| 1673 | // Get a better type for the phi (filtered thru if's) | ||||||||||||
| 1674 | const TypeInteger* phi_ft = filtered_type(phi); | ||||||||||||
| 1675 | |||||||||||||
| 1676 | // Can iv take on a value that will wrap? | ||||||||||||
| 1677 | // | ||||||||||||
| 1678 | // Ensure iv's limit is not within "stride" of the wrap value. | ||||||||||||
| 1679 | // | ||||||||||||
| 1680 | // Example for "short" type | ||||||||||||
| 1681 | // Truncation ensures value is in the range -32768..32767 (iv_trunc_t) | ||||||||||||
| 1682 | // If the stride is +10, then the last value of the induction | ||||||||||||
| 1683 | // variable before the increment (phi_ft->_hi) must be | ||||||||||||
| 1684 | // <= 32767 - 10 and (phi_ft->_lo) must be >= -32768 to | ||||||||||||
| 1685 | // ensure no truncation occurs after the increment. | ||||||||||||
| 1686 | |||||||||||||
| 1687 | if (stride_con > 0) { | ||||||||||||
| 1688 | if (iv_trunc_t->hi_as_long() - phi_ft->hi_as_long() < stride_con || | ||||||||||||
| 1689 | iv_trunc_t->lo_as_long() > phi_ft->lo_as_long()) { | ||||||||||||
| 1690 | return false; // truncation may occur | ||||||||||||
| 1691 | } | ||||||||||||
| 1692 | } else if (stride_con < 0) { | ||||||||||||
| 1693 | if (iv_trunc_t->lo_as_long() - phi_ft->lo_as_long() > stride_con || | ||||||||||||
| 1694 | iv_trunc_t->hi_as_long() < phi_ft->hi_as_long()) { | ||||||||||||
| 1695 | return false; // truncation may occur | ||||||||||||
| 1696 | } | ||||||||||||
| 1697 | } | ||||||||||||
| 1698 | // No possibility of wrap so truncation can be discarded | ||||||||||||
| 1699 | // Promote iv type to Int | ||||||||||||
| 1700 | } else { | ||||||||||||
| 1701 | assert(trunc1 == NULL && trunc2 == NULL, "no truncation for int")do { if (!(trunc1 == __null && trunc2 == __null)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1701, "assert(" "trunc1 == __null && trunc2 == __null" ") failed", "no truncation for int"); ::breakpoint(); } } while (0); | ||||||||||||
| 1702 | } | ||||||||||||
| 1703 | |||||||||||||
| 1704 | if (!condition_stride_ok(bt, stride_con)) { | ||||||||||||
| 1705 | return false; | ||||||||||||
| 1706 | } | ||||||||||||
| 1707 | |||||||||||||
| 1708 | const TypeInteger* init_t = gvn->type(init_trip)->is_integer(iv_bt); | ||||||||||||
| 1709 | |||||||||||||
| 1710 | if (stride_con > 0) { | ||||||||||||
| 1711 | if (init_t->lo_as_long() > max_signed_integer(iv_bt) - stride_con) { | ||||||||||||
| 1712 | return false; // cyclic loop | ||||||||||||
| 1713 | } | ||||||||||||
| 1714 | } else { | ||||||||||||
| 1715 | if (init_t->hi_as_long() < min_signed_integer(iv_bt) - stride_con) { | ||||||||||||
| 1716 | return false; // cyclic loop | ||||||||||||
| 1717 | } | ||||||||||||
| 1718 | } | ||||||||||||
| 1719 | |||||||||||||
| 1720 | if (phi_incr != NULL__null && bt != BoolTest::ne) { | ||||||||||||
| 1721 | // check if there is a possiblity of IV overflowing after the first increment | ||||||||||||
| 1722 | if (stride_con > 0) { | ||||||||||||
| 1723 | if (init_t->hi_as_long() > max_signed_integer(iv_bt) - stride_con) { | ||||||||||||
| 1724 | return false; | ||||||||||||
| 1725 | } | ||||||||||||
| 1726 | } else { | ||||||||||||
| 1727 | if (init_t->lo_as_long() < min_signed_integer(iv_bt) - stride_con) { | ||||||||||||
| 1728 | return false; | ||||||||||||
| 1729 | } | ||||||||||||
| 1730 | } | ||||||||||||
| 1731 | } | ||||||||||||
| 1732 | |||||||||||||
| 1733 | // ================================================= | ||||||||||||
| 1734 | // ---- SUCCESS! Found A Trip-Counted Loop! ----- | ||||||||||||
| 1735 | // | ||||||||||||
| 1736 | assert(x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop, "regular loops only")do { if (!(x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1736, "assert(" "x->Opcode() == Op_Loop || x->Opcode() == Op_LongCountedLoop" ") failed", "regular loops only"); ::breakpoint(); } } while (0); | ||||||||||||
| 1737 | C->print_method(PHASE_BEFORE_CLOOPS, 3); | ||||||||||||
| 1738 | |||||||||||||
| 1739 | // =================================================== | ||||||||||||
| 1740 | // Generate loop limit check to avoid integer overflow | ||||||||||||
| 1741 | // in cases like next (cyclic loops): | ||||||||||||
| 1742 | // | ||||||||||||
| 1743 | // for (i=0; i <= max_jint; i++) {} | ||||||||||||
| 1744 | // for (i=0; i < max_jint; i+=2) {} | ||||||||||||
| 1745 | // | ||||||||||||
| 1746 | // | ||||||||||||
| 1747 | // Limit check predicate depends on the loop test: | ||||||||||||
| 1748 | // | ||||||||||||
| 1749 | // for(;i != limit; i++) --> limit <= (max_jint) | ||||||||||||
| 1750 | // for(;i < limit; i+=stride) --> limit <= (max_jint - stride + 1) | ||||||||||||
| 1751 | // for(;i <= limit; i+=stride) --> limit <= (max_jint - stride ) | ||||||||||||
| 1752 | // | ||||||||||||
| 1753 | |||||||||||||
| 1754 | // Check if limit is excluded to do more precise int overflow check. | ||||||||||||
| 1755 | bool incl_limit = (bt == BoolTest::le || bt == BoolTest::ge); | ||||||||||||
| 1756 | jlong stride_m = stride_con - (incl_limit ? 0 : (stride_con > 0 ? 1 : -1)); | ||||||||||||
| 1757 | |||||||||||||
| 1758 | // If compare points directly to the phi we need to adjust | ||||||||||||
| 1759 | // the compare so that it points to the incr. Limit have | ||||||||||||
| 1760 | // to be adjusted to keep trip count the same and the | ||||||||||||
| 1761 | // adjusted limit should be checked for int overflow. | ||||||||||||
| 1762 | Node* adjusted_limit = limit; | ||||||||||||
| 1763 | if (phi_incr != NULL__null) { | ||||||||||||
| 1764 | stride_m += stride_con; | ||||||||||||
| 1765 | } | ||||||||||||
| 1766 | |||||||||||||
| 1767 | Node *init_control = x->in(LoopNode::EntryControl); | ||||||||||||
| 1768 | |||||||||||||
| 1769 | int sov = check_stride_overflow(stride_m, limit_t, iv_bt); | ||||||||||||
| 1770 | // If sov==0, limit's type always satisfies the condition, for | ||||||||||||
| 1771 | // example, when it is an array length. | ||||||||||||
| 1772 | if (sov != 0) { | ||||||||||||
| 1773 | if (sov < 0) { | ||||||||||||
| 1774 | return false; // Bailout: integer overflow is certain. | ||||||||||||
| 1775 | } | ||||||||||||
| 1776 | assert(!x->as_Loop()->is_loop_nest_inner_loop(), "loop was transformed")do { if (!(!x->as_Loop()->is_loop_nest_inner_loop())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1776, "assert(" "!x->as_Loop()->is_loop_nest_inner_loop()" ") failed", "loop was transformed"); ::breakpoint(); } } while (0); | ||||||||||||
| 1777 | // Generate loop's limit check. | ||||||||||||
| 1778 | // Loop limit check predicate should be near the loop. | ||||||||||||
| 1779 | ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check); | ||||||||||||
| 1780 | if (!limit_check_proj) { | ||||||||||||
| 1781 | // The limit check predicate is not generated if this method trapped here before. | ||||||||||||
| 1782 | #ifdef ASSERT1 | ||||||||||||
| 1783 | if (TraceLoopLimitCheck) { | ||||||||||||
| 1784 | tty->print("missing loop limit check:"); | ||||||||||||
| 1785 | loop->dump_head(); | ||||||||||||
| 1786 | x->dump(1); | ||||||||||||
| 1787 | } | ||||||||||||
| 1788 | #endif | ||||||||||||
| 1789 | return false; | ||||||||||||
| 1790 | } | ||||||||||||
| 1791 | |||||||||||||
| 1792 | IfNode* check_iff = limit_check_proj->in(0)->as_If(); | ||||||||||||
| 1793 | |||||||||||||
| 1794 | if (!is_dominator(get_ctrl(limit), check_iff->in(0))) { | ||||||||||||
| 1795 | return false; | ||||||||||||
| 1796 | } | ||||||||||||
| 1797 | |||||||||||||
| 1798 | Node* cmp_limit; | ||||||||||||
| 1799 | Node* bol; | ||||||||||||
| 1800 | |||||||||||||
| 1801 | if (stride_con > 0) { | ||||||||||||
| 1802 | cmp_limit = CmpNode::make(limit, _igvn.integercon(max_signed_integer(iv_bt) - stride_m, iv_bt), iv_bt); | ||||||||||||
| 1803 | bol = new BoolNode(cmp_limit, BoolTest::le); | ||||||||||||
| 1804 | } else { | ||||||||||||
| 1805 | cmp_limit = CmpNode::make(limit, _igvn.integercon(min_signed_integer(iv_bt) - stride_m, iv_bt), iv_bt); | ||||||||||||
| 1806 | bol = new BoolNode(cmp_limit, BoolTest::ge); | ||||||||||||
| 1807 | } | ||||||||||||
| 1808 | |||||||||||||
| 1809 | insert_loop_limit_check(limit_check_proj, cmp_limit, bol); | ||||||||||||
| 1810 | } | ||||||||||||
| 1811 | |||||||||||||
| 1812 | // Now we need to canonicalize loop condition. | ||||||||||||
| 1813 | if (bt == BoolTest::ne) { | ||||||||||||
| 1814 | assert(stride_con == 1 || stride_con == -1, "simple increment only")do { if (!(stride_con == 1 || stride_con == -1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1814, "assert(" "stride_con == 1 || stride_con == -1" ") failed" , "simple increment only"); ::breakpoint(); } } while (0); | ||||||||||||
| 1815 | if (stride_con > 0 && init_t->hi_as_long() < limit_t->lo_as_long()) { | ||||||||||||
| 1816 | // 'ne' can be replaced with 'lt' only when init < limit. | ||||||||||||
| 1817 | bt = BoolTest::lt; | ||||||||||||
| 1818 | } else if (stride_con < 0 && init_t->lo_as_long() > limit_t->hi_as_long()) { | ||||||||||||
| 1819 | // 'ne' can be replaced with 'gt' only when init > limit. | ||||||||||||
| 1820 | bt = BoolTest::gt; | ||||||||||||
| 1821 | } else { | ||||||||||||
| 1822 | ProjNode *limit_check_proj = find_predicate_insertion_point(init_control, Deoptimization::Reason_loop_limit_check); | ||||||||||||
| 1823 | if (!limit_check_proj) { | ||||||||||||
| 1824 | // The limit check predicate is not generated if this method trapped here before. | ||||||||||||
| 1825 | #ifdef ASSERT1 | ||||||||||||
| 1826 | if (TraceLoopLimitCheck) { | ||||||||||||
| 1827 | tty->print("missing loop limit check:"); | ||||||||||||
| 1828 | loop->dump_head(); | ||||||||||||
| 1829 | x->dump(1); | ||||||||||||
| 1830 | } | ||||||||||||
| 1831 | #endif | ||||||||||||
| 1832 | return false; | ||||||||||||
| 1833 | } | ||||||||||||
| 1834 | IfNode* check_iff = limit_check_proj->in(0)->as_If(); | ||||||||||||
| 1835 | |||||||||||||
| 1836 | if (!is_dominator(get_ctrl(limit), check_iff->in(0)) || | ||||||||||||
| 1837 | !is_dominator(get_ctrl(init_trip), check_iff->in(0))) { | ||||||||||||
| 1838 | return false; | ||||||||||||
| 1839 | } | ||||||||||||
| 1840 | |||||||||||||
| 1841 | Node* cmp_limit; | ||||||||||||
| 1842 | Node* bol; | ||||||||||||
| 1843 | |||||||||||||
| 1844 | if (stride_con > 0) { | ||||||||||||
| 1845 | cmp_limit = CmpNode::make(init_trip, limit, iv_bt); | ||||||||||||
| 1846 | bol = new BoolNode(cmp_limit, BoolTest::lt); | ||||||||||||
| 1847 | } else { | ||||||||||||
| 1848 | cmp_limit = CmpNode::make(init_trip, limit, iv_bt); | ||||||||||||
| 1849 | bol = new BoolNode(cmp_limit, BoolTest::gt); | ||||||||||||
| 1850 | } | ||||||||||||
| 1851 | |||||||||||||
| 1852 | insert_loop_limit_check(limit_check_proj, cmp_limit, bol); | ||||||||||||
| 1853 | |||||||||||||
| 1854 | if (stride_con > 0) { | ||||||||||||
| 1855 | // 'ne' can be replaced with 'lt' only when init < limit. | ||||||||||||
| 1856 | bt = BoolTest::lt; | ||||||||||||
| 1857 | } else if (stride_con < 0) { | ||||||||||||
| 1858 | // 'ne' can be replaced with 'gt' only when init > limit. | ||||||||||||
| 1859 | bt = BoolTest::gt; | ||||||||||||
| 1860 | } | ||||||||||||
| 1861 | } | ||||||||||||
| 1862 | } | ||||||||||||
| 1863 | |||||||||||||
| 1864 | #ifdef ASSERT1 | ||||||||||||
| 1865 | if (iv_bt == T_INT && | ||||||||||||
| 1866 | !x->as_Loop()->is_loop_nest_inner_loop() && | ||||||||||||
| 1867 | StressLongCountedLoop > 0 && | ||||||||||||
| 1868 | trunc1 == NULL__null && | ||||||||||||
| 1869 | convert_to_long_loop(cmp, phi, loop)) { | ||||||||||||
| 1870 | return false; | ||||||||||||
| 1871 | } | ||||||||||||
| 1872 | #endif | ||||||||||||
| 1873 | |||||||||||||
| 1874 | if (phi_incr != NULL__null) { | ||||||||||||
| 1875 | // If compare points directly to the phi we need to adjust | ||||||||||||
| 1876 | // the compare so that it points to the incr. Limit have | ||||||||||||
| 1877 | // to be adjusted to keep trip count the same and we | ||||||||||||
| 1878 | // should avoid int overflow. | ||||||||||||
| 1879 | // | ||||||||||||
| 1880 | // i = init; do {} while(i++ < limit); | ||||||||||||
| 1881 | // is converted to | ||||||||||||
| 1882 | // i = init; do {} while(++i < limit+1); | ||||||||||||
| 1883 | // | ||||||||||||
| 1884 | adjusted_limit = gvn->transform(AddNode::make(limit, stride, iv_bt)); | ||||||||||||
| 1885 | } | ||||||||||||
| 1886 | |||||||||||||
| 1887 | if (incl_limit) { | ||||||||||||
| 1888 | // The limit check guaranties that 'limit <= (max_jint - stride)' so | ||||||||||||
| 1889 | // we can convert 'i <= limit' to 'i < limit+1' since stride != 0. | ||||||||||||
| 1890 | // | ||||||||||||
| 1891 | Node* one = (stride_con > 0) ? gvn->integercon( 1, iv_bt) : gvn->integercon(-1, iv_bt); | ||||||||||||
| 1892 | adjusted_limit = gvn->transform(AddNode::make(adjusted_limit, one, iv_bt)); | ||||||||||||
| 1893 | if (bt == BoolTest::le) | ||||||||||||
| 1894 | bt = BoolTest::lt; | ||||||||||||
| 1895 | else if (bt == BoolTest::ge) | ||||||||||||
| 1896 | bt = BoolTest::gt; | ||||||||||||
| 1897 | else | ||||||||||||
| 1898 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1898); ::breakpoint(); } while (0); | ||||||||||||
| 1899 | } | ||||||||||||
| 1900 | set_subtree_ctrl(adjusted_limit, false); | ||||||||||||
| 1901 | |||||||||||||
| 1902 | if (iv_bt == T_INT && LoopStripMiningIter == 0) { | ||||||||||||
| 1903 | // Check for SafePoint on backedge and remove | ||||||||||||
| 1904 | Node *sfpt = x->in(LoopNode::LoopBackControl); | ||||||||||||
| 1905 | if (sfpt->Opcode() == Op_SafePoint && is_deleteable_safept(sfpt)) { | ||||||||||||
| 1906 | lazy_replace( sfpt, iftrue ); | ||||||||||||
| 1907 | if (loop->_safepts != NULL__null) { | ||||||||||||
| 1908 | loop->_safepts->yank(sfpt); | ||||||||||||
| 1909 | } | ||||||||||||
| 1910 | loop->_tail = iftrue; | ||||||||||||
| 1911 | } | ||||||||||||
| 1912 | } | ||||||||||||
| 1913 | |||||||||||||
| 1914 | // Build a canonical trip test. | ||||||||||||
| 1915 | // Clone code, as old values may be in use. | ||||||||||||
| 1916 | incr = incr->clone(); | ||||||||||||
| 1917 | incr->set_req(1,phi); | ||||||||||||
| 1918 | incr->set_req(2,stride); | ||||||||||||
| 1919 | incr = _igvn.register_new_node_with_optimizer(incr); | ||||||||||||
| 1920 | set_early_ctrl(incr, false); | ||||||||||||
| 1921 | _igvn.rehash_node_delayed(phi); | ||||||||||||
| 1922 | phi->set_req_X( LoopNode::LoopBackControl, incr, &_igvn ); | ||||||||||||
| 1923 | |||||||||||||
| 1924 | // If phi type is more restrictive than Int, raise to | ||||||||||||
| 1925 | // Int to prevent (almost) infinite recursion in igvn | ||||||||||||
| 1926 | // which can only handle integer types for constants or minint..maxint. | ||||||||||||
| 1927 | if (!TypeInteger::bottom(iv_bt)->higher_equal(phi->bottom_type())) { | ||||||||||||
| 1928 | Node* nphi = PhiNode::make(phi->in(0), phi->in(LoopNode::EntryControl), TypeInteger::bottom(iv_bt)); | ||||||||||||
| 1929 | nphi->set_req(LoopNode::LoopBackControl, phi->in(LoopNode::LoopBackControl)); | ||||||||||||
| 1930 | nphi = _igvn.register_new_node_with_optimizer(nphi); | ||||||||||||
| 1931 | set_ctrl(nphi, get_ctrl(phi)); | ||||||||||||
| 1932 | _igvn.replace_node(phi, nphi); | ||||||||||||
| 1933 | phi = nphi->as_Phi(); | ||||||||||||
| 1934 | } | ||||||||||||
| 1935 | cmp = cmp->clone(); | ||||||||||||
| 1936 | cmp->set_req(1,incr); | ||||||||||||
| 1937 | cmp->set_req(2, adjusted_limit); | ||||||||||||
| 1938 | cmp = _igvn.register_new_node_with_optimizer(cmp); | ||||||||||||
| 1939 | set_ctrl(cmp, iff->in(0)); | ||||||||||||
| 1940 | |||||||||||||
| 1941 | test = test->clone()->as_Bool(); | ||||||||||||
| 1942 | (*(BoolTest*)&test->_test)._test = bt; | ||||||||||||
| 1943 | test->set_req(1,cmp); | ||||||||||||
| 1944 | _igvn.register_new_node_with_optimizer(test); | ||||||||||||
| 1945 | set_ctrl(test, iff->in(0)); | ||||||||||||
| 1946 | |||||||||||||
| 1947 | // Replace the old IfNode with a new LoopEndNode | ||||||||||||
| 1948 | Node *lex = _igvn.register_new_node_with_optimizer(BaseCountedLoopEndNode::make(iff->in(0), test, cl_prob, iff->as_If()->_fcnt, iv_bt)); | ||||||||||||
| 1949 | IfNode *le = lex->as_If(); | ||||||||||||
| 1950 | uint dd = dom_depth(iff); | ||||||||||||
| 1951 | set_idom(le, le->in(0), dd); // Update dominance for loop exit | ||||||||||||
| 1952 | set_loop(le, loop); | ||||||||||||
| 1953 | |||||||||||||
| 1954 | // Get the loop-exit control | ||||||||||||
| 1955 | Node *iffalse = iff->as_If()->proj_out(!(iftrue_op == Op_IfTrue)); | ||||||||||||
| 1956 | |||||||||||||
| 1957 | // Need to swap loop-exit and loop-back control? | ||||||||||||
| 1958 | if (iftrue_op == Op_IfFalse) { | ||||||||||||
| 1959 | Node *ift2=_igvn.register_new_node_with_optimizer(new IfTrueNode (le)); | ||||||||||||
| 1960 | Node *iff2=_igvn.register_new_node_with_optimizer(new IfFalseNode(le)); | ||||||||||||
| 1961 | |||||||||||||
| 1962 | loop->_tail = back_control = ift2; | ||||||||||||
| 1963 | set_loop(ift2, loop); | ||||||||||||
| 1964 | set_loop(iff2, get_loop(iffalse)); | ||||||||||||
| 1965 | |||||||||||||
| 1966 | // Lazy update of 'get_ctrl' mechanism. | ||||||||||||
| 1967 | lazy_replace(iffalse, iff2); | ||||||||||||
| 1968 | lazy_replace(iftrue, ift2); | ||||||||||||
| 1969 | |||||||||||||
| 1970 | // Swap names | ||||||||||||
| 1971 | iffalse = iff2; | ||||||||||||
| 1972 | iftrue = ift2; | ||||||||||||
| 1973 | } else { | ||||||||||||
| 1974 | _igvn.rehash_node_delayed(iffalse); | ||||||||||||
| 1975 | _igvn.rehash_node_delayed(iftrue); | ||||||||||||
| 1976 | iffalse->set_req_X( 0, le, &_igvn ); | ||||||||||||
| 1977 | iftrue ->set_req_X( 0, le, &_igvn ); | ||||||||||||
| 1978 | } | ||||||||||||
| 1979 | |||||||||||||
| 1980 | set_idom(iftrue, le, dd+1); | ||||||||||||
| 1981 | set_idom(iffalse, le, dd+1); | ||||||||||||
| 1982 | assert(iff->outcnt() == 0, "should be dead now")do { if (!(iff->outcnt() == 0)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 1982, "assert(" "iff->outcnt() == 0" ") failed", "should be dead now" ); ::breakpoint(); } } while (0); | ||||||||||||
| 1983 | lazy_replace( iff, le ); // fix 'get_ctrl' | ||||||||||||
| 1984 | |||||||||||||
| 1985 | Node *sfpt2 = le->in(0); | ||||||||||||
| 1986 | |||||||||||||
| 1987 | Node* entry_control = init_control; | ||||||||||||
| 1988 | bool strip_mine_loop = iv_bt == T_INT && | ||||||||||||
| 1989 | LoopStripMiningIter > 1 && | ||||||||||||
| 1990 | loop->_child == NULL__null && | ||||||||||||
| 1991 | sfpt2->Opcode() == Op_SafePoint && | ||||||||||||
| 1992 | !loop->_has_call; | ||||||||||||
| 1993 | IdealLoopTree* outer_ilt = NULL__null; | ||||||||||||
| 1994 | if (strip_mine_loop) { | ||||||||||||
| 1995 | outer_ilt = create_outer_strip_mined_loop(test, cmp, init_control, loop, | ||||||||||||
| 1996 | cl_prob, le->_fcnt, entry_control, | ||||||||||||
| 1997 | iffalse); | ||||||||||||
| 1998 | } | ||||||||||||
| 1999 | |||||||||||||
| 2000 | // Now setup a new CountedLoopNode to replace the existing LoopNode | ||||||||||||
| 2001 | BaseCountedLoopNode *l = BaseCountedLoopNode::make(entry_control, back_control, iv_bt); | ||||||||||||
| 2002 | l->set_unswitch_count(x->as_Loop()->unswitch_count()); // Preserve | ||||||||||||
| 2003 | // The following assert is approximately true, and defines the intention | ||||||||||||
| 2004 | // of can_be_counted_loop. It fails, however, because phase->type | ||||||||||||
| 2005 | // is not yet initialized for this loop and its parts. | ||||||||||||
| 2006 | //assert(l->can_be_counted_loop(this), "sanity"); | ||||||||||||
| 2007 | _igvn.register_new_node_with_optimizer(l); | ||||||||||||
| 2008 | set_loop(l, loop); | ||||||||||||
| 2009 | loop->_head = l; | ||||||||||||
| 2010 | // Fix all data nodes placed at the old loop head. | ||||||||||||
| 2011 | // Uses the lazy-update mechanism of 'get_ctrl'. | ||||||||||||
| 2012 | lazy_replace( x, l ); | ||||||||||||
| 2013 | set_idom(l, entry_control, dom_depth(entry_control) + 1); | ||||||||||||
| 2014 | |||||||||||||
| 2015 | if (iv_bt == T_INT && (LoopStripMiningIter == 0 || strip_mine_loop)) { | ||||||||||||
| 2016 | // Check for immediately preceding SafePoint and remove | ||||||||||||
| 2017 | if (sfpt2->Opcode() == Op_SafePoint && (LoopStripMiningIter != 0 || is_deleteable_safept(sfpt2))) { | ||||||||||||
| 2018 | if (strip_mine_loop) { | ||||||||||||
| 2019 | Node* outer_le = outer_ilt->_tail->in(0); | ||||||||||||
| 2020 | Node* sfpt = sfpt2->clone(); | ||||||||||||
| 2021 | sfpt->set_req(0, iffalse); | ||||||||||||
| 2022 | outer_le->set_req(0, sfpt); | ||||||||||||
| 2023 | |||||||||||||
| 2024 | Node* polladdr = sfpt->in(TypeFunc::Parms); | ||||||||||||
| 2025 | if (polladdr != nullptr && polladdr->is_Load()) { | ||||||||||||
| 2026 | // Polling load should be pinned outside inner loop. | ||||||||||||
| 2027 | Node* new_polladdr = polladdr->clone(); | ||||||||||||
| 2028 | new_polladdr->set_req(0, iffalse); | ||||||||||||
| 2029 | _igvn.register_new_node_with_optimizer(new_polladdr, polladdr); | ||||||||||||
| 2030 | set_ctrl(new_polladdr, iffalse); | ||||||||||||
| 2031 | sfpt->set_req(TypeFunc::Parms, new_polladdr); | ||||||||||||
| 2032 | } | ||||||||||||
| 2033 | // When this code runs, loop bodies have not yet been populated. | ||||||||||||
| 2034 | const bool body_populated = false; | ||||||||||||
| 2035 | register_control(sfpt, outer_ilt, iffalse, body_populated); | ||||||||||||
| 2036 | set_idom(outer_le, sfpt, dom_depth(sfpt)); | ||||||||||||
| 2037 | } | ||||||||||||
| 2038 | lazy_replace( sfpt2, sfpt2->in(TypeFunc::Control)); | ||||||||||||
| 2039 | if (loop->_safepts != NULL__null) { | ||||||||||||
| 2040 | loop->_safepts->yank(sfpt2); | ||||||||||||
| 2041 | } | ||||||||||||
| 2042 | } | ||||||||||||
| 2043 | } | ||||||||||||
| 2044 | |||||||||||||
| 2045 | #ifdef ASSERT1 | ||||||||||||
| 2046 | assert(l->is_valid_counted_loop(iv_bt), "counted loop shape is messed up")do { if (!(l->is_valid_counted_loop(iv_bt))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2046, "assert(" "l->is_valid_counted_loop(iv_bt)" ") failed" , "counted loop shape is messed up"); ::breakpoint(); } } while (0); | ||||||||||||
| 2047 | assert(l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex, "" )do { if (!(l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2047, "assert(" "l == loop->_head && l->phi() == phi && l->loopexit_or_null() == lex" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 2048 | #endif | ||||||||||||
| 2049 | #ifndef PRODUCT | ||||||||||||
| 2050 | if (TraceLoopOpts) { | ||||||||||||
| 2051 | tty->print("Counted "); | ||||||||||||
| 2052 | loop->dump_head(); | ||||||||||||
| 2053 | } | ||||||||||||
| 2054 | #endif | ||||||||||||
| 2055 | |||||||||||||
| 2056 | C->print_method(PHASE_AFTER_CLOOPS, 3); | ||||||||||||
| 2057 | |||||||||||||
| 2058 | // Capture bounds of the loop in the induction variable Phi before | ||||||||||||
| 2059 | // subsequent transformation (iteration splitting) obscures the | ||||||||||||
| 2060 | // bounds | ||||||||||||
| 2061 | l->phi()->as_Phi()->set_type(l->phi()->Value(&_igvn)); | ||||||||||||
| 2062 | |||||||||||||
| 2063 | if (strip_mine_loop) { | ||||||||||||
| 2064 | l->mark_strip_mined(); | ||||||||||||
| 2065 | l->verify_strip_mined(1); | ||||||||||||
| 2066 | outer_ilt->_head->as_Loop()->verify_strip_mined(1); | ||||||||||||
| 2067 | loop = outer_ilt; | ||||||||||||
| 2068 | } | ||||||||||||
| 2069 | |||||||||||||
| 2070 | #ifndef PRODUCT | ||||||||||||
| 2071 | if (x->as_Loop()->is_loop_nest_inner_loop() && iv_bt == T_LONG) { | ||||||||||||
| 2072 | Atomic::inc(&_long_loop_counted_loops); | ||||||||||||
| 2073 | } | ||||||||||||
| 2074 | #endif | ||||||||||||
| 2075 | if (iv_bt == T_LONG && x->as_Loop()->is_loop_nest_outer_loop()) { | ||||||||||||
| 2076 | l->mark_loop_nest_outer_loop(); | ||||||||||||
| 2077 | } | ||||||||||||
| 2078 | |||||||||||||
| 2079 | return true; | ||||||||||||
| 2080 | } | ||||||||||||
| 2081 | |||||||||||||
| 2082 | //----------------------exact_limit------------------------------------------- | ||||||||||||
| 2083 | Node* PhaseIdealLoop::exact_limit( IdealLoopTree *loop ) { | ||||||||||||
| 2084 | assert(loop->_head->is_CountedLoop(), "")do { if (!(loop->_head->is_CountedLoop())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2084, "assert(" "loop->_head->is_CountedLoop()" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 2085 | CountedLoopNode *cl = loop->_head->as_CountedLoop(); | ||||||||||||
| 2086 | assert(cl->is_valid_counted_loop(T_INT), "")do { if (!(cl->is_valid_counted_loop(T_INT))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2086, "assert(" "cl->is_valid_counted_loop(T_INT)" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 2087 | |||||||||||||
| 2088 | if (ABS(cl->stride_con()) == 1 || | ||||||||||||
| 2089 | cl->limit()->Opcode() == Op_LoopLimit) { | ||||||||||||
| 2090 | // Old code has exact limit (it could be incorrect in case of int overflow). | ||||||||||||
| 2091 | // Loop limit is exact with stride == 1. And loop may already have exact limit. | ||||||||||||
| 2092 | return cl->limit(); | ||||||||||||
| 2093 | } | ||||||||||||
| 2094 | Node *limit = NULL__null; | ||||||||||||
| 2095 | #ifdef ASSERT1 | ||||||||||||
| 2096 | BoolTest::mask bt = cl->loopexit()->test_trip(); | ||||||||||||
| 2097 | assert(bt == BoolTest::lt || bt == BoolTest::gt, "canonical test is expected")do { if (!(bt == BoolTest::lt || bt == BoolTest::gt)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2097, "assert(" "bt == BoolTest::lt || bt == BoolTest::gt" ") failed" , "canonical test is expected"); ::breakpoint(); } } while (0 ); | ||||||||||||
| 2098 | #endif | ||||||||||||
| 2099 | if (cl->has_exact_trip_count()) { | ||||||||||||
| 2100 | // Simple case: loop has constant boundaries. | ||||||||||||
| 2101 | // Use jlongs to avoid integer overflow. | ||||||||||||
| 2102 | int stride_con = cl->stride_con(); | ||||||||||||
| 2103 | jlong init_con = cl->init_trip()->get_int(); | ||||||||||||
| 2104 | jlong limit_con = cl->limit()->get_int(); | ||||||||||||
| 2105 | julong trip_cnt = cl->trip_count(); | ||||||||||||
| 2106 | jlong final_con = init_con + trip_cnt*stride_con; | ||||||||||||
| 2107 | int final_int = (int)final_con; | ||||||||||||
| 2108 | // The final value should be in integer range since the loop | ||||||||||||
| 2109 | // is counted and the limit was checked for overflow. | ||||||||||||
| 2110 | assert(final_con == (jlong)final_int, "final value should be integer")do { if (!(final_con == (jlong)final_int)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2110, "assert(" "final_con == (jlong)final_int" ") failed", "final value should be integer"); ::breakpoint(); } } while ( 0); | ||||||||||||
| 2111 | limit = _igvn.intcon(final_int); | ||||||||||||
| 2112 | } else { | ||||||||||||
| 2113 | // Create new LoopLimit node to get exact limit (final iv value). | ||||||||||||
| 2114 | limit = new LoopLimitNode(C, cl->init_trip(), cl->limit(), cl->stride()); | ||||||||||||
| 2115 | register_new_node(limit, cl->in(LoopNode::EntryControl)); | ||||||||||||
| 2116 | } | ||||||||||||
| 2117 | assert(limit != NULL, "sanity")do { if (!(limit != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2117, "assert(" "limit != __null" ") failed", "sanity"); :: breakpoint(); } } while (0); | ||||||||||||
| 2118 | return limit; | ||||||||||||
| 2119 | } | ||||||||||||
| 2120 | |||||||||||||
| 2121 | //------------------------------Ideal------------------------------------------ | ||||||||||||
| 2122 | // Return a node which is more "ideal" than the current node. | ||||||||||||
| 2123 | // Attempt to convert into a counted-loop. | ||||||||||||
| 2124 | Node *LoopNode::Ideal(PhaseGVN *phase, bool can_reshape) { | ||||||||||||
| 2125 | if (!can_be_counted_loop(phase) && !is_OuterStripMinedLoop()) { | ||||||||||||
| 2126 | phase->C->set_major_progress(); | ||||||||||||
| 2127 | } | ||||||||||||
| 2128 | return RegionNode::Ideal(phase, can_reshape); | ||||||||||||
| 2129 | } | ||||||||||||
| 2130 | |||||||||||||
| 2131 | #ifdef ASSERT1 | ||||||||||||
| 2132 | void LoopNode::verify_strip_mined(int expect_skeleton) const { | ||||||||||||
| 2133 | const OuterStripMinedLoopNode* outer = NULL__null; | ||||||||||||
| 2134 | const CountedLoopNode* inner = NULL__null; | ||||||||||||
| 2135 | if (is_strip_mined()) { | ||||||||||||
| 2136 | if (!is_valid_counted_loop(T_INT)) { | ||||||||||||
| 2137 | return; // Skip malformed counted loop | ||||||||||||
| 2138 | } | ||||||||||||
| 2139 | assert(is_CountedLoop(), "no Loop should be marked strip mined")do { if (!(is_CountedLoop())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2139, "assert(" "is_CountedLoop()" ") failed", "no Loop should be marked strip mined" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2140 | inner = as_CountedLoop(); | ||||||||||||
| 2141 | outer = inner->in(LoopNode::EntryControl)->as_OuterStripMinedLoop(); | ||||||||||||
| 2142 | } else if (is_OuterStripMinedLoop()) { | ||||||||||||
| 2143 | outer = this->as_OuterStripMinedLoop(); | ||||||||||||
| 2144 | inner = outer->unique_ctrl_out()->as_CountedLoop(); | ||||||||||||
| 2145 | assert(inner->is_valid_counted_loop(T_INT) && inner->is_strip_mined(), "OuterStripMinedLoop should have been removed")do { if (!(inner->is_valid_counted_loop(T_INT) && inner ->is_strip_mined())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2145, "assert(" "inner->is_valid_counted_loop(T_INT) && inner->is_strip_mined()" ") failed", "OuterStripMinedLoop should have been removed"); ::breakpoint(); } } while (0); | ||||||||||||
| 2146 | assert(!is_strip_mined(), "outer loop shouldn't be marked strip mined")do { if (!(!is_strip_mined())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2146, "assert(" "!is_strip_mined()" ") failed", "outer loop shouldn't be marked strip mined" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2147 | } | ||||||||||||
| 2148 | if (inner != NULL__null || outer != NULL__null) { | ||||||||||||
| 2149 | assert(inner != NULL && outer != NULL, "missing loop in strip mined nest")do { if (!(inner != __null && outer != __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2149, "assert(" "inner != __null && outer != __null" ") failed", "missing loop in strip mined nest"); ::breakpoint (); } } while (0); | ||||||||||||
| 2150 | Node* outer_tail = outer->in(LoopNode::LoopBackControl); | ||||||||||||
| 2151 | Node* outer_le = outer_tail->in(0); | ||||||||||||
| 2152 | assert(outer_le->Opcode() == Op_OuterStripMinedLoopEnd, "tail of outer loop should be an If")do { if (!(outer_le->Opcode() == Op_OuterStripMinedLoopEnd )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2152, "assert(" "outer_le->Opcode() == Op_OuterStripMinedLoopEnd" ") failed", "tail of outer loop should be an If"); ::breakpoint (); } } while (0); | ||||||||||||
| 2153 | Node* sfpt = outer_le->in(0); | ||||||||||||
| 2154 | assert(sfpt->Opcode() == Op_SafePoint, "where's the safepoint?")do { if (!(sfpt->Opcode() == Op_SafePoint)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2154, "assert(" "sfpt->Opcode() == Op_SafePoint" ") failed" , "where's the safepoint?"); ::breakpoint(); } } while (0); | ||||||||||||
| 2155 | Node* inner_out = sfpt->in(0); | ||||||||||||
| 2156 | CountedLoopEndNode* cle = inner_out->in(0)->as_CountedLoopEnd(); | ||||||||||||
| 2157 | assert(cle == inner->loopexit_or_null(), "mismatch")do { if (!(cle == inner->loopexit_or_null())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2157, "assert(" "cle == inner->loopexit_or_null()" ") failed" , "mismatch"); ::breakpoint(); } } while (0); | ||||||||||||
| 2158 | bool has_skeleton = outer_le->in(1)->bottom_type()->singleton() && outer_le->in(1)->bottom_type()->is_int()->get_con() == 0; | ||||||||||||
| 2159 | if (has_skeleton) { | ||||||||||||
| 2160 | assert(expect_skeleton == 1 || expect_skeleton == -1, "unexpected skeleton node")do { if (!(expect_skeleton == 1 || expect_skeleton == -1)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2160, "assert(" "expect_skeleton == 1 || expect_skeleton == -1" ") failed", "unexpected skeleton node"); ::breakpoint(); } } while (0); | ||||||||||||
| 2161 | assert(outer->outcnt() == 2, "only control nodes")do { if (!(outer->outcnt() == 2)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2161, "assert(" "outer->outcnt() == 2" ") failed", "only control nodes" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2162 | } else { | ||||||||||||
| 2163 | assert(expect_skeleton == 0 || expect_skeleton == -1, "no skeleton node?")do { if (!(expect_skeleton == 0 || expect_skeleton == -1)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2163, "assert(" "expect_skeleton == 0 || expect_skeleton == -1" ") failed", "no skeleton node?"); ::breakpoint(); } } while ( 0); | ||||||||||||
| 2164 | uint phis = 0; | ||||||||||||
| 2165 | uint be_loads = 0; | ||||||||||||
| 2166 | Node* be = inner->in(LoopNode::LoopBackControl); | ||||||||||||
| 2167 | for (DUIterator_Fast imax, i = inner->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 2168 | Node* u = inner->fast_out(i); | ||||||||||||
| 2169 | if (u->is_Phi()) { | ||||||||||||
| 2170 | phis++; | ||||||||||||
| 2171 | for (DUIterator_Fast jmax, j = be->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 2172 | Node* n = be->fast_out(j); | ||||||||||||
| 2173 | if (n->is_Load()) { | ||||||||||||
| 2174 | assert(n->in(0) == be || n->find_prec_edge(be) > 0, "should be on the backedge")do { if (!(n->in(0) == be || n->find_prec_edge(be) > 0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2174, "assert(" "n->in(0) == be || n->find_prec_edge(be) > 0" ") failed", "should be on the backedge"); ::breakpoint(); } } while (0); | ||||||||||||
| 2175 | do { | ||||||||||||
| 2176 | n = n->raw_out(0); | ||||||||||||
| 2177 | } while (!n->is_Phi()); | ||||||||||||
| 2178 | if (n == u) { | ||||||||||||
| 2179 | be_loads++; | ||||||||||||
| 2180 | break; | ||||||||||||
| 2181 | } | ||||||||||||
| 2182 | } | ||||||||||||
| 2183 | } | ||||||||||||
| 2184 | } | ||||||||||||
| 2185 | } | ||||||||||||
| 2186 | assert(be_loads <= phis, "wrong number phis that depends on a pinned load")do { if (!(be_loads <= phis)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2186, "assert(" "be_loads <= phis" ") failed", "wrong number phis that depends on a pinned load" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2187 | for (DUIterator_Fast imax, i = outer->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 2188 | Node* u = outer->fast_out(i); | ||||||||||||
| 2189 | assert(u == outer || u == inner || u->is_Phi(), "nothing between inner and outer loop")do { if (!(u == outer || u == inner || u->is_Phi())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2189, "assert(" "u == outer || u == inner || u->is_Phi()" ") failed", "nothing between inner and outer loop"); ::breakpoint (); } } while (0); | ||||||||||||
| 2190 | } | ||||||||||||
| 2191 | uint stores = 0; | ||||||||||||
| 2192 | for (DUIterator_Fast imax, i = inner_out->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 2193 | Node* u = inner_out->fast_out(i); | ||||||||||||
| 2194 | if (u->is_Store()) { | ||||||||||||
| 2195 | stores++; | ||||||||||||
| 2196 | } | ||||||||||||
| 2197 | } | ||||||||||||
| 2198 | // Late optimization of loads on backedge can cause Phi of outer loop to be eliminated but Phi of inner loop is | ||||||||||||
| 2199 | // not guaranteed to be optimized out. | ||||||||||||
| 2200 | assert(outer->outcnt() >= phis + 2 - be_loads && outer->outcnt() <= phis + 2 + stores + 1, "only phis")do { if (!(outer->outcnt() >= phis + 2 - be_loads && outer->outcnt() <= phis + 2 + stores + 1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2200, "assert(" "outer->outcnt() >= phis + 2 - be_loads && outer->outcnt() <= phis + 2 + stores + 1" ") failed", "only phis"); ::breakpoint(); } } while (0); | ||||||||||||
| 2201 | } | ||||||||||||
| 2202 | assert(sfpt->outcnt() == 1, "no data node")do { if (!(sfpt->outcnt() == 1)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2202, "assert(" "sfpt->outcnt() == 1" ") failed", "no data node" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2203 | assert(outer_tail->outcnt() == 1 || !has_skeleton, "no data node")do { if (!(outer_tail->outcnt() == 1 || !has_skeleton)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2203, "assert(" "outer_tail->outcnt() == 1 || !has_skeleton" ") failed", "no data node"); ::breakpoint(); } } while (0); | ||||||||||||
| 2204 | } | ||||||||||||
| 2205 | } | ||||||||||||
| 2206 | #endif | ||||||||||||
| 2207 | |||||||||||||
| 2208 | //============================================================================= | ||||||||||||
| 2209 | //------------------------------Ideal------------------------------------------ | ||||||||||||
| 2210 | // Return a node which is more "ideal" than the current node. | ||||||||||||
| 2211 | // Attempt to convert into a counted-loop. | ||||||||||||
| 2212 | Node *CountedLoopNode::Ideal(PhaseGVN *phase, bool can_reshape) { | ||||||||||||
| 2213 | return RegionNode::Ideal(phase, can_reshape); | ||||||||||||
| 2214 | } | ||||||||||||
| 2215 | |||||||||||||
| 2216 | //------------------------------dump_spec-------------------------------------- | ||||||||||||
| 2217 | // Dump special per-node info | ||||||||||||
| 2218 | #ifndef PRODUCT | ||||||||||||
| 2219 | void CountedLoopNode::dump_spec(outputStream *st) const { | ||||||||||||
| 2220 | LoopNode::dump_spec(st); | ||||||||||||
| 2221 | if (stride_is_con()) { | ||||||||||||
| 2222 | st->print("stride: %d ",stride_con()); | ||||||||||||
| 2223 | } | ||||||||||||
| 2224 | if (is_pre_loop ()) st->print("pre of N%d" , _main_idx); | ||||||||||||
| 2225 | if (is_main_loop()) st->print("main of N%d", _idx); | ||||||||||||
| 2226 | if (is_post_loop()) st->print("post of N%d", _main_idx); | ||||||||||||
| 2227 | if (is_strip_mined()) st->print(" strip mined"); | ||||||||||||
| 2228 | } | ||||||||||||
| 2229 | #endif | ||||||||||||
| 2230 | |||||||||||||
| 2231 | //============================================================================= | ||||||||||||
| 2232 | jlong BaseCountedLoopEndNode::stride_con() const { | ||||||||||||
| 2233 | return stride()->bottom_type()->is_integer(bt())->get_con_as_long(bt()); | ||||||||||||
| 2234 | } | ||||||||||||
| 2235 | |||||||||||||
| 2236 | |||||||||||||
| 2237 | BaseCountedLoopEndNode* BaseCountedLoopEndNode::make(Node* control, Node* test, float prob, float cnt, BasicType bt) { | ||||||||||||
| 2238 | if (bt == T_INT) { | ||||||||||||
| 2239 | return new CountedLoopEndNode(control, test, prob, cnt); | ||||||||||||
| 2240 | } | ||||||||||||
| 2241 | assert(bt == T_LONG, "unsupported")do { if (!(bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2241, "assert(" "bt == T_LONG" ") failed", "unsupported"); :: breakpoint(); } } while (0); | ||||||||||||
| 2242 | return new LongCountedLoopEndNode(control, test, prob, cnt); | ||||||||||||
| 2243 | } | ||||||||||||
| 2244 | |||||||||||||
| 2245 | //============================================================================= | ||||||||||||
| 2246 | //------------------------------Value----------------------------------------- | ||||||||||||
| 2247 | const Type* LoopLimitNode::Value(PhaseGVN* phase) const { | ||||||||||||
| 2248 | const Type* init_t = phase->type(in(Init)); | ||||||||||||
| 2249 | const Type* limit_t = phase->type(in(Limit)); | ||||||||||||
| 2250 | const Type* stride_t = phase->type(in(Stride)); | ||||||||||||
| 2251 | // Either input is TOP ==> the result is TOP | ||||||||||||
| 2252 | if (init_t == Type::TOP) return Type::TOP; | ||||||||||||
| 2253 | if (limit_t == Type::TOP) return Type::TOP; | ||||||||||||
| 2254 | if (stride_t == Type::TOP) return Type::TOP; | ||||||||||||
| 2255 | |||||||||||||
| 2256 | int stride_con = stride_t->is_int()->get_con(); | ||||||||||||
| 2257 | if (stride_con == 1) | ||||||||||||
| 2258 | return bottom_type(); // Identity | ||||||||||||
| 2259 | |||||||||||||
| 2260 | if (init_t->is_int()->is_con() && limit_t->is_int()->is_con()) { | ||||||||||||
| 2261 | // Use jlongs to avoid integer overflow. | ||||||||||||
| 2262 | jlong init_con = init_t->is_int()->get_con(); | ||||||||||||
| 2263 | jlong limit_con = limit_t->is_int()->get_con(); | ||||||||||||
| 2264 | int stride_m = stride_con - (stride_con > 0 ? 1 : -1); | ||||||||||||
| 2265 | jlong trip_count = (limit_con - init_con + stride_m)/stride_con; | ||||||||||||
| 2266 | jlong final_con = init_con + stride_con*trip_count; | ||||||||||||
| 2267 | int final_int = (int)final_con; | ||||||||||||
| 2268 | // The final value should be in integer range since the loop | ||||||||||||
| 2269 | // is counted and the limit was checked for overflow. | ||||||||||||
| 2270 | assert(final_con == (jlong)final_int, "final value should be integer")do { if (!(final_con == (jlong)final_int)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2270, "assert(" "final_con == (jlong)final_int" ") failed", "final value should be integer"); ::breakpoint(); } } while ( 0); | ||||||||||||
| 2271 | return TypeInt::make(final_int); | ||||||||||||
| 2272 | } | ||||||||||||
| 2273 | |||||||||||||
| 2274 | return bottom_type(); // TypeInt::INT | ||||||||||||
| 2275 | } | ||||||||||||
| 2276 | |||||||||||||
| 2277 | //------------------------------Ideal------------------------------------------ | ||||||||||||
| 2278 | // Return a node which is more "ideal" than the current node. | ||||||||||||
| 2279 | Node *LoopLimitNode::Ideal(PhaseGVN *phase, bool can_reshape) { | ||||||||||||
| 2280 | if (phase->type(in(Init)) == Type::TOP || | ||||||||||||
| 2281 | phase->type(in(Limit)) == Type::TOP || | ||||||||||||
| 2282 | phase->type(in(Stride)) == Type::TOP) | ||||||||||||
| 2283 | return NULL__null; // Dead | ||||||||||||
| 2284 | |||||||||||||
| 2285 | int stride_con = phase->type(in(Stride))->is_int()->get_con(); | ||||||||||||
| 2286 | if (stride_con == 1) | ||||||||||||
| 2287 | return NULL__null; // Identity | ||||||||||||
| 2288 | |||||||||||||
| 2289 | if (in(Init)->is_Con() && in(Limit)->is_Con()) | ||||||||||||
| 2290 | return NULL__null; // Value | ||||||||||||
| 2291 | |||||||||||||
| 2292 | // Delay following optimizations until all loop optimizations | ||||||||||||
| 2293 | // done to keep Ideal graph simple. | ||||||||||||
| 2294 | if (!can_reshape || !phase->C->post_loop_opts_phase()) { | ||||||||||||
| 2295 | return NULL__null; | ||||||||||||
| 2296 | } | ||||||||||||
| 2297 | |||||||||||||
| 2298 | const TypeInt* init_t = phase->type(in(Init) )->is_int(); | ||||||||||||
| 2299 | const TypeInt* limit_t = phase->type(in(Limit))->is_int(); | ||||||||||||
| 2300 | int stride_p; | ||||||||||||
| 2301 | jlong lim, ini; | ||||||||||||
| 2302 | julong max; | ||||||||||||
| 2303 | if (stride_con > 0) { | ||||||||||||
| 2304 | stride_p = stride_con; | ||||||||||||
| 2305 | lim = limit_t->_hi; | ||||||||||||
| 2306 | ini = init_t->_lo; | ||||||||||||
| 2307 | max = (julong)max_jint; | ||||||||||||
| 2308 | } else { | ||||||||||||
| 2309 | stride_p = -stride_con; | ||||||||||||
| 2310 | lim = init_t->_hi; | ||||||||||||
| 2311 | ini = limit_t->_lo; | ||||||||||||
| 2312 | max = (julong)min_jint; | ||||||||||||
| 2313 | } | ||||||||||||
| 2314 | julong range = lim - ini + stride_p; | ||||||||||||
| 2315 | if (range <= max) { | ||||||||||||
| 2316 | // Convert to integer expression if it is not overflow. | ||||||||||||
| 2317 | Node* stride_m = phase->intcon(stride_con - (stride_con > 0 ? 1 : -1)); | ||||||||||||
| 2318 | Node *range = phase->transform(new SubINode(in(Limit), in(Init))); | ||||||||||||
| 2319 | Node *bias = phase->transform(new AddINode(range, stride_m)); | ||||||||||||
| 2320 | Node *trip = phase->transform(new DivINode(0, bias, in(Stride))); | ||||||||||||
| 2321 | Node *span = phase->transform(new MulINode(trip, in(Stride))); | ||||||||||||
| 2322 | return new AddINode(span, in(Init)); // exact limit | ||||||||||||
| 2323 | } | ||||||||||||
| 2324 | |||||||||||||
| 2325 | if (is_power_of_2(stride_p) || // divisor is 2^n | ||||||||||||
| 2326 | !Matcher::has_match_rule(Op_LoopLimit)) { // or no specialized Mach node? | ||||||||||||
| 2327 | // Convert to long expression to avoid integer overflow | ||||||||||||
| 2328 | // and let igvn optimizer convert this division. | ||||||||||||
| 2329 | // | ||||||||||||
| 2330 | Node* init = phase->transform( new ConvI2LNode(in(Init))); | ||||||||||||
| 2331 | Node* limit = phase->transform( new ConvI2LNode(in(Limit))); | ||||||||||||
| 2332 | Node* stride = phase->longcon(stride_con); | ||||||||||||
| 2333 | Node* stride_m = phase->longcon(stride_con - (stride_con > 0 ? 1 : -1)); | ||||||||||||
| 2334 | |||||||||||||
| 2335 | Node *range = phase->transform(new SubLNode(limit, init)); | ||||||||||||
| 2336 | Node *bias = phase->transform(new AddLNode(range, stride_m)); | ||||||||||||
| 2337 | Node *span; | ||||||||||||
| 2338 | if (stride_con > 0 && is_power_of_2(stride_p)) { | ||||||||||||
| 2339 | // bias >= 0 if stride >0, so if stride is 2^n we can use &(-stride) | ||||||||||||
| 2340 | // and avoid generating rounding for division. Zero trip guard should | ||||||||||||
| 2341 | // guarantee that init < limit but sometimes the guard is missing and | ||||||||||||
| 2342 | // we can get situation when init > limit. Note, for the empty loop | ||||||||||||
| 2343 | // optimization zero trip guard is generated explicitly which leaves | ||||||||||||
| 2344 | // only RCE predicate where exact limit is used and the predicate | ||||||||||||
| 2345 | // will simply fail forcing recompilation. | ||||||||||||
| 2346 | Node* neg_stride = phase->longcon(-stride_con); | ||||||||||||
| 2347 | span = phase->transform(new AndLNode(bias, neg_stride)); | ||||||||||||
| 2348 | } else { | ||||||||||||
| 2349 | Node *trip = phase->transform(new DivLNode(0, bias, stride)); | ||||||||||||
| 2350 | span = phase->transform(new MulLNode(trip, stride)); | ||||||||||||
| 2351 | } | ||||||||||||
| 2352 | // Convert back to int | ||||||||||||
| 2353 | Node *span_int = phase->transform(new ConvL2INode(span)); | ||||||||||||
| 2354 | return new AddINode(span_int, in(Init)); // exact limit | ||||||||||||
| 2355 | } | ||||||||||||
| 2356 | |||||||||||||
| 2357 | return NULL__null; // No progress | ||||||||||||
| 2358 | } | ||||||||||||
| 2359 | |||||||||||||
| 2360 | //------------------------------Identity--------------------------------------- | ||||||||||||
| 2361 | // If stride == 1 return limit node. | ||||||||||||
| 2362 | Node* LoopLimitNode::Identity(PhaseGVN* phase) { | ||||||||||||
| 2363 | int stride_con = phase->type(in(Stride))->is_int()->get_con(); | ||||||||||||
| 2364 | if (stride_con == 1 || stride_con == -1) | ||||||||||||
| 2365 | return in(Limit); | ||||||||||||
| 2366 | return this; | ||||||||||||
| 2367 | } | ||||||||||||
| 2368 | |||||||||||||
| 2369 | //============================================================================= | ||||||||||||
| 2370 | //----------------------match_incr_with_optional_truncation-------------------- | ||||||||||||
| 2371 | // Match increment with optional truncation: | ||||||||||||
| 2372 | // CHAR: (i+1)&0x7fff, BYTE: ((i+1)<<8)>>8, or SHORT: ((i+1)<<16)>>16 | ||||||||||||
| 2373 | // Return NULL for failure. Success returns the increment node. | ||||||||||||
| 2374 | Node* CountedLoopNode::match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, | ||||||||||||
| 2375 | const TypeInteger** trunc_type, | ||||||||||||
| 2376 | BasicType bt) { | ||||||||||||
| 2377 | // Quick cutouts: | ||||||||||||
| 2378 | if (expr == NULL__null || expr->req() != 3) return NULL__null; | ||||||||||||
| 2379 | |||||||||||||
| 2380 | Node *t1 = NULL__null; | ||||||||||||
| 2381 | Node *t2 = NULL__null; | ||||||||||||
| 2382 | Node* n1 = expr; | ||||||||||||
| 2383 | int n1op = n1->Opcode(); | ||||||||||||
| 2384 | const TypeInteger* trunc_t = TypeInteger::bottom(bt); | ||||||||||||
| 2385 | |||||||||||||
| 2386 | if (bt == T_INT) { | ||||||||||||
| 2387 | // Try to strip (n1 & M) or (n1 << N >> N) from n1. | ||||||||||||
| 2388 | if (n1op == Op_AndI && | ||||||||||||
| 2389 | n1->in(2)->is_Con() && | ||||||||||||
| 2390 | n1->in(2)->bottom_type()->is_int()->get_con() == 0x7fff) { | ||||||||||||
| 2391 | // %%% This check should match any mask of 2**K-1. | ||||||||||||
| 2392 | t1 = n1; | ||||||||||||
| 2393 | n1 = t1->in(1); | ||||||||||||
| 2394 | n1op = n1->Opcode(); | ||||||||||||
| 2395 | trunc_t = TypeInt::CHAR; | ||||||||||||
| 2396 | } else if (n1op == Op_RShiftI && | ||||||||||||
| 2397 | n1->in(1) != NULL__null && | ||||||||||||
| 2398 | n1->in(1)->Opcode() == Op_LShiftI && | ||||||||||||
| 2399 | n1->in(2) == n1->in(1)->in(2) && | ||||||||||||
| 2400 | n1->in(2)->is_Con()) { | ||||||||||||
| 2401 | jint shift = n1->in(2)->bottom_type()->is_int()->get_con(); | ||||||||||||
| 2402 | // %%% This check should match any shift in [1..31]. | ||||||||||||
| 2403 | if (shift == 16 || shift == 8) { | ||||||||||||
| 2404 | t1 = n1; | ||||||||||||
| 2405 | t2 = t1->in(1); | ||||||||||||
| 2406 | n1 = t2->in(1); | ||||||||||||
| 2407 | n1op = n1->Opcode(); | ||||||||||||
| 2408 | if (shift == 16) { | ||||||||||||
| 2409 | trunc_t = TypeInt::SHORT; | ||||||||||||
| 2410 | } else if (shift == 8) { | ||||||||||||
| 2411 | trunc_t = TypeInt::BYTE; | ||||||||||||
| 2412 | } | ||||||||||||
| 2413 | } | ||||||||||||
| 2414 | } | ||||||||||||
| 2415 | } | ||||||||||||
| 2416 | |||||||||||||
| 2417 | // If (maybe after stripping) it is an AddI, we won: | ||||||||||||
| 2418 | if (n1op == Op_Add(bt)) { | ||||||||||||
| 2419 | *trunc1 = t1; | ||||||||||||
| 2420 | *trunc2 = t2; | ||||||||||||
| 2421 | *trunc_type = trunc_t; | ||||||||||||
| 2422 | return n1; | ||||||||||||
| 2423 | } | ||||||||||||
| 2424 | |||||||||||||
| 2425 | // failed | ||||||||||||
| 2426 | return NULL__null; | ||||||||||||
| 2427 | } | ||||||||||||
| 2428 | |||||||||||||
| 2429 | LoopNode* CountedLoopNode::skip_strip_mined(int expect_skeleton) { | ||||||||||||
| 2430 | if (is_strip_mined() && in(EntryControl) != NULL__null && in(EntryControl)->is_OuterStripMinedLoop()) { | ||||||||||||
| 2431 | verify_strip_mined(expect_skeleton); | ||||||||||||
| 2432 | return in(EntryControl)->as_Loop(); | ||||||||||||
| 2433 | } | ||||||||||||
| 2434 | return this; | ||||||||||||
| 2435 | } | ||||||||||||
| 2436 | |||||||||||||
| 2437 | OuterStripMinedLoopNode* CountedLoopNode::outer_loop() const { | ||||||||||||
| 2438 | assert(is_strip_mined(), "not a strip mined loop")do { if (!(is_strip_mined())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2438, "assert(" "is_strip_mined()" ") failed", "not a strip mined loop" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2439 | Node* c = in(EntryControl); | ||||||||||||
| 2440 | if (c == NULL__null || c->is_top() || !c->is_OuterStripMinedLoop()) { | ||||||||||||
| 2441 | return NULL__null; | ||||||||||||
| 2442 | } | ||||||||||||
| 2443 | return c->as_OuterStripMinedLoop(); | ||||||||||||
| 2444 | } | ||||||||||||
| 2445 | |||||||||||||
| 2446 | IfTrueNode* OuterStripMinedLoopNode::outer_loop_tail() const { | ||||||||||||
| 2447 | Node* c = in(LoopBackControl); | ||||||||||||
| 2448 | if (c == NULL__null || c->is_top()) { | ||||||||||||
| 2449 | return NULL__null; | ||||||||||||
| 2450 | } | ||||||||||||
| 2451 | return c->as_IfTrue(); | ||||||||||||
| 2452 | } | ||||||||||||
| 2453 | |||||||||||||
| 2454 | IfTrueNode* CountedLoopNode::outer_loop_tail() const { | ||||||||||||
| 2455 | LoopNode* l = outer_loop(); | ||||||||||||
| 2456 | if (l == NULL__null) { | ||||||||||||
| 2457 | return NULL__null; | ||||||||||||
| 2458 | } | ||||||||||||
| 2459 | return l->outer_loop_tail(); | ||||||||||||
| 2460 | } | ||||||||||||
| 2461 | |||||||||||||
| 2462 | OuterStripMinedLoopEndNode* OuterStripMinedLoopNode::outer_loop_end() const { | ||||||||||||
| 2463 | IfTrueNode* proj = outer_loop_tail(); | ||||||||||||
| 2464 | if (proj == NULL__null) { | ||||||||||||
| 2465 | return NULL__null; | ||||||||||||
| 2466 | } | ||||||||||||
| 2467 | Node* c = proj->in(0); | ||||||||||||
| 2468 | if (c == NULL__null || c->is_top() || c->outcnt() != 2) { | ||||||||||||
| 2469 | return NULL__null; | ||||||||||||
| 2470 | } | ||||||||||||
| 2471 | return c->as_OuterStripMinedLoopEnd(); | ||||||||||||
| 2472 | } | ||||||||||||
| 2473 | |||||||||||||
| 2474 | OuterStripMinedLoopEndNode* CountedLoopNode::outer_loop_end() const { | ||||||||||||
| 2475 | LoopNode* l = outer_loop(); | ||||||||||||
| 2476 | if (l == NULL__null) { | ||||||||||||
| 2477 | return NULL__null; | ||||||||||||
| 2478 | } | ||||||||||||
| 2479 | return l->outer_loop_end(); | ||||||||||||
| 2480 | } | ||||||||||||
| 2481 | |||||||||||||
| 2482 | IfFalseNode* OuterStripMinedLoopNode::outer_loop_exit() const { | ||||||||||||
| 2483 | IfNode* le = outer_loop_end(); | ||||||||||||
| 2484 | if (le == NULL__null) { | ||||||||||||
| 2485 | return NULL__null; | ||||||||||||
| 2486 | } | ||||||||||||
| 2487 | Node* c = le->proj_out_or_null(false); | ||||||||||||
| 2488 | if (c == NULL__null) { | ||||||||||||
| 2489 | return NULL__null; | ||||||||||||
| 2490 | } | ||||||||||||
| 2491 | return c->as_IfFalse(); | ||||||||||||
| 2492 | } | ||||||||||||
| 2493 | |||||||||||||
| 2494 | IfFalseNode* CountedLoopNode::outer_loop_exit() const { | ||||||||||||
| 2495 | LoopNode* l = outer_loop(); | ||||||||||||
| 2496 | if (l == NULL__null) { | ||||||||||||
| 2497 | return NULL__null; | ||||||||||||
| 2498 | } | ||||||||||||
| 2499 | return l->outer_loop_exit(); | ||||||||||||
| 2500 | } | ||||||||||||
| 2501 | |||||||||||||
| 2502 | SafePointNode* OuterStripMinedLoopNode::outer_safepoint() const { | ||||||||||||
| 2503 | IfNode* le = outer_loop_end(); | ||||||||||||
| 2504 | if (le == NULL__null) { | ||||||||||||
| 2505 | return NULL__null; | ||||||||||||
| 2506 | } | ||||||||||||
| 2507 | Node* c = le->in(0); | ||||||||||||
| 2508 | if (c == NULL__null || c->is_top()) { | ||||||||||||
| 2509 | return NULL__null; | ||||||||||||
| 2510 | } | ||||||||||||
| 2511 | assert(c->Opcode() == Op_SafePoint, "broken outer loop")do { if (!(c->Opcode() == Op_SafePoint)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2511, "assert(" "c->Opcode() == Op_SafePoint" ") failed" , "broken outer loop"); ::breakpoint(); } } while (0); | ||||||||||||
| 2512 | return c->as_SafePoint(); | ||||||||||||
| 2513 | } | ||||||||||||
| 2514 | |||||||||||||
| 2515 | SafePointNode* CountedLoopNode::outer_safepoint() const { | ||||||||||||
| 2516 | LoopNode* l = outer_loop(); | ||||||||||||
| 2517 | if (l == NULL__null) { | ||||||||||||
| 2518 | return NULL__null; | ||||||||||||
| 2519 | } | ||||||||||||
| 2520 | return l->outer_safepoint(); | ||||||||||||
| 2521 | } | ||||||||||||
| 2522 | |||||||||||||
| 2523 | Node* CountedLoopNode::skip_predicates_from_entry(Node* ctrl) { | ||||||||||||
| 2524 | while (ctrl != NULL__null && ctrl->is_Proj() && ctrl->in(0) != NULL__null && ctrl->in(0)->is_If() && | ||||||||||||
| 2525 | (ctrl->in(0)->as_If()->proj_out_or_null(1-ctrl->as_Proj()->_con) == NULL__null || | ||||||||||||
| 2526 | (ctrl->in(0)->as_If()->proj_out(1-ctrl->as_Proj()->_con)->outcnt() == 1 && | ||||||||||||
| 2527 | ctrl->in(0)->as_If()->proj_out(1-ctrl->as_Proj()->_con)->unique_out()->Opcode() == Op_Halt))) { | ||||||||||||
| 2528 | ctrl = ctrl->in(0)->in(0); | ||||||||||||
| 2529 | } | ||||||||||||
| 2530 | |||||||||||||
| 2531 | return ctrl; | ||||||||||||
| 2532 | } | ||||||||||||
| 2533 | |||||||||||||
| 2534 | Node* CountedLoopNode::skip_predicates() { | ||||||||||||
| 2535 | Node* ctrl = in(LoopNode::EntryControl); | ||||||||||||
| 2536 | if (is_main_loop()) { | ||||||||||||
| 2537 | ctrl = skip_strip_mined()->in(LoopNode::EntryControl); | ||||||||||||
| 2538 | } | ||||||||||||
| 2539 | if (is_main_loop() || is_post_loop()) { | ||||||||||||
| 2540 | return skip_predicates_from_entry(ctrl); | ||||||||||||
| 2541 | } | ||||||||||||
| 2542 | return ctrl; | ||||||||||||
| 2543 | } | ||||||||||||
| 2544 | |||||||||||||
| 2545 | |||||||||||||
| 2546 | int CountedLoopNode::stride_con() const { | ||||||||||||
| 2547 | CountedLoopEndNode* cle = loopexit_or_null(); | ||||||||||||
| 2548 | return cle != NULL__null ? cle->stride_con() : 0; | ||||||||||||
| 2549 | } | ||||||||||||
| 2550 | |||||||||||||
| 2551 | BaseCountedLoopNode* BaseCountedLoopNode::make(Node* entry, Node* backedge, BasicType bt) { | ||||||||||||
| 2552 | if (bt == T_INT) { | ||||||||||||
| 2553 | return new CountedLoopNode(entry, backedge); | ||||||||||||
| 2554 | } | ||||||||||||
| 2555 | assert(bt == T_LONG, "unsupported")do { if (!(bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2555, "assert(" "bt == T_LONG" ") failed", "unsupported"); :: breakpoint(); } } while (0); | ||||||||||||
| 2556 | return new LongCountedLoopNode(entry, backedge); | ||||||||||||
| 2557 | } | ||||||||||||
| 2558 | |||||||||||||
| 2559 | |||||||||||||
| 2560 | void OuterStripMinedLoopNode::adjust_strip_mined_loop(PhaseIterGVN* igvn) { | ||||||||||||
| 2561 | // Look for the outer & inner strip mined loop, reduce number of | ||||||||||||
| 2562 | // iterations of the inner loop, set exit condition of outer loop, | ||||||||||||
| 2563 | // construct required phi nodes for outer loop. | ||||||||||||
| 2564 | CountedLoopNode* inner_cl = unique_ctrl_out()->as_CountedLoop(); | ||||||||||||
| 2565 | assert(inner_cl->is_strip_mined(), "inner loop should be strip mined")do { if (!(inner_cl->is_strip_mined())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2565, "assert(" "inner_cl->is_strip_mined()" ") failed", "inner loop should be strip mined"); ::breakpoint(); } } while (0); | ||||||||||||
| 2566 | Node* inner_iv_phi = inner_cl->phi(); | ||||||||||||
| 2567 | if (inner_iv_phi == NULL__null) { | ||||||||||||
| 2568 | IfNode* outer_le = outer_loop_end(); | ||||||||||||
| 2569 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); | ||||||||||||
| 2570 | igvn->replace_node(outer_le, iff); | ||||||||||||
| 2571 | inner_cl->clear_strip_mined(); | ||||||||||||
| 2572 | return; | ||||||||||||
| 2573 | } | ||||||||||||
| 2574 | CountedLoopEndNode* inner_cle = inner_cl->loopexit(); | ||||||||||||
| 2575 | |||||||||||||
| 2576 | int stride = inner_cl->stride_con(); | ||||||||||||
| 2577 | jlong scaled_iters_long = ((jlong)LoopStripMiningIter) * ABS(stride); | ||||||||||||
| 2578 | int scaled_iters = (int)scaled_iters_long; | ||||||||||||
| 2579 | int short_scaled_iters = LoopStripMiningIterShortLoop* ABS(stride); | ||||||||||||
| 2580 | const TypeInt* inner_iv_t = igvn->type(inner_iv_phi)->is_int(); | ||||||||||||
| 2581 | jlong iter_estimate = (jlong)inner_iv_t->_hi - (jlong)inner_iv_t->_lo; | ||||||||||||
| 2582 | assert(iter_estimate > 0, "broken")do { if (!(iter_estimate > 0)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2582, "assert(" "iter_estimate > 0" ") failed", "broken" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2583 | if ((jlong)scaled_iters != scaled_iters_long || iter_estimate <= short_scaled_iters) { | ||||||||||||
| 2584 | // Remove outer loop and safepoint (too few iterations) | ||||||||||||
| 2585 | Node* outer_sfpt = outer_safepoint(); | ||||||||||||
| 2586 | Node* outer_out = outer_loop_exit(); | ||||||||||||
| 2587 | igvn->replace_node(outer_out, outer_sfpt->in(0)); | ||||||||||||
| 2588 | igvn->replace_input_of(outer_sfpt, 0, igvn->C->top()); | ||||||||||||
| 2589 | inner_cl->clear_strip_mined(); | ||||||||||||
| 2590 | return; | ||||||||||||
| 2591 | } | ||||||||||||
| 2592 | if (iter_estimate <= scaled_iters_long) { | ||||||||||||
| 2593 | // We would only go through one iteration of | ||||||||||||
| 2594 | // the outer loop: drop the outer loop but | ||||||||||||
| 2595 | // keep the safepoint so we don't run for | ||||||||||||
| 2596 | // too long without a safepoint | ||||||||||||
| 2597 | IfNode* outer_le = outer_loop_end(); | ||||||||||||
| 2598 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); | ||||||||||||
| 2599 | igvn->replace_node(outer_le, iff); | ||||||||||||
| 2600 | inner_cl->clear_strip_mined(); | ||||||||||||
| 2601 | return; | ||||||||||||
| 2602 | } | ||||||||||||
| 2603 | |||||||||||||
| 2604 | Node* cle_tail = inner_cle->proj_out(true); | ||||||||||||
| 2605 | ResourceMark rm; | ||||||||||||
| 2606 | Node_List old_new; | ||||||||||||
| 2607 | if (cle_tail->outcnt() > 1) { | ||||||||||||
| 2608 | // Look for nodes on backedge of inner loop and clone them | ||||||||||||
| 2609 | Unique_Node_List backedge_nodes; | ||||||||||||
| 2610 | for (DUIterator_Fast imax, i = cle_tail->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 2611 | Node* u = cle_tail->fast_out(i); | ||||||||||||
| 2612 | if (u != inner_cl) { | ||||||||||||
| 2613 | assert(!u->is_CFG(), "control flow on the backedge?")do { if (!(!u->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2613, "assert(" "!u->is_CFG()" ") failed", "control flow on the backedge?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2614 | backedge_nodes.push(u); | ||||||||||||
| 2615 | } | ||||||||||||
| 2616 | } | ||||||||||||
| 2617 | uint last = igvn->C->unique(); | ||||||||||||
| 2618 | for (uint next = 0; next < backedge_nodes.size(); next++) { | ||||||||||||
| 2619 | Node* n = backedge_nodes.at(next); | ||||||||||||
| 2620 | old_new.map(n->_idx, n->clone()); | ||||||||||||
| 2621 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 2622 | Node* u = n->fast_out(i); | ||||||||||||
| 2623 | assert(!u->is_CFG(), "broken")do { if (!(!u->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2623, "assert(" "!u->is_CFG()" ") failed", "broken"); :: breakpoint(); } } while (0); | ||||||||||||
| 2624 | if (u->_idx >= last) { | ||||||||||||
| 2625 | continue; | ||||||||||||
| 2626 | } | ||||||||||||
| 2627 | if (!u->is_Phi()) { | ||||||||||||
| 2628 | backedge_nodes.push(u); | ||||||||||||
| 2629 | } else { | ||||||||||||
| 2630 | assert(u->in(0) == inner_cl, "strange phi on the backedge")do { if (!(u->in(0) == inner_cl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2630, "assert(" "u->in(0) == inner_cl" ") failed", "strange phi on the backedge" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2631 | } | ||||||||||||
| 2632 | } | ||||||||||||
| 2633 | } | ||||||||||||
| 2634 | // Put the clones on the outer loop backedge | ||||||||||||
| 2635 | Node* le_tail = outer_loop_tail(); | ||||||||||||
| 2636 | for (uint next = 0; next < backedge_nodes.size(); next++) { | ||||||||||||
| 2637 | Node *n = old_new[backedge_nodes.at(next)->_idx]; | ||||||||||||
| 2638 | for (uint i = 1; i < n->req(); i++) { | ||||||||||||
| 2639 | if (n->in(i) != NULL__null && old_new[n->in(i)->_idx] != NULL__null) { | ||||||||||||
| 2640 | n->set_req(i, old_new[n->in(i)->_idx]); | ||||||||||||
| 2641 | } | ||||||||||||
| 2642 | } | ||||||||||||
| 2643 | if (n->in(0) != NULL__null && n->in(0) == cle_tail) { | ||||||||||||
| 2644 | n->set_req(0, le_tail); | ||||||||||||
| 2645 | } | ||||||||||||
| 2646 | igvn->register_new_node_with_optimizer(n); | ||||||||||||
| 2647 | } | ||||||||||||
| 2648 | } | ||||||||||||
| 2649 | |||||||||||||
| 2650 | Node* iv_phi = NULL__null; | ||||||||||||
| 2651 | // Make a clone of each phi in the inner loop | ||||||||||||
| 2652 | // for the outer loop | ||||||||||||
| 2653 | for (uint i = 0; i < inner_cl->outcnt(); i++) { | ||||||||||||
| 2654 | Node* u = inner_cl->raw_out(i); | ||||||||||||
| 2655 | if (u->is_Phi()) { | ||||||||||||
| 2656 | assert(u->in(0) == inner_cl, "inconsistent")do { if (!(u->in(0) == inner_cl)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2656, "assert(" "u->in(0) == inner_cl" ") failed", "inconsistent" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2657 | Node* phi = u->clone(); | ||||||||||||
| 2658 | phi->set_req(0, this); | ||||||||||||
| 2659 | Node* be = old_new[phi->in(LoopNode::LoopBackControl)->_idx]; | ||||||||||||
| 2660 | if (be != NULL__null) { | ||||||||||||
| 2661 | phi->set_req(LoopNode::LoopBackControl, be); | ||||||||||||
| 2662 | } | ||||||||||||
| 2663 | phi = igvn->transform(phi); | ||||||||||||
| 2664 | igvn->replace_input_of(u, LoopNode::EntryControl, phi); | ||||||||||||
| 2665 | if (u == inner_iv_phi) { | ||||||||||||
| 2666 | iv_phi = phi; | ||||||||||||
| 2667 | } | ||||||||||||
| 2668 | } | ||||||||||||
| 2669 | } | ||||||||||||
| 2670 | Node* cle_out = inner_cle->proj_out(false); | ||||||||||||
| 2671 | if (cle_out->outcnt() > 1) { | ||||||||||||
| 2672 | // Look for chains of stores that were sunk | ||||||||||||
| 2673 | // out of the inner loop and are in the outer loop | ||||||||||||
| 2674 | for (DUIterator_Fast imax, i = cle_out->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 2675 | Node* u = cle_out->fast_out(i); | ||||||||||||
| 2676 | if (u->is_Store()) { | ||||||||||||
| 2677 | Node* first = u; | ||||||||||||
| 2678 | for(;;) { | ||||||||||||
| 2679 | Node* next = first->in(MemNode::Memory); | ||||||||||||
| 2680 | if (!next->is_Store() || next->in(0) != cle_out) { | ||||||||||||
| 2681 | break; | ||||||||||||
| 2682 | } | ||||||||||||
| 2683 | first = next; | ||||||||||||
| 2684 | } | ||||||||||||
| 2685 | Node* last = u; | ||||||||||||
| 2686 | for(;;) { | ||||||||||||
| 2687 | Node* next = NULL__null; | ||||||||||||
| 2688 | for (DUIterator_Fast jmax, j = last->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 2689 | Node* uu = last->fast_out(j); | ||||||||||||
| 2690 | if (uu->is_Store() && uu->in(0) == cle_out) { | ||||||||||||
| 2691 | assert(next == NULL, "only one in the outer loop")do { if (!(next == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2691, "assert(" "next == __null" ") failed", "only one in the outer loop" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2692 | next = uu; | ||||||||||||
| 2693 | } | ||||||||||||
| 2694 | } | ||||||||||||
| 2695 | if (next == NULL__null) { | ||||||||||||
| 2696 | break; | ||||||||||||
| 2697 | } | ||||||||||||
| 2698 | last = next; | ||||||||||||
| 2699 | } | ||||||||||||
| 2700 | Node* phi = NULL__null; | ||||||||||||
| 2701 | for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 2702 | Node* uu = fast_out(j); | ||||||||||||
| 2703 | if (uu->is_Phi()) { | ||||||||||||
| 2704 | Node* be = uu->in(LoopNode::LoopBackControl); | ||||||||||||
| 2705 | if (be->is_Store() && old_new[be->_idx] != NULL__null) { | ||||||||||||
| 2706 | assert(false, "store on the backedge + sunk stores: unsupported")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2706, "assert(" "false" ") failed", "store on the backedge + sunk stores: unsupported" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2707 | // drop outer loop | ||||||||||||
| 2708 | IfNode* outer_le = outer_loop_end(); | ||||||||||||
| 2709 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); | ||||||||||||
| 2710 | igvn->replace_node(outer_le, iff); | ||||||||||||
| 2711 | inner_cl->clear_strip_mined(); | ||||||||||||
| 2712 | return; | ||||||||||||
| 2713 | } | ||||||||||||
| 2714 | if (be == last || be == first->in(MemNode::Memory)) { | ||||||||||||
| 2715 | assert(phi == NULL, "only one phi")do { if (!(phi == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2715, "assert(" "phi == __null" ") failed", "only one phi") ; ::breakpoint(); } } while (0); | ||||||||||||
| 2716 | phi = uu; | ||||||||||||
| 2717 | } | ||||||||||||
| 2718 | } | ||||||||||||
| 2719 | } | ||||||||||||
| 2720 | #ifdef ASSERT1 | ||||||||||||
| 2721 | for (DUIterator_Fast jmax, j = fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 2722 | Node* uu = fast_out(j); | ||||||||||||
| 2723 | if (uu->is_Phi() && uu->bottom_type() == Type::MEMORY) { | ||||||||||||
| 2724 | if (uu->adr_type() == igvn->C->get_adr_type(igvn->C->get_alias_index(u->adr_type()))) { | ||||||||||||
| 2725 | assert(phi == uu, "what's that phi?")do { if (!(phi == uu)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2725, "assert(" "phi == uu" ") failed", "what's that phi?") ; ::breakpoint(); } } while (0); | ||||||||||||
| 2726 | } else if (uu->adr_type() == TypePtr::BOTTOM) { | ||||||||||||
| 2727 | Node* n = uu->in(LoopNode::LoopBackControl); | ||||||||||||
| 2728 | uint limit = igvn->C->live_nodes(); | ||||||||||||
| 2729 | uint i = 0; | ||||||||||||
| 2730 | while (n != uu) { | ||||||||||||
| 2731 | i++; | ||||||||||||
| 2732 | assert(i < limit, "infinite loop")do { if (!(i < limit)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2732, "assert(" "i < limit" ") failed", "infinite loop") ; ::breakpoint(); } } while (0); | ||||||||||||
| 2733 | if (n->is_Proj()) { | ||||||||||||
| 2734 | n = n->in(0); | ||||||||||||
| 2735 | } else if (n->is_SafePoint() || n->is_MemBar()) { | ||||||||||||
| 2736 | n = n->in(TypeFunc::Memory); | ||||||||||||
| 2737 | } else if (n->is_Phi()) { | ||||||||||||
| 2738 | n = n->in(1); | ||||||||||||
| 2739 | } else if (n->is_MergeMem()) { | ||||||||||||
| 2740 | n = n->as_MergeMem()->memory_at(igvn->C->get_alias_index(u->adr_type())); | ||||||||||||
| 2741 | } else if (n->is_Store() || n->is_LoadStore() || n->is_ClearArray()) { | ||||||||||||
| 2742 | n = n->in(MemNode::Memory); | ||||||||||||
| 2743 | } else { | ||||||||||||
| 2744 | n->dump(); | ||||||||||||
| 2745 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2745); ::breakpoint(); } while (0); | ||||||||||||
| 2746 | } | ||||||||||||
| 2747 | } | ||||||||||||
| 2748 | } | ||||||||||||
| 2749 | } | ||||||||||||
| 2750 | } | ||||||||||||
| 2751 | #endif | ||||||||||||
| 2752 | if (phi == NULL__null) { | ||||||||||||
| 2753 | // If the an entire chains was sunk, the | ||||||||||||
| 2754 | // inner loop has no phi for that memory | ||||||||||||
| 2755 | // slice, create one for the outer loop | ||||||||||||
| 2756 | phi = PhiNode::make(this, first->in(MemNode::Memory), Type::MEMORY, | ||||||||||||
| 2757 | igvn->C->get_adr_type(igvn->C->get_alias_index(u->adr_type()))); | ||||||||||||
| 2758 | phi->set_req(LoopNode::LoopBackControl, last); | ||||||||||||
| 2759 | phi = igvn->transform(phi); | ||||||||||||
| 2760 | igvn->replace_input_of(first, MemNode::Memory, phi); | ||||||||||||
| 2761 | } else { | ||||||||||||
| 2762 | // Or fix the outer loop fix to include | ||||||||||||
| 2763 | // that chain of stores. | ||||||||||||
| 2764 | Node* be = phi->in(LoopNode::LoopBackControl); | ||||||||||||
| 2765 | assert(!(be->is_Store() && old_new[be->_idx] != NULL), "store on the backedge + sunk stores: unsupported")do { if (!(!(be->is_Store() && old_new[be->_idx ] != __null))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2765, "assert(" "!(be->is_Store() && old_new[be->_idx] != __null)" ") failed", "store on the backedge + sunk stores: unsupported" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2766 | if (be == first->in(MemNode::Memory)) { | ||||||||||||
| 2767 | if (be == phi->in(LoopNode::LoopBackControl)) { | ||||||||||||
| 2768 | igvn->replace_input_of(phi, LoopNode::LoopBackControl, last); | ||||||||||||
| 2769 | } else { | ||||||||||||
| 2770 | igvn->replace_input_of(be, MemNode::Memory, last); | ||||||||||||
| 2771 | } | ||||||||||||
| 2772 | } else { | ||||||||||||
| 2773 | #ifdef ASSERT1 | ||||||||||||
| 2774 | if (be == phi->in(LoopNode::LoopBackControl)) { | ||||||||||||
| 2775 | assert(phi->in(LoopNode::LoopBackControl) == last, "")do { if (!(phi->in(LoopNode::LoopBackControl) == last)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2775, "assert(" "phi->in(LoopNode::LoopBackControl) == last" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 2776 | } else { | ||||||||||||
| 2777 | assert(be->in(MemNode::Memory) == last, "")do { if (!(be->in(MemNode::Memory) == last)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2777, "assert(" "be->in(MemNode::Memory) == last" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 2778 | } | ||||||||||||
| 2779 | #endif | ||||||||||||
| 2780 | } | ||||||||||||
| 2781 | } | ||||||||||||
| 2782 | } | ||||||||||||
| 2783 | } | ||||||||||||
| 2784 | } | ||||||||||||
| 2785 | |||||||||||||
| 2786 | if (iv_phi != NULL__null) { | ||||||||||||
| 2787 | // Now adjust the inner loop's exit condition | ||||||||||||
| 2788 | Node* limit = inner_cl->limit(); | ||||||||||||
| 2789 | // If limit < init for stride > 0 (or limit > init for stride < 0), | ||||||||||||
| 2790 | // the loop body is run only once. Given limit - init (init - limit resp.) | ||||||||||||
| 2791 | // would be negative, the unsigned comparison below would cause | ||||||||||||
| 2792 | // the loop body to be run for LoopStripMiningIter. | ||||||||||||
| 2793 | Node* max = NULL__null; | ||||||||||||
| 2794 | if (stride > 0) { | ||||||||||||
| 2795 | max = MaxNode::max_diff_with_zero(limit, iv_phi, TypeInt::INT, *igvn); | ||||||||||||
| 2796 | } else { | ||||||||||||
| 2797 | max = MaxNode::max_diff_with_zero(iv_phi, limit, TypeInt::INT, *igvn); | ||||||||||||
| 2798 | } | ||||||||||||
| 2799 | // sub is positive and can be larger than the max signed int | ||||||||||||
| 2800 | // value. Use an unsigned min. | ||||||||||||
| 2801 | Node* const_iters = igvn->intcon(scaled_iters); | ||||||||||||
| 2802 | Node* min = MaxNode::unsigned_min(max, const_iters, TypeInt::make(0, scaled_iters, Type::WidenMin), *igvn); | ||||||||||||
| 2803 | // min is the number of iterations for the next inner loop execution: | ||||||||||||
| 2804 | // unsigned_min(max(limit - iv_phi, 0), scaled_iters) if stride > 0 | ||||||||||||
| 2805 | // unsigned_min(max(iv_phi - limit, 0), scaled_iters) if stride < 0 | ||||||||||||
| 2806 | |||||||||||||
| 2807 | Node* new_limit = NULL__null; | ||||||||||||
| 2808 | if (stride > 0) { | ||||||||||||
| 2809 | new_limit = igvn->transform(new AddINode(min, iv_phi)); | ||||||||||||
| 2810 | } else { | ||||||||||||
| 2811 | new_limit = igvn->transform(new SubINode(iv_phi, min)); | ||||||||||||
| 2812 | } | ||||||||||||
| 2813 | Node* inner_cmp = inner_cle->cmp_node(); | ||||||||||||
| 2814 | Node* inner_bol = inner_cle->in(CountedLoopEndNode::TestValue); | ||||||||||||
| 2815 | Node* outer_bol = inner_bol; | ||||||||||||
| 2816 | // cmp node for inner loop may be shared | ||||||||||||
| 2817 | inner_cmp = inner_cmp->clone(); | ||||||||||||
| 2818 | inner_cmp->set_req(2, new_limit); | ||||||||||||
| 2819 | inner_bol = inner_bol->clone(); | ||||||||||||
| 2820 | inner_bol->set_req(1, igvn->transform(inner_cmp)); | ||||||||||||
| 2821 | igvn->replace_input_of(inner_cle, CountedLoopEndNode::TestValue, igvn->transform(inner_bol)); | ||||||||||||
| 2822 | // Set the outer loop's exit condition too | ||||||||||||
| 2823 | igvn->replace_input_of(outer_loop_end(), 1, outer_bol); | ||||||||||||
| 2824 | } else { | ||||||||||||
| 2825 | assert(false, "should be able to adjust outer loop")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2825, "assert(" "false" ") failed", "should be able to adjust outer loop" ); ::breakpoint(); } } while (0); | ||||||||||||
| 2826 | IfNode* outer_le = outer_loop_end(); | ||||||||||||
| 2827 | Node* iff = igvn->transform(new IfNode(outer_le->in(0), outer_le->in(1), outer_le->_prob, outer_le->_fcnt)); | ||||||||||||
| 2828 | igvn->replace_node(outer_le, iff); | ||||||||||||
| 2829 | inner_cl->clear_strip_mined(); | ||||||||||||
| 2830 | } | ||||||||||||
| 2831 | } | ||||||||||||
| 2832 | |||||||||||||
| 2833 | const Type* OuterStripMinedLoopEndNode::Value(PhaseGVN* phase) const { | ||||||||||||
| 2834 | if (!in(0)) return Type::TOP; | ||||||||||||
| 2835 | if (phase->type(in(0)) == Type::TOP) | ||||||||||||
| 2836 | return Type::TOP; | ||||||||||||
| 2837 | |||||||||||||
| 2838 | // Until expansion, the loop end condition is not set so this should not constant fold. | ||||||||||||
| 2839 | if (is_expanded(phase)) { | ||||||||||||
| 2840 | return IfNode::Value(phase); | ||||||||||||
| 2841 | } | ||||||||||||
| 2842 | |||||||||||||
| 2843 | return TypeTuple::IFBOTH; | ||||||||||||
| 2844 | } | ||||||||||||
| 2845 | |||||||||||||
| 2846 | bool OuterStripMinedLoopEndNode::is_expanded(PhaseGVN *phase) const { | ||||||||||||
| 2847 | // The outer strip mined loop head only has Phi uses after expansion | ||||||||||||
| 2848 | if (phase->is_IterGVN()) { | ||||||||||||
| 2849 | Node* backedge = proj_out_or_null(true); | ||||||||||||
| 2850 | if (backedge != NULL__null) { | ||||||||||||
| 2851 | Node* head = backedge->unique_ctrl_out(); | ||||||||||||
| 2852 | if (head != NULL__null && head->is_OuterStripMinedLoop()) { | ||||||||||||
| 2853 | if (head->find_out_with(Op_Phi) != NULL__null) { | ||||||||||||
| 2854 | return true; | ||||||||||||
| 2855 | } | ||||||||||||
| 2856 | } | ||||||||||||
| 2857 | } | ||||||||||||
| 2858 | } | ||||||||||||
| 2859 | return false; | ||||||||||||
| 2860 | } | ||||||||||||
| 2861 | |||||||||||||
| 2862 | Node *OuterStripMinedLoopEndNode::Ideal(PhaseGVN *phase, bool can_reshape) { | ||||||||||||
| 2863 | if (remove_dead_region(phase, can_reshape)) return this; | ||||||||||||
| 2864 | |||||||||||||
| 2865 | return NULL__null; | ||||||||||||
| 2866 | } | ||||||||||||
| 2867 | |||||||||||||
| 2868 | //------------------------------filtered_type-------------------------------- | ||||||||||||
| 2869 | // Return a type based on condition control flow | ||||||||||||
| 2870 | // A successful return will be a type that is restricted due | ||||||||||||
| 2871 | // to a series of dominating if-tests, such as: | ||||||||||||
| 2872 | // if (i < 10) { | ||||||||||||
| 2873 | // if (i > 0) { | ||||||||||||
| 2874 | // here: "i" type is [1..10) | ||||||||||||
| 2875 | // } | ||||||||||||
| 2876 | // } | ||||||||||||
| 2877 | // or a control flow merge | ||||||||||||
| 2878 | // if (i < 10) { | ||||||||||||
| 2879 | // do { | ||||||||||||
| 2880 | // phi( , ) -- at top of loop type is [min_int..10) | ||||||||||||
| 2881 | // i = ? | ||||||||||||
| 2882 | // } while ( i < 10) | ||||||||||||
| 2883 | // | ||||||||||||
| 2884 | const TypeInt* PhaseIdealLoop::filtered_type( Node *n, Node* n_ctrl) { | ||||||||||||
| 2885 | assert(n && n->bottom_type()->is_int(), "must be int")do { if (!(n && n->bottom_type()->is_int())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2885, "assert(" "n && n->bottom_type()->is_int()" ") failed", "must be int"); ::breakpoint(); } } while (0); | ||||||||||||
| 2886 | const TypeInt* filtered_t = NULL__null; | ||||||||||||
| 2887 | if (!n->is_Phi()) { | ||||||||||||
| 2888 | assert(n_ctrl != NULL || n_ctrl == C->top(), "valid control")do { if (!(n_ctrl != __null || n_ctrl == C->top())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2888, "assert(" "n_ctrl != __null || n_ctrl == C->top()" ") failed", "valid control"); ::breakpoint(); } } while (0); | ||||||||||||
| 2889 | filtered_t = filtered_type_from_dominators(n, n_ctrl); | ||||||||||||
| 2890 | |||||||||||||
| 2891 | } else { | ||||||||||||
| 2892 | Node* phi = n->as_Phi(); | ||||||||||||
| 2893 | Node* region = phi->in(0); | ||||||||||||
| 2894 | assert(n_ctrl == NULL || n_ctrl == region, "ctrl parameter must be region")do { if (!(n_ctrl == __null || n_ctrl == region)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2894, "assert(" "n_ctrl == __null || n_ctrl == region" ") failed" , "ctrl parameter must be region"); ::breakpoint(); } } while (0); | ||||||||||||
| 2895 | if (region && region != C->top()) { | ||||||||||||
| 2896 | for (uint i = 1; i < phi->req(); i++) { | ||||||||||||
| 2897 | Node* val = phi->in(i); | ||||||||||||
| 2898 | Node* use_c = region->in(i); | ||||||||||||
| 2899 | const TypeInt* val_t = filtered_type_from_dominators(val, use_c); | ||||||||||||
| 2900 | if (val_t != NULL__null) { | ||||||||||||
| 2901 | if (filtered_t == NULL__null) { | ||||||||||||
| 2902 | filtered_t = val_t; | ||||||||||||
| 2903 | } else { | ||||||||||||
| 2904 | filtered_t = filtered_t->meet(val_t)->is_int(); | ||||||||||||
| 2905 | } | ||||||||||||
| 2906 | } | ||||||||||||
| 2907 | } | ||||||||||||
| 2908 | } | ||||||||||||
| 2909 | } | ||||||||||||
| 2910 | const TypeInt* n_t = _igvn.type(n)->is_int(); | ||||||||||||
| 2911 | if (filtered_t != NULL__null) { | ||||||||||||
| 2912 | n_t = n_t->join(filtered_t)->is_int(); | ||||||||||||
| 2913 | } | ||||||||||||
| 2914 | return n_t; | ||||||||||||
| 2915 | } | ||||||||||||
| 2916 | |||||||||||||
| 2917 | |||||||||||||
| 2918 | //------------------------------filtered_type_from_dominators-------------------------------- | ||||||||||||
| 2919 | // Return a possibly more restrictive type for val based on condition control flow of dominators | ||||||||||||
| 2920 | const TypeInt* PhaseIdealLoop::filtered_type_from_dominators( Node* val, Node *use_ctrl) { | ||||||||||||
| 2921 | if (val->is_Con()) { | ||||||||||||
| 2922 | return val->bottom_type()->is_int(); | ||||||||||||
| 2923 | } | ||||||||||||
| 2924 | uint if_limit = 10; // Max number of dominating if's visited | ||||||||||||
| 2925 | const TypeInt* rtn_t = NULL__null; | ||||||||||||
| 2926 | |||||||||||||
| 2927 | if (use_ctrl && use_ctrl != C->top()) { | ||||||||||||
| 2928 | Node* val_ctrl = get_ctrl(val); | ||||||||||||
| 2929 | uint val_dom_depth = dom_depth(val_ctrl); | ||||||||||||
| 2930 | Node* pred = use_ctrl; | ||||||||||||
| 2931 | uint if_cnt = 0; | ||||||||||||
| 2932 | while (if_cnt < if_limit) { | ||||||||||||
| 2933 | if ((pred->Opcode() == Op_IfTrue || pred->Opcode() == Op_IfFalse)) { | ||||||||||||
| 2934 | if_cnt++; | ||||||||||||
| 2935 | const TypeInt* if_t = IfNode::filtered_int_type(&_igvn, val, pred); | ||||||||||||
| 2936 | if (if_t != NULL__null) { | ||||||||||||
| 2937 | if (rtn_t == NULL__null) { | ||||||||||||
| 2938 | rtn_t = if_t; | ||||||||||||
| 2939 | } else { | ||||||||||||
| 2940 | rtn_t = rtn_t->join(if_t)->is_int(); | ||||||||||||
| 2941 | } | ||||||||||||
| 2942 | } | ||||||||||||
| 2943 | } | ||||||||||||
| 2944 | pred = idom(pred); | ||||||||||||
| 2945 | if (pred == NULL__null || pred == C->top()) { | ||||||||||||
| 2946 | break; | ||||||||||||
| 2947 | } | ||||||||||||
| 2948 | // Stop if going beyond definition block of val | ||||||||||||
| 2949 | if (dom_depth(pred) < val_dom_depth) { | ||||||||||||
| 2950 | break; | ||||||||||||
| 2951 | } | ||||||||||||
| 2952 | } | ||||||||||||
| 2953 | } | ||||||||||||
| 2954 | return rtn_t; | ||||||||||||
| 2955 | } | ||||||||||||
| 2956 | |||||||||||||
| 2957 | |||||||||||||
| 2958 | //------------------------------dump_spec-------------------------------------- | ||||||||||||
| 2959 | // Dump special per-node info | ||||||||||||
| 2960 | #ifndef PRODUCT | ||||||||||||
| 2961 | void CountedLoopEndNode::dump_spec(outputStream *st) const { | ||||||||||||
| 2962 | if( in(TestValue) != NULL__null && in(TestValue)->is_Bool() ) { | ||||||||||||
| 2963 | BoolTest bt( test_trip()); // Added this for g++. | ||||||||||||
| 2964 | |||||||||||||
| 2965 | st->print("["); | ||||||||||||
| 2966 | bt.dump_on(st); | ||||||||||||
| 2967 | st->print("]"); | ||||||||||||
| 2968 | } | ||||||||||||
| 2969 | st->print(" "); | ||||||||||||
| 2970 | IfNode::dump_spec(st); | ||||||||||||
| 2971 | } | ||||||||||||
| 2972 | #endif | ||||||||||||
| 2973 | |||||||||||||
| 2974 | //============================================================================= | ||||||||||||
| 2975 | //------------------------------is_member-------------------------------------- | ||||||||||||
| 2976 | // Is 'l' a member of 'this'? | ||||||||||||
| 2977 | bool IdealLoopTree::is_member(const IdealLoopTree *l) const { | ||||||||||||
| 2978 | while( l->_nest > _nest ) l = l->_parent; | ||||||||||||
| 2979 | return l == this; | ||||||||||||
| 2980 | } | ||||||||||||
| 2981 | |||||||||||||
| 2982 | //------------------------------set_nest--------------------------------------- | ||||||||||||
| 2983 | // Set loop tree nesting depth. Accumulate _has_call bits. | ||||||||||||
| 2984 | int IdealLoopTree::set_nest( uint depth ) { | ||||||||||||
| 2985 | assert(depth <= SHRT_MAX, "sanity")do { if (!(depth <= 32767)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 2985, "assert(" "depth <= 32767" ") failed", "sanity"); :: breakpoint(); } } while (0); | ||||||||||||
| 2986 | _nest = depth; | ||||||||||||
| 2987 | int bits = _has_call; | ||||||||||||
| 2988 | if( _child ) bits |= _child->set_nest(depth+1); | ||||||||||||
| 2989 | if( bits ) _has_call = 1; | ||||||||||||
| 2990 | if( _next ) bits |= _next ->set_nest(depth ); | ||||||||||||
| 2991 | return bits; | ||||||||||||
| 2992 | } | ||||||||||||
| 2993 | |||||||||||||
| 2994 | //------------------------------split_fall_in---------------------------------- | ||||||||||||
| 2995 | // Split out multiple fall-in edges from the loop header. Move them to a | ||||||||||||
| 2996 | // private RegionNode before the loop. This becomes the loop landing pad. | ||||||||||||
| 2997 | void IdealLoopTree::split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ) { | ||||||||||||
| 2998 | PhaseIterGVN &igvn = phase->_igvn; | ||||||||||||
| 2999 | uint i; | ||||||||||||
| 3000 | |||||||||||||
| 3001 | // Make a new RegionNode to be the landing pad. | ||||||||||||
| 3002 | Node *landing_pad = new RegionNode( fall_in_cnt+1 ); | ||||||||||||
| 3003 | phase->set_loop(landing_pad,_parent); | ||||||||||||
| 3004 | // Gather all the fall-in control paths into the landing pad | ||||||||||||
| 3005 | uint icnt = fall_in_cnt; | ||||||||||||
| 3006 | uint oreq = _head->req(); | ||||||||||||
| 3007 | for( i = oreq-1; i>0; i-- ) | ||||||||||||
| 3008 | if( !phase->is_member( this, _head->in(i) ) ) | ||||||||||||
| 3009 | landing_pad->set_req(icnt--,_head->in(i)); | ||||||||||||
| 3010 | |||||||||||||
| 3011 | // Peel off PhiNode edges as well | ||||||||||||
| 3012 | for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 3013 | Node *oj = _head->fast_out(j); | ||||||||||||
| 3014 | if( oj->is_Phi() ) { | ||||||||||||
| 3015 | PhiNode* old_phi = oj->as_Phi(); | ||||||||||||
| 3016 | assert( old_phi->region() == _head, "" )do { if (!(old_phi->region() == _head)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3016, "assert(" "old_phi->region() == _head" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 3017 | igvn.hash_delete(old_phi); // Yank from hash before hacking edges | ||||||||||||
| 3018 | Node *p = PhiNode::make_blank(landing_pad, old_phi); | ||||||||||||
| 3019 | uint icnt = fall_in_cnt; | ||||||||||||
| 3020 | for( i = oreq-1; i>0; i-- ) { | ||||||||||||
| 3021 | if( !phase->is_member( this, _head->in(i) ) ) { | ||||||||||||
| 3022 | p->init_req(icnt--, old_phi->in(i)); | ||||||||||||
| 3023 | // Go ahead and clean out old edges from old phi | ||||||||||||
| 3024 | old_phi->del_req(i); | ||||||||||||
| 3025 | } | ||||||||||||
| 3026 | } | ||||||||||||
| 3027 | // Search for CSE's here, because ZKM.jar does a lot of | ||||||||||||
| 3028 | // loop hackery and we need to be a little incremental | ||||||||||||
| 3029 | // with the CSE to avoid O(N^2) node blow-up. | ||||||||||||
| 3030 | Node *p2 = igvn.hash_find_insert(p); // Look for a CSE | ||||||||||||
| 3031 | if( p2 ) { // Found CSE | ||||||||||||
| 3032 | p->destruct(&igvn); // Recover useless new node | ||||||||||||
| 3033 | p = p2; // Use old node | ||||||||||||
| 3034 | } else { | ||||||||||||
| 3035 | igvn.register_new_node_with_optimizer(p, old_phi); | ||||||||||||
| 3036 | } | ||||||||||||
| 3037 | // Make old Phi refer to new Phi. | ||||||||||||
| 3038 | old_phi->add_req(p); | ||||||||||||
| 3039 | // Check for the special case of making the old phi useless and | ||||||||||||
| 3040 | // disappear it. In JavaGrande I have a case where this useless | ||||||||||||
| 3041 | // Phi is the loop limit and prevents recognizing a CountedLoop | ||||||||||||
| 3042 | // which in turn prevents removing an empty loop. | ||||||||||||
| 3043 | Node *id_old_phi = old_phi->Identity(&igvn); | ||||||||||||
| 3044 | if( id_old_phi != old_phi ) { // Found a simple identity? | ||||||||||||
| 3045 | // Note that I cannot call 'replace_node' here, because | ||||||||||||
| 3046 | // that will yank the edge from old_phi to the Region and | ||||||||||||
| 3047 | // I'm mid-iteration over the Region's uses. | ||||||||||||
| 3048 | for (DUIterator_Last imin, i = old_phi->last_outs(imin); i >= imin; ) { | ||||||||||||
| 3049 | Node* use = old_phi->last_out(i); | ||||||||||||
| 3050 | igvn.rehash_node_delayed(use); | ||||||||||||
| 3051 | uint uses_found = 0; | ||||||||||||
| 3052 | for (uint j = 0; j < use->len(); j++) { | ||||||||||||
| 3053 | if (use->in(j) == old_phi) { | ||||||||||||
| 3054 | if (j < use->req()) use->set_req (j, id_old_phi); | ||||||||||||
| 3055 | else use->set_prec(j, id_old_phi); | ||||||||||||
| 3056 | uses_found++; | ||||||||||||
| 3057 | } | ||||||||||||
| 3058 | } | ||||||||||||
| 3059 | i -= uses_found; // we deleted 1 or more copies of this edge | ||||||||||||
| 3060 | } | ||||||||||||
| 3061 | } | ||||||||||||
| 3062 | igvn._worklist.push(old_phi); | ||||||||||||
| 3063 | } | ||||||||||||
| 3064 | } | ||||||||||||
| 3065 | // Finally clean out the fall-in edges from the RegionNode | ||||||||||||
| 3066 | for( i = oreq-1; i>0; i-- ) { | ||||||||||||
| 3067 | if( !phase->is_member( this, _head->in(i) ) ) { | ||||||||||||
| 3068 | _head->del_req(i); | ||||||||||||
| 3069 | } | ||||||||||||
| 3070 | } | ||||||||||||
| 3071 | igvn.rehash_node_delayed(_head); | ||||||||||||
| 3072 | // Transform landing pad | ||||||||||||
| 3073 | igvn.register_new_node_with_optimizer(landing_pad, _head); | ||||||||||||
| 3074 | // Insert landing pad into the header | ||||||||||||
| 3075 | _head->add_req(landing_pad); | ||||||||||||
| 3076 | } | ||||||||||||
| 3077 | |||||||||||||
| 3078 | //------------------------------split_outer_loop------------------------------- | ||||||||||||
| 3079 | // Split out the outermost loop from this shared header. | ||||||||||||
| 3080 | void IdealLoopTree::split_outer_loop( PhaseIdealLoop *phase ) { | ||||||||||||
| 3081 | PhaseIterGVN &igvn = phase->_igvn; | ||||||||||||
| 3082 | |||||||||||||
| 3083 | // Find index of outermost loop; it should also be my tail. | ||||||||||||
| 3084 | uint outer_idx = 1; | ||||||||||||
| 3085 | while( _head->in(outer_idx) != _tail ) outer_idx++; | ||||||||||||
| 3086 | |||||||||||||
| 3087 | // Make a LoopNode for the outermost loop. | ||||||||||||
| 3088 | Node *ctl = _head->in(LoopNode::EntryControl); | ||||||||||||
| 3089 | Node *outer = new LoopNode( ctl, _head->in(outer_idx) ); | ||||||||||||
| 3090 | outer = igvn.register_new_node_with_optimizer(outer, _head); | ||||||||||||
| 3091 | phase->set_created_loop_node(); | ||||||||||||
| 3092 | |||||||||||||
| 3093 | // Outermost loop falls into '_head' loop | ||||||||||||
| 3094 | _head->set_req(LoopNode::EntryControl, outer); | ||||||||||||
| 3095 | _head->del_req(outer_idx); | ||||||||||||
| 3096 | // Split all the Phis up between '_head' loop and 'outer' loop. | ||||||||||||
| 3097 | for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 3098 | Node *out = _head->fast_out(j); | ||||||||||||
| 3099 | if( out->is_Phi() ) { | ||||||||||||
| 3100 | PhiNode *old_phi = out->as_Phi(); | ||||||||||||
| 3101 | assert( old_phi->region() == _head, "" )do { if (!(old_phi->region() == _head)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3101, "assert(" "old_phi->region() == _head" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 3102 | Node *phi = PhiNode::make_blank(outer, old_phi); | ||||||||||||
| 3103 | phi->init_req(LoopNode::EntryControl, old_phi->in(LoopNode::EntryControl)); | ||||||||||||
| 3104 | phi->init_req(LoopNode::LoopBackControl, old_phi->in(outer_idx)); | ||||||||||||
| 3105 | phi = igvn.register_new_node_with_optimizer(phi, old_phi); | ||||||||||||
| 3106 | // Make old Phi point to new Phi on the fall-in path | ||||||||||||
| 3107 | igvn.replace_input_of(old_phi, LoopNode::EntryControl, phi); | ||||||||||||
| 3108 | old_phi->del_req(outer_idx); | ||||||||||||
| 3109 | } | ||||||||||||
| 3110 | } | ||||||||||||
| 3111 | |||||||||||||
| 3112 | // Use the new loop head instead of the old shared one | ||||||||||||
| 3113 | _head = outer; | ||||||||||||
| 3114 | phase->set_loop(_head, this); | ||||||||||||
| 3115 | } | ||||||||||||
| 3116 | |||||||||||||
| 3117 | //------------------------------fix_parent------------------------------------- | ||||||||||||
| 3118 | static void fix_parent( IdealLoopTree *loop, IdealLoopTree *parent ) { | ||||||||||||
| 3119 | loop->_parent = parent; | ||||||||||||
| 3120 | if( loop->_child ) fix_parent( loop->_child, loop ); | ||||||||||||
| 3121 | if( loop->_next ) fix_parent( loop->_next , parent ); | ||||||||||||
| 3122 | } | ||||||||||||
| 3123 | |||||||||||||
| 3124 | //------------------------------estimate_path_freq----------------------------- | ||||||||||||
| 3125 | static float estimate_path_freq( Node *n ) { | ||||||||||||
| 3126 | // Try to extract some path frequency info | ||||||||||||
| 3127 | IfNode *iff; | ||||||||||||
| 3128 | for( int i = 0; i < 50; i++ ) { // Skip through a bunch of uncommon tests | ||||||||||||
| 3129 | uint nop = n->Opcode(); | ||||||||||||
| 3130 | if( nop == Op_SafePoint ) { // Skip any safepoint | ||||||||||||
| 3131 | n = n->in(0); | ||||||||||||
| 3132 | continue; | ||||||||||||
| 3133 | } | ||||||||||||
| 3134 | if( nop == Op_CatchProj ) { // Get count from a prior call | ||||||||||||
| 3135 | // Assume call does not always throw exceptions: means the call-site | ||||||||||||
| 3136 | // count is also the frequency of the fall-through path. | ||||||||||||
| 3137 | assert( n->is_CatchProj(), "" )do { if (!(n->is_CatchProj())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3137, "assert(" "n->is_CatchProj()" ") failed", ""); ::breakpoint (); } } while (0); | ||||||||||||
| 3138 | if( ((CatchProjNode*)n)->_con != CatchProjNode::fall_through_index ) | ||||||||||||
| 3139 | return 0.0f; // Assume call exception path is rare | ||||||||||||
| 3140 | Node *call = n->in(0)->in(0)->in(0); | ||||||||||||
| 3141 | assert( call->is_Call(), "expect a call here" )do { if (!(call->is_Call())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3141, "assert(" "call->is_Call()" ") failed", "expect a call here" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3142 | const JVMState *jvms = ((CallNode*)call)->jvms(); | ||||||||||||
| 3143 | ciMethodData* methodData = jvms->method()->method_data(); | ||||||||||||
| 3144 | if (!methodData->is_mature()) return 0.0f; // No call-site data | ||||||||||||
| 3145 | ciProfileData* data = methodData->bci_to_data(jvms->bci()); | ||||||||||||
| 3146 | if ((data == NULL__null) || !data->is_CounterData()) { | ||||||||||||
| 3147 | // no call profile available, try call's control input | ||||||||||||
| 3148 | n = n->in(0); | ||||||||||||
| 3149 | continue; | ||||||||||||
| 3150 | } | ||||||||||||
| 3151 | return data->as_CounterData()->count()/FreqCountInvocations; | ||||||||||||
| 3152 | } | ||||||||||||
| 3153 | // See if there's a gating IF test | ||||||||||||
| 3154 | Node *n_c = n->in(0); | ||||||||||||
| 3155 | if( !n_c->is_If() ) break; // No estimate available | ||||||||||||
| 3156 | iff = n_c->as_If(); | ||||||||||||
| 3157 | if( iff->_fcnt != COUNT_UNKNOWN(-1.0f) ) // Have a valid count? | ||||||||||||
| 3158 | // Compute how much count comes on this path | ||||||||||||
| 3159 | return ((nop == Op_IfTrue) ? iff->_prob : 1.0f - iff->_prob) * iff->_fcnt; | ||||||||||||
| 3160 | // Have no count info. Skip dull uncommon-trap like branches. | ||||||||||||
| 3161 | if( (nop == Op_IfTrue && iff->_prob < PROB_LIKELY_MAG(5)(1.0f-(1e-5f))) || | ||||||||||||
| 3162 | (nop == Op_IfFalse && iff->_prob > PROB_UNLIKELY_MAG(5)(1e-5f)) ) | ||||||||||||
| 3163 | break; | ||||||||||||
| 3164 | // Skip through never-taken branch; look for a real loop exit. | ||||||||||||
| 3165 | n = iff->in(0); | ||||||||||||
| 3166 | } | ||||||||||||
| 3167 | return 0.0f; // No estimate available | ||||||||||||
| 3168 | } | ||||||||||||
| 3169 | |||||||||||||
| 3170 | //------------------------------merge_many_backedges--------------------------- | ||||||||||||
| 3171 | // Merge all the backedges from the shared header into a private Region. | ||||||||||||
| 3172 | // Feed that region as the one backedge to this loop. | ||||||||||||
| 3173 | void IdealLoopTree::merge_many_backedges( PhaseIdealLoop *phase ) { | ||||||||||||
| 3174 | uint i; | ||||||||||||
| 3175 | |||||||||||||
| 3176 | // Scan for the top 2 hottest backedges | ||||||||||||
| 3177 | float hotcnt = 0.0f; | ||||||||||||
| 3178 | float warmcnt = 0.0f; | ||||||||||||
| 3179 | uint hot_idx = 0; | ||||||||||||
| 3180 | // Loop starts at 2 because slot 1 is the fall-in path | ||||||||||||
| 3181 | for( i = 2; i < _head->req(); i++ ) { | ||||||||||||
| 3182 | float cnt = estimate_path_freq(_head->in(i)); | ||||||||||||
| 3183 | if( cnt > hotcnt ) { // Grab hottest path | ||||||||||||
| 3184 | warmcnt = hotcnt; | ||||||||||||
| 3185 | hotcnt = cnt; | ||||||||||||
| 3186 | hot_idx = i; | ||||||||||||
| 3187 | } else if( cnt > warmcnt ) { // And 2nd hottest path | ||||||||||||
| 3188 | warmcnt = cnt; | ||||||||||||
| 3189 | } | ||||||||||||
| 3190 | } | ||||||||||||
| 3191 | |||||||||||||
| 3192 | // See if the hottest backedge is worthy of being an inner loop | ||||||||||||
| 3193 | // by being much hotter than the next hottest backedge. | ||||||||||||
| 3194 | if( hotcnt <= 0.0001 || | ||||||||||||
| 3195 | hotcnt < 2.0*warmcnt ) hot_idx = 0;// No hot backedge | ||||||||||||
| 3196 | |||||||||||||
| 3197 | // Peel out the backedges into a private merge point; peel | ||||||||||||
| 3198 | // them all except optionally hot_idx. | ||||||||||||
| 3199 | PhaseIterGVN &igvn = phase->_igvn; | ||||||||||||
| 3200 | |||||||||||||
| 3201 | Node *hot_tail = NULL__null; | ||||||||||||
| 3202 | // Make a Region for the merge point | ||||||||||||
| 3203 | Node *r = new RegionNode(1); | ||||||||||||
| 3204 | for( i = 2; i < _head->req(); i++ ) { | ||||||||||||
| 3205 | if( i != hot_idx ) | ||||||||||||
| 3206 | r->add_req( _head->in(i) ); | ||||||||||||
| 3207 | else hot_tail = _head->in(i); | ||||||||||||
| 3208 | } | ||||||||||||
| 3209 | igvn.register_new_node_with_optimizer(r, _head); | ||||||||||||
| 3210 | // Plug region into end of loop _head, followed by hot_tail | ||||||||||||
| 3211 | while( _head->req() > 3 ) _head->del_req( _head->req()-1 ); | ||||||||||||
| 3212 | igvn.replace_input_of(_head, 2, r); | ||||||||||||
| 3213 | if( hot_idx ) _head->add_req(hot_tail); | ||||||||||||
| 3214 | |||||||||||||
| 3215 | // Split all the Phis up between '_head' loop and the Region 'r' | ||||||||||||
| 3216 | for (DUIterator_Fast jmax, j = _head->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 3217 | Node *out = _head->fast_out(j); | ||||||||||||
| 3218 | if( out->is_Phi() ) { | ||||||||||||
| 3219 | PhiNode* n = out->as_Phi(); | ||||||||||||
| 3220 | igvn.hash_delete(n); // Delete from hash before hacking edges | ||||||||||||
| 3221 | Node *hot_phi = NULL__null; | ||||||||||||
| 3222 | Node *phi = new PhiNode(r, n->type(), n->adr_type()); | ||||||||||||
| 3223 | // Check all inputs for the ones to peel out | ||||||||||||
| 3224 | uint j = 1; | ||||||||||||
| 3225 | for( uint i = 2; i < n->req(); i++ ) { | ||||||||||||
| 3226 | if( i != hot_idx ) | ||||||||||||
| 3227 | phi->set_req( j++, n->in(i) ); | ||||||||||||
| 3228 | else hot_phi = n->in(i); | ||||||||||||
| 3229 | } | ||||||||||||
| 3230 | // Register the phi but do not transform until whole place transforms | ||||||||||||
| 3231 | igvn.register_new_node_with_optimizer(phi, n); | ||||||||||||
| 3232 | // Add the merge phi to the old Phi | ||||||||||||
| 3233 | while( n->req() > 3 ) n->del_req( n->req()-1 ); | ||||||||||||
| 3234 | igvn.replace_input_of(n, 2, phi); | ||||||||||||
| 3235 | if( hot_idx ) n->add_req(hot_phi); | ||||||||||||
| 3236 | } | ||||||||||||
| 3237 | } | ||||||||||||
| 3238 | |||||||||||||
| 3239 | |||||||||||||
| 3240 | // Insert a new IdealLoopTree inserted below me. Turn it into a clone | ||||||||||||
| 3241 | // of self loop tree. Turn self into a loop headed by _head and with | ||||||||||||
| 3242 | // tail being the new merge point. | ||||||||||||
| 3243 | IdealLoopTree *ilt = new IdealLoopTree( phase, _head, _tail ); | ||||||||||||
| 3244 | phase->set_loop(_tail,ilt); // Adjust tail | ||||||||||||
| 3245 | _tail = r; // Self's tail is new merge point | ||||||||||||
| 3246 | phase->set_loop(r,this); | ||||||||||||
| 3247 | ilt->_child = _child; // New guy has my children | ||||||||||||
| 3248 | _child = ilt; // Self has new guy as only child | ||||||||||||
| 3249 | ilt->_parent = this; // new guy has self for parent | ||||||||||||
| 3250 | ilt->_nest = _nest; // Same nesting depth (for now) | ||||||||||||
| 3251 | |||||||||||||
| 3252 | // Starting with 'ilt', look for child loop trees using the same shared | ||||||||||||
| 3253 | // header. Flatten these out; they will no longer be loops in the end. | ||||||||||||
| 3254 | IdealLoopTree **pilt = &_child; | ||||||||||||
| 3255 | while( ilt ) { | ||||||||||||
| 3256 | if( ilt->_head == _head ) { | ||||||||||||
| 3257 | uint i; | ||||||||||||
| 3258 | for( i = 2; i < _head->req(); i++ ) | ||||||||||||
| 3259 | if( _head->in(i) == ilt->_tail ) | ||||||||||||
| 3260 | break; // Still a loop | ||||||||||||
| 3261 | if( i == _head->req() ) { // No longer a loop | ||||||||||||
| 3262 | // Flatten ilt. Hang ilt's "_next" list from the end of | ||||||||||||
| 3263 | // ilt's '_child' list. Move the ilt's _child up to replace ilt. | ||||||||||||
| 3264 | IdealLoopTree **cp = &ilt->_child; | ||||||||||||
| 3265 | while( *cp ) cp = &(*cp)->_next; // Find end of child list | ||||||||||||
| 3266 | *cp = ilt->_next; // Hang next list at end of child list | ||||||||||||
| 3267 | *pilt = ilt->_child; // Move child up to replace ilt | ||||||||||||
| 3268 | ilt->_head = NULL__null; // Flag as a loop UNIONED into parent | ||||||||||||
| 3269 | ilt = ilt->_child; // Repeat using new ilt | ||||||||||||
| 3270 | continue; // do not advance over ilt->_child | ||||||||||||
| 3271 | } | ||||||||||||
| 3272 | assert( ilt->_tail == hot_tail, "expected to only find the hot inner loop here" )do { if (!(ilt->_tail == hot_tail)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3272, "assert(" "ilt->_tail == hot_tail" ") failed", "expected to only find the hot inner loop here" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3273 | phase->set_loop(_head,ilt); | ||||||||||||
| 3274 | } | ||||||||||||
| 3275 | pilt = &ilt->_child; // Advance to next | ||||||||||||
| 3276 | ilt = *pilt; | ||||||||||||
| 3277 | } | ||||||||||||
| 3278 | |||||||||||||
| 3279 | if( _child ) fix_parent( _child, this ); | ||||||||||||
| 3280 | } | ||||||||||||
| 3281 | |||||||||||||
| 3282 | //------------------------------beautify_loops--------------------------------- | ||||||||||||
| 3283 | // Split shared headers and insert loop landing pads. | ||||||||||||
| 3284 | // Insert a LoopNode to replace the RegionNode. | ||||||||||||
| 3285 | // Return TRUE if loop tree is structurally changed. | ||||||||||||
| 3286 | bool IdealLoopTree::beautify_loops( PhaseIdealLoop *phase ) { | ||||||||||||
| 3287 | bool result = false; | ||||||||||||
| 3288 | // Cache parts in locals for easy | ||||||||||||
| 3289 | PhaseIterGVN &igvn = phase->_igvn; | ||||||||||||
| 3290 | |||||||||||||
| 3291 | igvn.hash_delete(_head); // Yank from hash before hacking edges | ||||||||||||
| 3292 | |||||||||||||
| 3293 | // Check for multiple fall-in paths. Peel off a landing pad if need be. | ||||||||||||
| 3294 | int fall_in_cnt = 0; | ||||||||||||
| 3295 | for( uint i = 1; i < _head->req(); i++ ) | ||||||||||||
| 3296 | if( !phase->is_member( this, _head->in(i) ) ) | ||||||||||||
| 3297 | fall_in_cnt++; | ||||||||||||
| 3298 | assert( fall_in_cnt, "at least 1 fall-in path" )do { if (!(fall_in_cnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3298, "assert(" "fall_in_cnt" ") failed", "at least 1 fall-in path" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3299 | if( fall_in_cnt > 1 ) // Need a loop landing pad to merge fall-ins | ||||||||||||
| 3300 | split_fall_in( phase, fall_in_cnt ); | ||||||||||||
| 3301 | |||||||||||||
| 3302 | // Swap inputs to the _head and all Phis to move the fall-in edge to | ||||||||||||
| 3303 | // the left. | ||||||||||||
| 3304 | fall_in_cnt = 1; | ||||||||||||
| 3305 | while( phase->is_member( this, _head->in(fall_in_cnt) ) ) | ||||||||||||
| 3306 | fall_in_cnt++; | ||||||||||||
| 3307 | if( fall_in_cnt > 1 ) { | ||||||||||||
| 3308 | // Since I am just swapping inputs I do not need to update def-use info | ||||||||||||
| 3309 | Node *tmp = _head->in(1); | ||||||||||||
| 3310 | igvn.rehash_node_delayed(_head); | ||||||||||||
| 3311 | _head->set_req( 1, _head->in(fall_in_cnt) ); | ||||||||||||
| 3312 | _head->set_req( fall_in_cnt, tmp ); | ||||||||||||
| 3313 | // Swap also all Phis | ||||||||||||
| 3314 | for (DUIterator_Fast imax, i = _head->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 3315 | Node* phi = _head->fast_out(i); | ||||||||||||
| 3316 | if( phi->is_Phi() ) { | ||||||||||||
| 3317 | igvn.rehash_node_delayed(phi); // Yank from hash before hacking edges | ||||||||||||
| 3318 | tmp = phi->in(1); | ||||||||||||
| 3319 | phi->set_req( 1, phi->in(fall_in_cnt) ); | ||||||||||||
| 3320 | phi->set_req( fall_in_cnt, tmp ); | ||||||||||||
| 3321 | } | ||||||||||||
| 3322 | } | ||||||||||||
| 3323 | } | ||||||||||||
| 3324 | assert( !phase->is_member( this, _head->in(1) ), "left edge is fall-in" )do { if (!(!phase->is_member( this, _head->in(1) ))) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3324, "assert(" "!phase->is_member( this, _head->in(1) )" ") failed", "left edge is fall-in"); ::breakpoint(); } } while (0); | ||||||||||||
| 3325 | assert( phase->is_member( this, _head->in(2) ), "right edge is loop" )do { if (!(phase->is_member( this, _head->in(2) ))) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3325, "assert(" "phase->is_member( this, _head->in(2) )" ") failed", "right edge is loop"); ::breakpoint(); } } while (0); | ||||||||||||
| 3326 | |||||||||||||
| 3327 | // If I am a shared header (multiple backedges), peel off the many | ||||||||||||
| 3328 | // backedges into a private merge point and use the merge point as | ||||||||||||
| 3329 | // the one true backedge. | ||||||||||||
| 3330 | if (_head->req() > 3) { | ||||||||||||
| 3331 | // Merge the many backedges into a single backedge but leave | ||||||||||||
| 3332 | // the hottest backedge as separate edge for the following peel. | ||||||||||||
| 3333 | if (!_irreducible) { | ||||||||||||
| 3334 | merge_many_backedges( phase ); | ||||||||||||
| 3335 | } | ||||||||||||
| 3336 | |||||||||||||
| 3337 | // When recursively beautify my children, split_fall_in can change | ||||||||||||
| 3338 | // loop tree structure when I am an irreducible loop. Then the head | ||||||||||||
| 3339 | // of my children has a req() not bigger than 3. Here we need to set | ||||||||||||
| 3340 | // result to true to catch that case in order to tell the caller to | ||||||||||||
| 3341 | // rebuild loop tree. See issue JDK-8244407 for details. | ||||||||||||
| 3342 | result = true; | ||||||||||||
| 3343 | } | ||||||||||||
| 3344 | |||||||||||||
| 3345 | // If I have one hot backedge, peel off myself loop. | ||||||||||||
| 3346 | // I better be the outermost loop. | ||||||||||||
| 3347 | if (_head->req() > 3 && !_irreducible) { | ||||||||||||
| 3348 | split_outer_loop( phase ); | ||||||||||||
| 3349 | result = true; | ||||||||||||
| 3350 | |||||||||||||
| 3351 | } else if (!_head->is_Loop() && !_irreducible) { | ||||||||||||
| 3352 | // Make a new LoopNode to replace the old loop head | ||||||||||||
| 3353 | Node *l = new LoopNode( _head->in(1), _head->in(2) ); | ||||||||||||
| 3354 | l = igvn.register_new_node_with_optimizer(l, _head); | ||||||||||||
| 3355 | phase->set_created_loop_node(); | ||||||||||||
| 3356 | // Go ahead and replace _head | ||||||||||||
| 3357 | phase->_igvn.replace_node( _head, l ); | ||||||||||||
| 3358 | _head = l; | ||||||||||||
| 3359 | phase->set_loop(_head, this); | ||||||||||||
| 3360 | } | ||||||||||||
| 3361 | |||||||||||||
| 3362 | // Now recursively beautify nested loops | ||||||||||||
| 3363 | if( _child ) result |= _child->beautify_loops( phase ); | ||||||||||||
| 3364 | if( _next ) result |= _next ->beautify_loops( phase ); | ||||||||||||
| 3365 | return result; | ||||||||||||
| 3366 | } | ||||||||||||
| 3367 | |||||||||||||
| 3368 | //------------------------------allpaths_check_safepts---------------------------- | ||||||||||||
| 3369 | // Allpaths backwards scan from loop tail, terminating each path at first safepoint | ||||||||||||
| 3370 | // encountered. Helper for check_safepts. | ||||||||||||
| 3371 | void IdealLoopTree::allpaths_check_safepts(VectorSet &visited, Node_List &stack) { | ||||||||||||
| 3372 | assert(stack.size() == 0, "empty stack")do { if (!(stack.size() == 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3372, "assert(" "stack.size() == 0" ") failed", "empty stack" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3373 | stack.push(_tail); | ||||||||||||
| 3374 | visited.clear(); | ||||||||||||
| 3375 | visited.set(_tail->_idx); | ||||||||||||
| 3376 | while (stack.size() > 0) { | ||||||||||||
| 3377 | Node* n = stack.pop(); | ||||||||||||
| 3378 | if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) { | ||||||||||||
| 3379 | // Terminate this path | ||||||||||||
| 3380 | } else if (n->Opcode() == Op_SafePoint) { | ||||||||||||
| 3381 | if (_phase->get_loop(n) != this) { | ||||||||||||
| 3382 | if (_required_safept == NULL__null) _required_safept = new Node_List(); | ||||||||||||
| 3383 | _required_safept->push(n); // save the one closest to the tail | ||||||||||||
| 3384 | } | ||||||||||||
| 3385 | // Terminate this path | ||||||||||||
| 3386 | } else { | ||||||||||||
| 3387 | uint start = n->is_Region() ? 1 : 0; | ||||||||||||
| 3388 | uint end = n->is_Region() && !n->is_Loop() ? n->req() : start + 1; | ||||||||||||
| 3389 | for (uint i = start; i < end; i++) { | ||||||||||||
| 3390 | Node* in = n->in(i); | ||||||||||||
| 3391 | assert(in->is_CFG(), "must be")do { if (!(in->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3391, "assert(" "in->is_CFG()" ") failed", "must be"); :: breakpoint(); } } while (0); | ||||||||||||
| 3392 | if (!visited.test_set(in->_idx) && is_member(_phase->get_loop(in))) { | ||||||||||||
| 3393 | stack.push(in); | ||||||||||||
| 3394 | } | ||||||||||||
| 3395 | } | ||||||||||||
| 3396 | } | ||||||||||||
| 3397 | } | ||||||||||||
| 3398 | } | ||||||||||||
| 3399 | |||||||||||||
| 3400 | //------------------------------check_safepts---------------------------- | ||||||||||||
| 3401 | // Given dominators, try to find loops with calls that must always be | ||||||||||||
| 3402 | // executed (call dominates loop tail). These loops do not need non-call | ||||||||||||
| 3403 | // safepoints (ncsfpt). | ||||||||||||
| 3404 | // | ||||||||||||
| 3405 | // A complication is that a safepoint in a inner loop may be needed | ||||||||||||
| 3406 | // by an outer loop. In the following, the inner loop sees it has a | ||||||||||||
| 3407 | // call (block 3) on every path from the head (block 2) to the | ||||||||||||
| 3408 | // backedge (arc 3->2). So it deletes the ncsfpt (non-call safepoint) | ||||||||||||
| 3409 | // in block 2, _but_ this leaves the outer loop without a safepoint. | ||||||||||||
| 3410 | // | ||||||||||||
| 3411 | // entry 0 | ||||||||||||
| 3412 | // | | ||||||||||||
| 3413 | // v | ||||||||||||
| 3414 | // outer 1,2 +->1 | ||||||||||||
| 3415 | // | | | ||||||||||||
| 3416 | // | v | ||||||||||||
| 3417 | // | 2<---+ ncsfpt in 2 | ||||||||||||
| 3418 | // |_/|\ | | ||||||||||||
| 3419 | // | v | | ||||||||||||
| 3420 | // inner 2,3 / 3 | call in 3 | ||||||||||||
| 3421 | // / | | | ||||||||||||
| 3422 | // v +--+ | ||||||||||||
| 3423 | // exit 4 | ||||||||||||
| 3424 | // | ||||||||||||
| 3425 | // | ||||||||||||
| 3426 | // This method creates a list (_required_safept) of ncsfpt nodes that must | ||||||||||||
| 3427 | // be protected is created for each loop. When a ncsfpt maybe deleted, it | ||||||||||||
| 3428 | // is first looked for in the lists for the outer loops of the current loop. | ||||||||||||
| 3429 | // | ||||||||||||
| 3430 | // The insights into the problem: | ||||||||||||
| 3431 | // A) counted loops are okay | ||||||||||||
| 3432 | // B) innermost loops are okay (only an inner loop can delete | ||||||||||||
| 3433 | // a ncsfpt needed by an outer loop) | ||||||||||||
| 3434 | // C) a loop is immune from an inner loop deleting a safepoint | ||||||||||||
| 3435 | // if the loop has a call on the idom-path | ||||||||||||
| 3436 | // D) a loop is also immune if it has a ncsfpt (non-call safepoint) on the | ||||||||||||
| 3437 | // idom-path that is not in a nested loop | ||||||||||||
| 3438 | // E) otherwise, an ncsfpt on the idom-path that is nested in an inner | ||||||||||||
| 3439 | // loop needs to be prevented from deletion by an inner loop | ||||||||||||
| 3440 | // | ||||||||||||
| 3441 | // There are two analyses: | ||||||||||||
| 3442 | // 1) The first, and cheaper one, scans the loop body from | ||||||||||||
| 3443 | // tail to head following the idom (immediate dominator) | ||||||||||||
| 3444 | // chain, looking for the cases (C,D,E) above. | ||||||||||||
| 3445 | // Since inner loops are scanned before outer loops, there is summary | ||||||||||||
| 3446 | // information about inner loops. Inner loops can be skipped over | ||||||||||||
| 3447 | // when the tail of an inner loop is encountered. | ||||||||||||
| 3448 | // | ||||||||||||
| 3449 | // 2) The second, invoked if the first fails to find a call or ncsfpt on | ||||||||||||
| 3450 | // the idom path (which is rare), scans all predecessor control paths | ||||||||||||
| 3451 | // from the tail to the head, terminating a path when a call or sfpt | ||||||||||||
| 3452 | // is encountered, to find the ncsfpt's that are closest to the tail. | ||||||||||||
| 3453 | // | ||||||||||||
| 3454 | void IdealLoopTree::check_safepts(VectorSet &visited, Node_List &stack) { | ||||||||||||
| 3455 | // Bottom up traversal | ||||||||||||
| 3456 | IdealLoopTree* ch = _child; | ||||||||||||
| 3457 | if (_child) _child->check_safepts(visited, stack); | ||||||||||||
| 3458 | if (_next) _next ->check_safepts(visited, stack); | ||||||||||||
| 3459 | |||||||||||||
| 3460 | if (!_head->is_CountedLoop() && !_has_sfpt && _parent != NULL__null && !_irreducible) { | ||||||||||||
| 3461 | bool has_call = false; // call on dom-path | ||||||||||||
| 3462 | bool has_local_ncsfpt = false; // ncsfpt on dom-path at this loop depth | ||||||||||||
| 3463 | Node* nonlocal_ncsfpt = NULL__null; // ncsfpt on dom-path at a deeper depth | ||||||||||||
| 3464 | // Scan the dom-path nodes from tail to head | ||||||||||||
| 3465 | for (Node* n = tail(); n != _head; n = _phase->idom(n)) { | ||||||||||||
| 3466 | if (n->is_Call() && n->as_Call()->guaranteed_safepoint()) { | ||||||||||||
| 3467 | has_call = true; | ||||||||||||
| 3468 | _has_sfpt = 1; // Then no need for a safept! | ||||||||||||
| 3469 | break; | ||||||||||||
| 3470 | } else if (n->Opcode() == Op_SafePoint) { | ||||||||||||
| 3471 | if (_phase->get_loop(n) == this) { | ||||||||||||
| 3472 | has_local_ncsfpt = true; | ||||||||||||
| 3473 | break; | ||||||||||||
| 3474 | } | ||||||||||||
| 3475 | if (nonlocal_ncsfpt == NULL__null) { | ||||||||||||
| 3476 | nonlocal_ncsfpt = n; // save the one closest to the tail | ||||||||||||
| 3477 | } | ||||||||||||
| 3478 | } else { | ||||||||||||
| 3479 | IdealLoopTree* nlpt = _phase->get_loop(n); | ||||||||||||
| 3480 | if (this != nlpt) { | ||||||||||||
| 3481 | // If at an inner loop tail, see if the inner loop has already | ||||||||||||
| 3482 | // recorded seeing a call on the dom-path (and stop.) If not, | ||||||||||||
| 3483 | // jump to the head of the inner loop. | ||||||||||||
| 3484 | assert(is_member(nlpt), "nested loop")do { if (!(is_member(nlpt))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3484, "assert(" "is_member(nlpt)" ") failed", "nested loop" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3485 | Node* tail = nlpt->_tail; | ||||||||||||
| 3486 | if (tail->in(0)->is_If()) tail = tail->in(0); | ||||||||||||
| 3487 | if (n == tail) { | ||||||||||||
| 3488 | // If inner loop has call on dom-path, so does outer loop | ||||||||||||
| 3489 | if (nlpt->_has_sfpt) { | ||||||||||||
| 3490 | has_call = true; | ||||||||||||
| 3491 | _has_sfpt = 1; | ||||||||||||
| 3492 | break; | ||||||||||||
| 3493 | } | ||||||||||||
| 3494 | // Skip to head of inner loop | ||||||||||||
| 3495 | assert(_phase->is_dominator(_head, nlpt->_head), "inner head dominated by outer head")do { if (!(_phase->is_dominator(_head, nlpt->_head))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3495, "assert(" "_phase->is_dominator(_head, nlpt->_head)" ") failed", "inner head dominated by outer head"); ::breakpoint (); } } while (0); | ||||||||||||
| 3496 | n = nlpt->_head; | ||||||||||||
| 3497 | } | ||||||||||||
| 3498 | } | ||||||||||||
| 3499 | } | ||||||||||||
| 3500 | } | ||||||||||||
| 3501 | // Record safept's that this loop needs preserved when an | ||||||||||||
| 3502 | // inner loop attempts to delete it's safepoints. | ||||||||||||
| 3503 | if (_child != NULL__null && !has_call && !has_local_ncsfpt) { | ||||||||||||
| 3504 | if (nonlocal_ncsfpt != NULL__null) { | ||||||||||||
| 3505 | if (_required_safept == NULL__null) _required_safept = new Node_List(); | ||||||||||||
| 3506 | _required_safept->push(nonlocal_ncsfpt); | ||||||||||||
| 3507 | } else { | ||||||||||||
| 3508 | // Failed to find a suitable safept on the dom-path. Now use | ||||||||||||
| 3509 | // an all paths walk from tail to head, looking for safepoints to preserve. | ||||||||||||
| 3510 | allpaths_check_safepts(visited, stack); | ||||||||||||
| 3511 | } | ||||||||||||
| 3512 | } | ||||||||||||
| 3513 | } | ||||||||||||
| 3514 | } | ||||||||||||
| 3515 | |||||||||||||
| 3516 | //---------------------------is_deleteable_safept---------------------------- | ||||||||||||
| 3517 | // Is safept not required by an outer loop? | ||||||||||||
| 3518 | bool PhaseIdealLoop::is_deleteable_safept(Node* sfpt) { | ||||||||||||
| 3519 | assert(sfpt->Opcode() == Op_SafePoint, "")do { if (!(sfpt->Opcode() == Op_SafePoint)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3519, "assert(" "sfpt->Opcode() == Op_SafePoint" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 3520 | IdealLoopTree* lp = get_loop(sfpt)->_parent; | ||||||||||||
| 3521 | while (lp != NULL__null) { | ||||||||||||
| 3522 | Node_List* sfpts = lp->_required_safept; | ||||||||||||
| 3523 | if (sfpts != NULL__null) { | ||||||||||||
| 3524 | for (uint i = 0; i < sfpts->size(); i++) { | ||||||||||||
| 3525 | if (sfpt == sfpts->at(i)) | ||||||||||||
| 3526 | return false; | ||||||||||||
| 3527 | } | ||||||||||||
| 3528 | } | ||||||||||||
| 3529 | lp = lp->_parent; | ||||||||||||
| 3530 | } | ||||||||||||
| 3531 | return true; | ||||||||||||
| 3532 | } | ||||||||||||
| 3533 | |||||||||||||
| 3534 | //---------------------------replace_parallel_iv------------------------------- | ||||||||||||
| 3535 | // Replace parallel induction variable (parallel to trip counter) | ||||||||||||
| 3536 | void PhaseIdealLoop::replace_parallel_iv(IdealLoopTree *loop) { | ||||||||||||
| 3537 | assert(loop->_head->is_CountedLoop(), "")do { if (!(loop->_head->is_CountedLoop())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3537, "assert(" "loop->_head->is_CountedLoop()" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 3538 | CountedLoopNode *cl = loop->_head->as_CountedLoop(); | ||||||||||||
| 3539 | if (!cl->is_valid_counted_loop(T_INT)) | ||||||||||||
| 3540 | return; // skip malformed counted loop | ||||||||||||
| 3541 | Node *incr = cl->incr(); | ||||||||||||
| 3542 | if (incr == NULL__null) | ||||||||||||
| 3543 | return; // Dead loop? | ||||||||||||
| 3544 | Node *init = cl->init_trip(); | ||||||||||||
| 3545 | Node *phi = cl->phi(); | ||||||||||||
| 3546 | int stride_con = cl->stride_con(); | ||||||||||||
| 3547 | |||||||||||||
| 3548 | // Visit all children, looking for Phis | ||||||||||||
| 3549 | for (DUIterator i = cl->outs(); cl->has_out(i); i++) { | ||||||||||||
| 3550 | Node *out = cl->out(i); | ||||||||||||
| 3551 | // Look for other phis (secondary IVs). Skip dead ones | ||||||||||||
| 3552 | if (!out->is_Phi() || out
| ||||||||||||
| 3553 | continue; | ||||||||||||
| 3554 | PhiNode* phi2 = out->as_Phi(); | ||||||||||||
| 3555 | Node *incr2 = phi2->in( LoopNode::LoopBackControl ); | ||||||||||||
| 3556 | // Look for induction variables of the form: X += constant | ||||||||||||
| 3557 | if (phi2->region() != loop->_head || | ||||||||||||
| 3558 | incr2->req() != 3 || | ||||||||||||
| 3559 | incr2->in(1)->uncast() != phi2 || | ||||||||||||
| 3560 | incr2 == incr || | ||||||||||||
| 3561 | incr2->Opcode() != Op_AddI || | ||||||||||||
| 3562 | !incr2->in(2)->is_Con()) | ||||||||||||
| 3563 | continue; | ||||||||||||
| 3564 | |||||||||||||
| 3565 | // Check for parallel induction variable (parallel to trip counter) | ||||||||||||
| 3566 | // via an affine function. In particular, count-down loops with | ||||||||||||
| 3567 | // count-up array indices are common. We only RCE references off | ||||||||||||
| 3568 | // the trip-counter, so we need to convert all these to trip-counter | ||||||||||||
| 3569 | // expressions. | ||||||||||||
| 3570 | Node *init2 = phi2->in( LoopNode::EntryControl ); | ||||||||||||
| 3571 | int stride_con2 = incr2->in(2)->get_int(); | ||||||||||||
| 3572 | |||||||||||||
| 3573 | // The ratio of the two strides cannot be represented as an int | ||||||||||||
| 3574 | // if stride_con2 is min_int and stride_con is -1. | ||||||||||||
| 3575 | if (stride_con2 == min_jint && stride_con == -1) { | ||||||||||||
| 3576 | continue; | ||||||||||||
| 3577 | } | ||||||||||||
| 3578 | |||||||||||||
| 3579 | // The general case here gets a little tricky. We want to find the | ||||||||||||
| 3580 | // GCD of all possible parallel IV's and make a new IV using this | ||||||||||||
| 3581 | // GCD for the loop. Then all possible IVs are simple multiples of | ||||||||||||
| 3582 | // the GCD. In practice, this will cover very few extra loops. | ||||||||||||
| 3583 | // Instead we require 'stride_con2' to be a multiple of 'stride_con', | ||||||||||||
| 3584 | // where +/-1 is the common case, but other integer multiples are | ||||||||||||
| 3585 | // also easy to handle. | ||||||||||||
| 3586 | int ratio_con = stride_con2/stride_con; | ||||||||||||
| |||||||||||||
| 3587 | |||||||||||||
| 3588 | if ((ratio_con * stride_con) == stride_con2) { // Check for exact | ||||||||||||
| 3589 | #ifndef PRODUCT | ||||||||||||
| 3590 | if (TraceLoopOpts) { | ||||||||||||
| 3591 | tty->print("Parallel IV: %d ", phi2->_idx); | ||||||||||||
| 3592 | loop->dump_head(); | ||||||||||||
| 3593 | } | ||||||||||||
| 3594 | #endif | ||||||||||||
| 3595 | // Convert to using the trip counter. The parallel induction | ||||||||||||
| 3596 | // variable differs from the trip counter by a loop-invariant | ||||||||||||
| 3597 | // amount, the difference between their respective initial values. | ||||||||||||
| 3598 | // It is scaled by the 'ratio_con'. | ||||||||||||
| 3599 | Node* ratio = _igvn.intcon(ratio_con); | ||||||||||||
| 3600 | set_ctrl(ratio, C->root()); | ||||||||||||
| 3601 | Node* ratio_init = new MulINode(init, ratio); | ||||||||||||
| 3602 | _igvn.register_new_node_with_optimizer(ratio_init, init); | ||||||||||||
| 3603 | set_early_ctrl(ratio_init, false); | ||||||||||||
| 3604 | Node* diff = new SubINode(init2, ratio_init); | ||||||||||||
| 3605 | _igvn.register_new_node_with_optimizer(diff, init2); | ||||||||||||
| 3606 | set_early_ctrl(diff, false); | ||||||||||||
| 3607 | Node* ratio_idx = new MulINode(phi, ratio); | ||||||||||||
| 3608 | _igvn.register_new_node_with_optimizer(ratio_idx, phi); | ||||||||||||
| 3609 | set_ctrl(ratio_idx, cl); | ||||||||||||
| 3610 | Node* add = new AddINode(ratio_idx, diff); | ||||||||||||
| 3611 | _igvn.register_new_node_with_optimizer(add); | ||||||||||||
| 3612 | set_ctrl(add, cl); | ||||||||||||
| 3613 | _igvn.replace_node( phi2, add ); | ||||||||||||
| 3614 | // Sometimes an induction variable is unused | ||||||||||||
| 3615 | if (add->outcnt() == 0) { | ||||||||||||
| 3616 | _igvn.remove_dead_node(add); | ||||||||||||
| 3617 | } | ||||||||||||
| 3618 | --i; // deleted this phi; rescan starting with next position | ||||||||||||
| 3619 | continue; | ||||||||||||
| 3620 | } | ||||||||||||
| 3621 | } | ||||||||||||
| 3622 | } | ||||||||||||
| 3623 | |||||||||||||
| 3624 | void IdealLoopTree::remove_safepoints(PhaseIdealLoop* phase, bool keep_one) { | ||||||||||||
| 3625 | Node* keep = NULL__null; | ||||||||||||
| 3626 | if (keep_one) { | ||||||||||||
| 3627 | // Look for a safepoint on the idom-path. | ||||||||||||
| 3628 | for (Node* i = tail(); i != _head; i = phase->idom(i)) { | ||||||||||||
| 3629 | if (i->Opcode() == Op_SafePoint && phase->get_loop(i) == this) { | ||||||||||||
| 3630 | keep = i; | ||||||||||||
| 3631 | break; // Found one | ||||||||||||
| 3632 | } | ||||||||||||
| 3633 | } | ||||||||||||
| 3634 | } | ||||||||||||
| 3635 | |||||||||||||
| 3636 | // Don't remove any safepoints if it is requested to keep a single safepoint and | ||||||||||||
| 3637 | // no safepoint was found on idom-path. It is not safe to remove any safepoint | ||||||||||||
| 3638 | // in this case since there's no safepoint dominating all paths in the loop body. | ||||||||||||
| 3639 | bool prune = !keep_one || keep != NULL__null; | ||||||||||||
| 3640 | |||||||||||||
| 3641 | // Delete other safepoints in this loop. | ||||||||||||
| 3642 | Node_List* sfpts = _safepts; | ||||||||||||
| 3643 | if (prune && sfpts != NULL__null) { | ||||||||||||
| 3644 | assert(keep == NULL || keep->Opcode() == Op_SafePoint, "not safepoint")do { if (!(keep == __null || keep->Opcode() == Op_SafePoint )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3644, "assert(" "keep == __null || keep->Opcode() == Op_SafePoint" ") failed", "not safepoint"); ::breakpoint(); } } while (0); | ||||||||||||
| 3645 | for (uint i = 0; i < sfpts->size(); i++) { | ||||||||||||
| 3646 | Node* n = sfpts->at(i); | ||||||||||||
| 3647 | assert(phase->get_loop(n) == this, "")do { if (!(phase->get_loop(n) == this)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3647, "assert(" "phase->get_loop(n) == this" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 3648 | if (n != keep && phase->is_deleteable_safept(n)) { | ||||||||||||
| 3649 | phase->lazy_replace(n, n->in(TypeFunc::Control)); | ||||||||||||
| 3650 | } | ||||||||||||
| 3651 | } | ||||||||||||
| 3652 | } | ||||||||||||
| 3653 | } | ||||||||||||
| 3654 | |||||||||||||
| 3655 | //------------------------------counted_loop----------------------------------- | ||||||||||||
| 3656 | // Convert to counted loops where possible | ||||||||||||
| 3657 | void IdealLoopTree::counted_loop( PhaseIdealLoop *phase ) { | ||||||||||||
| 3658 | |||||||||||||
| 3659 | // For grins, set the inner-loop flag here | ||||||||||||
| 3660 | if (!_child) { | ||||||||||||
| 3661 | if (_head->is_Loop()) _head->as_Loop()->set_inner_loop(); | ||||||||||||
| 3662 | } | ||||||||||||
| 3663 | |||||||||||||
| 3664 | IdealLoopTree* loop = this; | ||||||||||||
| 3665 | if (_head->is_CountedLoop() || | ||||||||||||
| 3666 | phase->is_counted_loop(_head, loop, T_INT)) { | ||||||||||||
| 3667 | |||||||||||||
| 3668 | if (LoopStripMiningIter == 0 || (LoopStripMiningIter > 1 && _child == NULL__null)) { | ||||||||||||
| 3669 | // Indicate we do not need a safepoint here | ||||||||||||
| 3670 | _has_sfpt = 1; | ||||||||||||
| 3671 | } | ||||||||||||
| 3672 | |||||||||||||
| 3673 | // Remove safepoints | ||||||||||||
| 3674 | bool keep_one_sfpt = !(_has_call || _has_sfpt); | ||||||||||||
| 3675 | remove_safepoints(phase, keep_one_sfpt); | ||||||||||||
| 3676 | |||||||||||||
| 3677 | // Look for induction variables | ||||||||||||
| 3678 | phase->replace_parallel_iv(this); | ||||||||||||
| 3679 | } else if (_head->is_LongCountedLoop() || | ||||||||||||
| 3680 | phase->is_counted_loop(_head, loop, T_LONG)) { | ||||||||||||
| 3681 | remove_safepoints(phase, true); | ||||||||||||
| 3682 | } else { | ||||||||||||
| 3683 | assert(!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop(), "transformation to counted loop should not fail")do { if (!(!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3683, "assert(" "!_head->is_Loop() || !_head->as_Loop()->is_loop_nest_inner_loop()" ") failed", "transformation to counted loop should not fail" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3684 | if (_parent != NULL__null && !_irreducible) { | ||||||||||||
| 3685 | // Not a counted loop. Keep one safepoint. | ||||||||||||
| 3686 | bool keep_one_sfpt = true; | ||||||||||||
| 3687 | remove_safepoints(phase, keep_one_sfpt); | ||||||||||||
| 3688 | } | ||||||||||||
| 3689 | } | ||||||||||||
| 3690 | |||||||||||||
| 3691 | // Recursively | ||||||||||||
| 3692 | assert(loop->_child != this || (loop->_head->as_Loop()->is_OuterStripMinedLoop() && _head->as_CountedLoop()->is_strip_mined()), "what kind of loop was added?")do { if (!(loop->_child != this || (loop->_head->as_Loop ()->is_OuterStripMinedLoop() && _head->as_CountedLoop ()->is_strip_mined()))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3692, "assert(" "loop->_child != this || (loop->_head->as_Loop()->is_OuterStripMinedLoop() && _head->as_CountedLoop()->is_strip_mined())" ") failed", "what kind of loop was added?"); ::breakpoint(); } } while (0); | ||||||||||||
| 3693 | assert(loop->_child != this || (loop->_child->_child == NULL && loop->_child->_next == NULL), "would miss some loops")do { if (!(loop->_child != this || (loop->_child->_child == __null && loop->_child->_next == __null))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3693, "assert(" "loop->_child != this || (loop->_child->_child == __null && loop->_child->_next == __null)" ") failed", "would miss some loops"); ::breakpoint(); } } while (0); | ||||||||||||
| 3694 | if (loop->_child && loop->_child != this) loop->_child->counted_loop(phase); | ||||||||||||
| 3695 | if (loop->_next) loop->_next ->counted_loop(phase); | ||||||||||||
| 3696 | } | ||||||||||||
| 3697 | |||||||||||||
| 3698 | |||||||||||||
| 3699 | // The Estimated Loop Clone Size: | ||||||||||||
| 3700 | // CloneFactor * (~112% * BodySize + BC) + CC + FanOutTerm, | ||||||||||||
| 3701 | // where BC and CC are totally ad-hoc/magic "body" and "clone" constants, | ||||||||||||
| 3702 | // respectively, used to ensure that the node usage estimates made are on the | ||||||||||||
| 3703 | // safe side, for the most part. The FanOutTerm is an attempt to estimate the | ||||||||||||
| 3704 | // possible additional/excessive nodes generated due to data and control flow | ||||||||||||
| 3705 | // merging, for edges reaching outside the loop. | ||||||||||||
| 3706 | uint IdealLoopTree::est_loop_clone_sz(uint factor) const { | ||||||||||||
| 3707 | |||||||||||||
| 3708 | precond(0 < factor && factor < 16)do { if (!(0 < factor && factor < 16)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3708, "assert(" "0 < factor && factor < 16" ") failed" , "precond"); ::breakpoint(); } } while (0); | ||||||||||||
| 3709 | |||||||||||||
| 3710 | uint const bc = 13; | ||||||||||||
| 3711 | uint const cc = 17; | ||||||||||||
| 3712 | uint const sz = _body.size() + (_body.size() + 7) / 2; | ||||||||||||
| 3713 | uint estimate = factor * (sz + bc) + cc; | ||||||||||||
| 3714 | |||||||||||||
| 3715 | assert((estimate - cc) / factor == sz + bc, "overflow")do { if (!((estimate - cc) / factor == sz + bc)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3715, "assert(" "(estimate - cc) / factor == sz + bc" ") failed" , "overflow"); ::breakpoint(); } } while (0); | ||||||||||||
| 3716 | |||||||||||||
| 3717 | return estimate + est_loop_flow_merge_sz(); | ||||||||||||
| 3718 | } | ||||||||||||
| 3719 | |||||||||||||
| 3720 | // The Estimated Loop (full-) Unroll Size: | ||||||||||||
| 3721 | // UnrollFactor * (~106% * BodySize) + CC + FanOutTerm, | ||||||||||||
| 3722 | // where CC is a (totally) ad-hoc/magic "clone" constant, used to ensure that | ||||||||||||
| 3723 | // node usage estimates made are on the safe side, for the most part. This is | ||||||||||||
| 3724 | // a "light" version of the loop clone size calculation (above), based on the | ||||||||||||
| 3725 | // assumption that most of the loop-construct overhead will be unraveled when | ||||||||||||
| 3726 | // (fully) unrolled. Defined for unroll factors larger or equal to one (>=1), | ||||||||||||
| 3727 | // including an overflow check and returning UINT_MAX in case of an overflow. | ||||||||||||
| 3728 | uint IdealLoopTree::est_loop_unroll_sz(uint factor) const { | ||||||||||||
| 3729 | |||||||||||||
| 3730 | precond(factor > 0)do { if (!(factor > 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3730, "assert(" "factor > 0" ") failed", "precond"); ::breakpoint (); } } while (0); | ||||||||||||
| 3731 | |||||||||||||
| 3732 | // Take into account that after unroll conjoined heads and tails will fold. | ||||||||||||
| 3733 | uint const b0 = _body.size() - EMPTY_LOOP_SIZE; | ||||||||||||
| 3734 | uint const cc = 7; | ||||||||||||
| 3735 | uint const sz = b0 + (b0 + 15) / 16; | ||||||||||||
| 3736 | uint estimate = factor * sz + cc; | ||||||||||||
| 3737 | |||||||||||||
| 3738 | if ((estimate - cc) / factor != sz) { | ||||||||||||
| 3739 | return UINT_MAX(2147483647 *2U +1U); | ||||||||||||
| 3740 | } | ||||||||||||
| 3741 | |||||||||||||
| 3742 | return estimate + est_loop_flow_merge_sz(); | ||||||||||||
| 3743 | } | ||||||||||||
| 3744 | |||||||||||||
| 3745 | // Estimate the growth effect (in nodes) of merging control and data flow when | ||||||||||||
| 3746 | // cloning a loop body, based on the amount of control and data flow reaching | ||||||||||||
| 3747 | // outside of the (current) loop body. | ||||||||||||
| 3748 | uint IdealLoopTree::est_loop_flow_merge_sz() const { | ||||||||||||
| 3749 | |||||||||||||
| 3750 | uint ctrl_edge_out_cnt = 0; | ||||||||||||
| 3751 | uint data_edge_out_cnt = 0; | ||||||||||||
| 3752 | |||||||||||||
| 3753 | for (uint i = 0; i < _body.size(); i++) { | ||||||||||||
| 3754 | Node* node = _body.at(i); | ||||||||||||
| 3755 | uint outcnt = node->outcnt(); | ||||||||||||
| 3756 | |||||||||||||
| 3757 | for (uint k = 0; k < outcnt; k++) { | ||||||||||||
| 3758 | Node* out = node->raw_out(k); | ||||||||||||
| 3759 | if (out == NULL__null) continue; | ||||||||||||
| 3760 | if (out->is_CFG()) { | ||||||||||||
| 3761 | if (!is_member(_phase->get_loop(out))) { | ||||||||||||
| 3762 | ctrl_edge_out_cnt++; | ||||||||||||
| 3763 | } | ||||||||||||
| 3764 | } else if (_phase->has_ctrl(out)) { | ||||||||||||
| 3765 | Node* ctrl = _phase->get_ctrl(out); | ||||||||||||
| 3766 | assert(ctrl != NULL, "must be")do { if (!(ctrl != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3766, "assert(" "ctrl != __null" ") failed", "must be"); :: breakpoint(); } } while (0); | ||||||||||||
| 3767 | assert(ctrl->is_CFG(), "must be")do { if (!(ctrl->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3767, "assert(" "ctrl->is_CFG()" ") failed", "must be"); ::breakpoint(); } } while (0); | ||||||||||||
| 3768 | if (!is_member(_phase->get_loop(ctrl))) { | ||||||||||||
| 3769 | data_edge_out_cnt++; | ||||||||||||
| 3770 | } | ||||||||||||
| 3771 | } | ||||||||||||
| 3772 | } | ||||||||||||
| 3773 | } | ||||||||||||
| 3774 | // Use data and control count (x2.0) in estimate iff both are > 0. This is | ||||||||||||
| 3775 | // a rather pessimistic estimate for the most part, in particular for some | ||||||||||||
| 3776 | // complex loops, but still not enough to capture all loops. | ||||||||||||
| 3777 | if (ctrl_edge_out_cnt > 0 && data_edge_out_cnt > 0) { | ||||||||||||
| 3778 | return 2 * (ctrl_edge_out_cnt + data_edge_out_cnt); | ||||||||||||
| 3779 | } | ||||||||||||
| 3780 | return 0; | ||||||||||||
| 3781 | } | ||||||||||||
| 3782 | |||||||||||||
| 3783 | #ifndef PRODUCT | ||||||||||||
| 3784 | //------------------------------dump_head-------------------------------------- | ||||||||||||
| 3785 | // Dump 1 liner for loop header info | ||||||||||||
| 3786 | void IdealLoopTree::dump_head() const { | ||||||||||||
| 3787 | tty->sp(2 * _nest); | ||||||||||||
| 3788 | tty->print("Loop: N%d/N%d ", _head->_idx, _tail->_idx); | ||||||||||||
| 3789 | if (_irreducible) tty->print(" IRREDUCIBLE"); | ||||||||||||
| 3790 | Node* entry = _head->is_Loop() ? _head->as_Loop()->skip_strip_mined(-1)->in(LoopNode::EntryControl) : _head->in(LoopNode::EntryControl); | ||||||||||||
| 3791 | Node* predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | ||||||||||||
| 3792 | if (predicate != NULL__null ) { | ||||||||||||
| 3793 | tty->print(" limit_check"); | ||||||||||||
| 3794 | entry = PhaseIdealLoop::skip_loop_predicates(entry); | ||||||||||||
| 3795 | } | ||||||||||||
| 3796 | if (UseProfiledLoopPredicate) { | ||||||||||||
| 3797 | predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); | ||||||||||||
| 3798 | if (predicate != NULL__null) { | ||||||||||||
| 3799 | tty->print(" profile_predicated"); | ||||||||||||
| 3800 | entry = PhaseIdealLoop::skip_loop_predicates(entry); | ||||||||||||
| 3801 | } | ||||||||||||
| 3802 | } | ||||||||||||
| 3803 | if (UseLoopPredicate) { | ||||||||||||
| 3804 | predicate = PhaseIdealLoop::find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | ||||||||||||
| 3805 | if (predicate != NULL__null) { | ||||||||||||
| 3806 | tty->print(" predicated"); | ||||||||||||
| 3807 | } | ||||||||||||
| 3808 | } | ||||||||||||
| 3809 | if (_head->is_CountedLoop()) { | ||||||||||||
| 3810 | CountedLoopNode *cl = _head->as_CountedLoop(); | ||||||||||||
| 3811 | tty->print(" counted"); | ||||||||||||
| 3812 | |||||||||||||
| 3813 | Node* init_n = cl->init_trip(); | ||||||||||||
| 3814 | if (init_n != NULL__null && init_n->is_Con()) | ||||||||||||
| 3815 | tty->print(" [%d,", cl->init_trip()->get_int()); | ||||||||||||
| 3816 | else | ||||||||||||
| 3817 | tty->print(" [int,"); | ||||||||||||
| 3818 | Node* limit_n = cl->limit(); | ||||||||||||
| 3819 | if (limit_n != NULL__null && limit_n->is_Con()) | ||||||||||||
| 3820 | tty->print("%d),", cl->limit()->get_int()); | ||||||||||||
| 3821 | else | ||||||||||||
| 3822 | tty->print("int),"); | ||||||||||||
| 3823 | int stride_con = cl->stride_con(); | ||||||||||||
| 3824 | if (stride_con > 0) tty->print("+"); | ||||||||||||
| 3825 | tty->print("%d", stride_con); | ||||||||||||
| 3826 | |||||||||||||
| 3827 | tty->print(" (%0.f iters) ", cl->profile_trip_cnt()); | ||||||||||||
| 3828 | |||||||||||||
| 3829 | if (cl->is_pre_loop ()) tty->print(" pre" ); | ||||||||||||
| 3830 | if (cl->is_main_loop()) tty->print(" main"); | ||||||||||||
| 3831 | if (cl->is_post_loop()) tty->print(" post"); | ||||||||||||
| 3832 | if (cl->is_vectorized_loop()) tty->print(" vector"); | ||||||||||||
| 3833 | if (cl->range_checks_present()) tty->print(" rc "); | ||||||||||||
| 3834 | if (cl->is_multiversioned()) tty->print(" multi "); | ||||||||||||
| 3835 | } | ||||||||||||
| 3836 | if (_has_call) tty->print(" has_call"); | ||||||||||||
| 3837 | if (_has_sfpt) tty->print(" has_sfpt"); | ||||||||||||
| 3838 | if (_rce_candidate) tty->print(" rce"); | ||||||||||||
| 3839 | if (_safepts != NULL__null && _safepts->size() > 0) { | ||||||||||||
| 3840 | tty->print(" sfpts={"); _safepts->dump_simple(); tty->print(" }"); | ||||||||||||
| 3841 | } | ||||||||||||
| 3842 | if (_required_safept != NULL__null && _required_safept->size() > 0) { | ||||||||||||
| 3843 | tty->print(" req={"); _required_safept->dump_simple(); tty->print(" }"); | ||||||||||||
| 3844 | } | ||||||||||||
| 3845 | if (Verbose) { | ||||||||||||
| 3846 | tty->print(" body={"); _body.dump_simple(); tty->print(" }"); | ||||||||||||
| 3847 | } | ||||||||||||
| 3848 | if (_head->is_Loop() && _head->as_Loop()->is_strip_mined()) { | ||||||||||||
| 3849 | tty->print(" strip_mined"); | ||||||||||||
| 3850 | } | ||||||||||||
| 3851 | tty->cr(); | ||||||||||||
| 3852 | } | ||||||||||||
| 3853 | |||||||||||||
| 3854 | //------------------------------dump------------------------------------------- | ||||||||||||
| 3855 | // Dump loops by loop tree | ||||||||||||
| 3856 | void IdealLoopTree::dump() const { | ||||||||||||
| 3857 | dump_head(); | ||||||||||||
| 3858 | if (_child) _child->dump(); | ||||||||||||
| 3859 | if (_next) _next ->dump(); | ||||||||||||
| 3860 | } | ||||||||||||
| 3861 | |||||||||||||
| 3862 | #endif | ||||||||||||
| 3863 | |||||||||||||
| 3864 | static void log_loop_tree_helper(IdealLoopTree* root, IdealLoopTree* loop, CompileLog* log) { | ||||||||||||
| 3865 | if (loop == root) { | ||||||||||||
| 3866 | if (loop->_child != NULL__null) { | ||||||||||||
| 3867 | log->begin_head("loop_tree"); | ||||||||||||
| 3868 | log->end_head(); | ||||||||||||
| 3869 | log_loop_tree_helper(root, loop->_child, log); | ||||||||||||
| 3870 | log->tail("loop_tree"); | ||||||||||||
| 3871 | assert(loop->_next == NULL, "what?")do { if (!(loop->_next == __null)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3871, "assert(" "loop->_next == __null" ") failed", "what?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3872 | } | ||||||||||||
| 3873 | } else if (loop != NULL__null) { | ||||||||||||
| 3874 | Node* head = loop->_head; | ||||||||||||
| 3875 | log->begin_head("loop"); | ||||||||||||
| 3876 | log->print(" idx='%d' ", head->_idx); | ||||||||||||
| 3877 | if (loop->_irreducible) log->print("irreducible='1' "); | ||||||||||||
| 3878 | if (head->is_Loop()) { | ||||||||||||
| 3879 | if (head->as_Loop()->is_inner_loop()) log->print("inner_loop='1' "); | ||||||||||||
| 3880 | if (head->as_Loop()->is_partial_peel_loop()) log->print("partial_peel_loop='1' "); | ||||||||||||
| 3881 | } else if (head->is_CountedLoop()) { | ||||||||||||
| 3882 | CountedLoopNode* cl = head->as_CountedLoop(); | ||||||||||||
| 3883 | if (cl->is_pre_loop()) log->print("pre_loop='%d' ", cl->main_idx()); | ||||||||||||
| 3884 | if (cl->is_main_loop()) log->print("main_loop='%d' ", cl->_idx); | ||||||||||||
| 3885 | if (cl->is_post_loop()) log->print("post_loop='%d' ", cl->main_idx()); | ||||||||||||
| 3886 | } | ||||||||||||
| 3887 | log->end_head(); | ||||||||||||
| 3888 | log_loop_tree_helper(root, loop->_child, log); | ||||||||||||
| 3889 | log->tail("loop"); | ||||||||||||
| 3890 | log_loop_tree_helper(root, loop->_next, log); | ||||||||||||
| 3891 | } | ||||||||||||
| 3892 | } | ||||||||||||
| 3893 | |||||||||||||
| 3894 | void PhaseIdealLoop::log_loop_tree() { | ||||||||||||
| 3895 | if (C->log() != NULL__null) { | ||||||||||||
| 3896 | log_loop_tree_helper(_ltree_root, _ltree_root, C->log()); | ||||||||||||
| 3897 | } | ||||||||||||
| 3898 | } | ||||||||||||
| 3899 | |||||||||||||
| 3900 | //---------------------collect_potentially_useful_predicates----------------------- | ||||||||||||
| 3901 | // Helper function to collect potentially useful predicates to prevent them from | ||||||||||||
| 3902 | // being eliminated by PhaseIdealLoop::eliminate_useless_predicates | ||||||||||||
| 3903 | void PhaseIdealLoop::collect_potentially_useful_predicates(IdealLoopTree* loop, Unique_Node_List &useful_predicates) { | ||||||||||||
| 3904 | if (loop->_child) { // child | ||||||||||||
| 3905 | collect_potentially_useful_predicates(loop->_child, useful_predicates); | ||||||||||||
| 3906 | } | ||||||||||||
| 3907 | |||||||||||||
| 3908 | // self (only loops that we can apply loop predication may use their predicates) | ||||||||||||
| 3909 | if (loop->_head->is_Loop() && | ||||||||||||
| 3910 | !loop->_irreducible && | ||||||||||||
| 3911 | !loop->tail()->is_top()) { | ||||||||||||
| 3912 | LoopNode* lpn = loop->_head->as_Loop(); | ||||||||||||
| 3913 | Node* entry = lpn->in(LoopNode::EntryControl); | ||||||||||||
| 3914 | |||||||||||||
| 3915 | Node* predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_loop_limit_check); | ||||||||||||
| 3916 | if (predicate != NULL__null) { // right pattern that can be used by loop predication | ||||||||||||
| 3917 | assert(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1, "must be")do { if (!(entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3917, "assert(" "entry->in(0)->in(1)->in(1)->Opcode() == Op_Opaque1" ") failed", "must be"); ::breakpoint(); } } while (0); | ||||||||||||
| 3918 | useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one | ||||||||||||
| 3919 | entry = skip_loop_predicates(entry); | ||||||||||||
| 3920 | } | ||||||||||||
| 3921 | if (UseProfiledLoopPredicate) { | ||||||||||||
| 3922 | predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_profile_predicate); | ||||||||||||
| 3923 | if (predicate != NULL__null) { // right pattern that can be used by loop predication | ||||||||||||
| 3924 | useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one | ||||||||||||
| 3925 | get_skeleton_predicates(entry, useful_predicates, true); | ||||||||||||
| 3926 | entry = skip_loop_predicates(entry); | ||||||||||||
| 3927 | } | ||||||||||||
| 3928 | } | ||||||||||||
| 3929 | |||||||||||||
| 3930 | if (UseLoopPredicate) { | ||||||||||||
| 3931 | predicate = find_predicate_insertion_point(entry, Deoptimization::Reason_predicate); | ||||||||||||
| 3932 | if (predicate != NULL__null) { // right pattern that can be used by loop predication | ||||||||||||
| 3933 | useful_predicates.push(entry->in(0)->in(1)->in(1)); // good one | ||||||||||||
| 3934 | get_skeleton_predicates(entry, useful_predicates, true); | ||||||||||||
| 3935 | } | ||||||||||||
| 3936 | } | ||||||||||||
| 3937 | } | ||||||||||||
| 3938 | |||||||||||||
| 3939 | if (loop->_next) { // sibling | ||||||||||||
| 3940 | collect_potentially_useful_predicates(loop->_next, useful_predicates); | ||||||||||||
| 3941 | } | ||||||||||||
| 3942 | } | ||||||||||||
| 3943 | |||||||||||||
| 3944 | //------------------------eliminate_useless_predicates----------------------------- | ||||||||||||
| 3945 | // Eliminate all inserted predicates if they could not be used by loop predication. | ||||||||||||
| 3946 | // Note: it will also eliminates loop limits check predicate since it also uses | ||||||||||||
| 3947 | // Opaque1 node (see Parse::add_predicate()). | ||||||||||||
| 3948 | void PhaseIdealLoop::eliminate_useless_predicates() { | ||||||||||||
| 3949 | if (C->predicate_count() == 0 && C->skeleton_predicate_count() == 0) { | ||||||||||||
| 3950 | return; // no predicate left | ||||||||||||
| 3951 | } | ||||||||||||
| 3952 | |||||||||||||
| 3953 | Unique_Node_List useful_predicates; // to store useful predicates | ||||||||||||
| 3954 | if (C->has_loops()) { | ||||||||||||
| 3955 | collect_potentially_useful_predicates(_ltree_root->_child, useful_predicates); | ||||||||||||
| 3956 | } | ||||||||||||
| 3957 | |||||||||||||
| 3958 | for (int i = C->predicate_count(); i > 0; i--) { | ||||||||||||
| 3959 | Node* n = C->predicate_opaque1_node(i - 1); | ||||||||||||
| 3960 | assert(n->Opcode() == Op_Opaque1, "must be")do { if (!(n->Opcode() == Op_Opaque1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3960, "assert(" "n->Opcode() == Op_Opaque1" ") failed", "must be" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3961 | if (!useful_predicates.member(n)) { // not in the useful list | ||||||||||||
| 3962 | _igvn.replace_node(n, n->in(1)); | ||||||||||||
| 3963 | } | ||||||||||||
| 3964 | } | ||||||||||||
| 3965 | |||||||||||||
| 3966 | for (int i = C->skeleton_predicate_count(); i > 0; i--) { | ||||||||||||
| 3967 | Node* n = C->skeleton_predicate_opaque4_node(i - 1); | ||||||||||||
| 3968 | assert(n->Opcode() == Op_Opaque4, "must be")do { if (!(n->Opcode() == Op_Opaque4)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3968, "assert(" "n->Opcode() == Op_Opaque4" ") failed", "must be" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3969 | if (!useful_predicates.member(n)) { // not in the useful list | ||||||||||||
| 3970 | _igvn.replace_node(n, n->in(2)); | ||||||||||||
| 3971 | } | ||||||||||||
| 3972 | } | ||||||||||||
| 3973 | } | ||||||||||||
| 3974 | |||||||||||||
| 3975 | //------------------------process_expensive_nodes----------------------------- | ||||||||||||
| 3976 | // Expensive nodes have their control input set to prevent the GVN | ||||||||||||
| 3977 | // from commoning them and as a result forcing the resulting node to | ||||||||||||
| 3978 | // be in a more frequent path. Use CFG information here, to change the | ||||||||||||
| 3979 | // control inputs so that some expensive nodes can be commoned while | ||||||||||||
| 3980 | // not executed more frequently. | ||||||||||||
| 3981 | bool PhaseIdealLoop::process_expensive_nodes() { | ||||||||||||
| 3982 | assert(OptimizeExpensiveOps, "optimization off?")do { if (!(OptimizeExpensiveOps)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 3982, "assert(" "OptimizeExpensiveOps" ") failed", "optimization off?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 3983 | |||||||||||||
| 3984 | // Sort nodes to bring similar nodes together | ||||||||||||
| 3985 | C->sort_expensive_nodes(); | ||||||||||||
| 3986 | |||||||||||||
| 3987 | bool progress = false; | ||||||||||||
| 3988 | |||||||||||||
| 3989 | for (int i = 0; i < C->expensive_count(); ) { | ||||||||||||
| 3990 | Node* n = C->expensive_node(i); | ||||||||||||
| 3991 | int start = i; | ||||||||||||
| 3992 | // Find nodes similar to n | ||||||||||||
| 3993 | i++; | ||||||||||||
| 3994 | for (; i < C->expensive_count() && Compile::cmp_expensive_nodes(n, C->expensive_node(i)) == 0; i++); | ||||||||||||
| 3995 | int end = i; | ||||||||||||
| 3996 | // And compare them two by two | ||||||||||||
| 3997 | for (int j = start; j < end; j++) { | ||||||||||||
| 3998 | Node* n1 = C->expensive_node(j); | ||||||||||||
| 3999 | if (is_node_unreachable(n1)) { | ||||||||||||
| 4000 | continue; | ||||||||||||
| 4001 | } | ||||||||||||
| 4002 | for (int k = j+1; k < end; k++) { | ||||||||||||
| 4003 | Node* n2 = C->expensive_node(k); | ||||||||||||
| 4004 | if (is_node_unreachable(n2)) { | ||||||||||||
| 4005 | continue; | ||||||||||||
| 4006 | } | ||||||||||||
| 4007 | |||||||||||||
| 4008 | assert(n1 != n2, "should be pair of nodes")do { if (!(n1 != n2)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4008, "assert(" "n1 != n2" ") failed", "should be pair of nodes" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4009 | |||||||||||||
| 4010 | Node* c1 = n1->in(0); | ||||||||||||
| 4011 | Node* c2 = n2->in(0); | ||||||||||||
| 4012 | |||||||||||||
| 4013 | Node* parent_c1 = c1; | ||||||||||||
| 4014 | Node* parent_c2 = c2; | ||||||||||||
| 4015 | |||||||||||||
| 4016 | // The call to get_early_ctrl_for_expensive() moves the | ||||||||||||
| 4017 | // expensive nodes up but stops at loops that are in a if | ||||||||||||
| 4018 | // branch. See whether we can exit the loop and move above the | ||||||||||||
| 4019 | // If. | ||||||||||||
| 4020 | if (c1->is_Loop()) { | ||||||||||||
| 4021 | parent_c1 = c1->in(1); | ||||||||||||
| 4022 | } | ||||||||||||
| 4023 | if (c2->is_Loop()) { | ||||||||||||
| 4024 | parent_c2 = c2->in(1); | ||||||||||||
| 4025 | } | ||||||||||||
| 4026 | |||||||||||||
| 4027 | if (parent_c1 == parent_c2) { | ||||||||||||
| 4028 | _igvn._worklist.push(n1); | ||||||||||||
| 4029 | _igvn._worklist.push(n2); | ||||||||||||
| 4030 | continue; | ||||||||||||
| 4031 | } | ||||||||||||
| 4032 | |||||||||||||
| 4033 | // Look for identical expensive node up the dominator chain. | ||||||||||||
| 4034 | if (is_dominator(c1, c2)) { | ||||||||||||
| 4035 | c2 = c1; | ||||||||||||
| 4036 | } else if (is_dominator(c2, c1)) { | ||||||||||||
| 4037 | c1 = c2; | ||||||||||||
| 4038 | } else if (parent_c1->is_Proj() && parent_c1->in(0)->is_If() && | ||||||||||||
| 4039 | parent_c2->is_Proj() && parent_c1->in(0) == parent_c2->in(0)) { | ||||||||||||
| 4040 | // Both branches have the same expensive node so move it up | ||||||||||||
| 4041 | // before the if. | ||||||||||||
| 4042 | c1 = c2 = idom(parent_c1->in(0)); | ||||||||||||
| 4043 | } | ||||||||||||
| 4044 | // Do the actual moves | ||||||||||||
| 4045 | if (n1->in(0) != c1) { | ||||||||||||
| 4046 | _igvn.hash_delete(n1); | ||||||||||||
| 4047 | n1->set_req(0, c1); | ||||||||||||
| 4048 | _igvn.hash_insert(n1); | ||||||||||||
| 4049 | _igvn._worklist.push(n1); | ||||||||||||
| 4050 | progress = true; | ||||||||||||
| 4051 | } | ||||||||||||
| 4052 | if (n2->in(0) != c2) { | ||||||||||||
| 4053 | _igvn.hash_delete(n2); | ||||||||||||
| 4054 | n2->set_req(0, c2); | ||||||||||||
| 4055 | _igvn.hash_insert(n2); | ||||||||||||
| 4056 | _igvn._worklist.push(n2); | ||||||||||||
| 4057 | progress = true; | ||||||||||||
| 4058 | } | ||||||||||||
| 4059 | } | ||||||||||||
| 4060 | } | ||||||||||||
| 4061 | } | ||||||||||||
| 4062 | |||||||||||||
| 4063 | return progress; | ||||||||||||
| 4064 | } | ||||||||||||
| 4065 | |||||||||||||
| 4066 | #ifdef ASSERT1 | ||||||||||||
| 4067 | bool PhaseIdealLoop::only_has_infinite_loops() { | ||||||||||||
| 4068 | for (IdealLoopTree* l = _ltree_root->_child; l != NULL__null; l = l->_next) { | ||||||||||||
| 4069 | uint i = 1; | ||||||||||||
| 4070 | for (; i < C->root()->req(); i++) { | ||||||||||||
| 4071 | Node* in = C->root()->in(i); | ||||||||||||
| 4072 | if (in != NULL__null && | ||||||||||||
| 4073 | in->Opcode() == Op_Halt && | ||||||||||||
| 4074 | in->in(0)->is_Proj() && | ||||||||||||
| 4075 | in->in(0)->in(0)->Opcode() == Op_NeverBranch && | ||||||||||||
| 4076 | in->in(0)->in(0)->in(0) == l->_head) { | ||||||||||||
| 4077 | break; | ||||||||||||
| 4078 | } | ||||||||||||
| 4079 | } | ||||||||||||
| 4080 | if (i == C->root()->req()) { | ||||||||||||
| 4081 | return false; | ||||||||||||
| 4082 | } | ||||||||||||
| 4083 | } | ||||||||||||
| 4084 | |||||||||||||
| 4085 | return true; | ||||||||||||
| 4086 | } | ||||||||||||
| 4087 | #endif | ||||||||||||
| 4088 | |||||||||||||
| 4089 | |||||||||||||
| 4090 | //============================================================================= | ||||||||||||
| 4091 | //----------------------------build_and_optimize------------------------------- | ||||||||||||
| 4092 | // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to | ||||||||||||
| 4093 | // its corresponding LoopNode. If 'optimize' is true, do some loop cleanups. | ||||||||||||
| 4094 | void PhaseIdealLoop::build_and_optimize(LoopOptsMode mode) { | ||||||||||||
| 4095 | assert(!C->post_loop_opts_phase(), "no loop opts allowed")do { if (!(!C->post_loop_opts_phase())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4095, "assert(" "!C->post_loop_opts_phase()" ") failed", "no loop opts allowed"); ::breakpoint(); } } while (0); | ||||||||||||
| |||||||||||||
| 4096 | |||||||||||||
| 4097 | bool do_split_ifs = (mode == LoopOptsDefault); | ||||||||||||
| 4098 | bool skip_loop_opts = (mode == LoopOptsNone); | ||||||||||||
| 4099 | |||||||||||||
| 4100 | int old_progress = C->major_progress(); | ||||||||||||
| 4101 | uint orig_worklist_size = _igvn._worklist.size(); | ||||||||||||
| 4102 | |||||||||||||
| 4103 | // Reset major-progress flag for the driver's heuristics | ||||||||||||
| 4104 | C->clear_major_progress(); | ||||||||||||
| 4105 | |||||||||||||
| 4106 | #ifndef PRODUCT | ||||||||||||
| 4107 | // Capture for later assert | ||||||||||||
| 4108 | uint unique = C->unique(); | ||||||||||||
| 4109 | _loop_invokes++; | ||||||||||||
| 4110 | _loop_work += unique; | ||||||||||||
| 4111 | #endif | ||||||||||||
| 4112 | |||||||||||||
| 4113 | // True if the method has at least 1 irreducible loop | ||||||||||||
| 4114 | _has_irreducible_loops = false; | ||||||||||||
| 4115 | |||||||||||||
| 4116 | _created_loop_node = false; | ||||||||||||
| 4117 | |||||||||||||
| 4118 | VectorSet visited; | ||||||||||||
| 4119 | // Pre-grow the mapping from Nodes to IdealLoopTrees. | ||||||||||||
| 4120 | _nodes.map(C->unique(), NULL__null); | ||||||||||||
| 4121 | memset(_nodes.adr(), 0, wordSize * C->unique()); | ||||||||||||
| 4122 | |||||||||||||
| 4123 | // Pre-build the top-level outermost loop tree entry | ||||||||||||
| 4124 | _ltree_root = new IdealLoopTree( this, C->root(), C->root() ); | ||||||||||||
| 4125 | // Do not need a safepoint at the top level | ||||||||||||
| 4126 | _ltree_root->_has_sfpt = 1; | ||||||||||||
| 4127 | |||||||||||||
| 4128 | // Initialize Dominators. | ||||||||||||
| 4129 | // Checked in clone_loop_predicate() during beautify_loops(). | ||||||||||||
| 4130 | _idom_size = 0; | ||||||||||||
| 4131 | _idom = NULL__null; | ||||||||||||
| 4132 | _dom_depth = NULL__null; | ||||||||||||
| 4133 | _dom_stk = NULL__null; | ||||||||||||
| 4134 | |||||||||||||
| 4135 | // Empty pre-order array | ||||||||||||
| 4136 | allocate_preorders(); | ||||||||||||
| 4137 | |||||||||||||
| 4138 | // Build a loop tree on the fly. Build a mapping from CFG nodes to | ||||||||||||
| 4139 | // IdealLoopTree entries. Data nodes are NOT walked. | ||||||||||||
| 4140 | build_loop_tree(); | ||||||||||||
| 4141 | // Check for bailout, and return | ||||||||||||
| 4142 | if (C->failing()) { | ||||||||||||
| 4143 | return; | ||||||||||||
| 4144 | } | ||||||||||||
| 4145 | |||||||||||||
| 4146 | // Verify that the has_loops() flag set at parse time is consistent | ||||||||||||
| 4147 | // with the just built loop tree. With infinite loops, it could be | ||||||||||||
| 4148 | // that one pass of loop opts only finds infinite loops, clears the | ||||||||||||
| 4149 | // has_loops() flag but adds NeverBranch nodes so the next loop opts | ||||||||||||
| 4150 | // verification pass finds a non empty loop tree. When the back edge | ||||||||||||
| 4151 | // is an exception edge, parsing doesn't set has_loops(). | ||||||||||||
| 4152 | assert(_ltree_root->_child == NULL || C->has_loops() || only_has_infinite_loops() || C->has_exception_backedge(), "parsing found no loops but there are some")do { if (!(_ltree_root->_child == __null || C->has_loops () || only_has_infinite_loops() || C->has_exception_backedge ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4152, "assert(" "_ltree_root->_child == __null || C->has_loops() || only_has_infinite_loops() || C->has_exception_backedge()" ") failed", "parsing found no loops but there are some"); :: breakpoint(); } } while (0); | ||||||||||||
| 4153 | // No loops after all | ||||||||||||
| 4154 | if( !_ltree_root->_child
| ||||||||||||
| 4155 | |||||||||||||
| 4156 | // There should always be an outer loop containing the Root and Return nodes. | ||||||||||||
| 4157 | // If not, we have a degenerate empty program. Bail out in this case. | ||||||||||||
| 4158 | if (!has_node(C->root())) { | ||||||||||||
| 4159 | if (!_verify_only) { | ||||||||||||
| 4160 | C->clear_major_progress(); | ||||||||||||
| 4161 | C->record_method_not_compilable("empty program detected during loop optimization"); | ||||||||||||
| 4162 | } | ||||||||||||
| 4163 | return; | ||||||||||||
| 4164 | } | ||||||||||||
| 4165 | |||||||||||||
| 4166 | BarrierSetC2* bs = BarrierSet::barrier_set()->barrier_set_c2(); | ||||||||||||
| 4167 | // Nothing to do, so get out | ||||||||||||
| 4168 | bool stop_early = !C->has_loops() && !skip_loop_opts
| ||||||||||||
| 4169 | !bs->is_gc_specific_loop_opts_pass(mode); | ||||||||||||
| 4170 | bool do_expensive_nodes = C->should_optimize_expensive_nodes(_igvn); | ||||||||||||
| 4171 | bool strip_mined_loops_expanded = bs->strip_mined_loops_expanded(mode); | ||||||||||||
| 4172 | if (stop_early
| ||||||||||||
| 4173 | return; | ||||||||||||
| 4174 | } | ||||||||||||
| 4175 | |||||||||||||
| 4176 | // Set loop nesting depth | ||||||||||||
| 4177 | _ltree_root->set_nest( 0 ); | ||||||||||||
| 4178 | |||||||||||||
| 4179 | // Split shared headers and insert loop landing pads. | ||||||||||||
| 4180 | // Do not bother doing this on the Root loop of course. | ||||||||||||
| 4181 | if( !_verify_me
| ||||||||||||
| 4182 | C->print_method(PHASE_BEFORE_BEAUTIFY_LOOPS, 3); | ||||||||||||
| 4183 | if( _ltree_root->_child->beautify_loops( this ) ) { | ||||||||||||
| 4184 | // Re-build loop tree! | ||||||||||||
| 4185 | _ltree_root->_child = NULL__null; | ||||||||||||
| 4186 | _nodes.clear(); | ||||||||||||
| 4187 | reallocate_preorders(); | ||||||||||||
| 4188 | build_loop_tree(); | ||||||||||||
| 4189 | // Check for bailout, and return | ||||||||||||
| 4190 | if (C->failing()) { | ||||||||||||
| 4191 | return; | ||||||||||||
| 4192 | } | ||||||||||||
| 4193 | // Reset loop nesting depth | ||||||||||||
| 4194 | _ltree_root->set_nest( 0 ); | ||||||||||||
| 4195 | |||||||||||||
| 4196 | C->print_method(PHASE_AFTER_BEAUTIFY_LOOPS, 3); | ||||||||||||
| 4197 | } | ||||||||||||
| 4198 | } | ||||||||||||
| 4199 | |||||||||||||
| 4200 | // Build Dominators for elision of NULL checks & loop finding. | ||||||||||||
| 4201 | // Since nodes do not have a slot for immediate dominator, make | ||||||||||||
| 4202 | // a persistent side array for that info indexed on node->_idx. | ||||||||||||
| 4203 | _idom_size = C->unique(); | ||||||||||||
| 4204 | _idom = NEW_RESOURCE_ARRAY( Node*, _idom_size )(Node**) resource_allocate_bytes((_idom_size) * sizeof(Node*) ); | ||||||||||||
| 4205 | _dom_depth = NEW_RESOURCE_ARRAY( uint, _idom_size )(uint*) resource_allocate_bytes((_idom_size) * sizeof(uint)); | ||||||||||||
| 4206 | _dom_stk = NULL__null; // Allocated on demand in recompute_dom_depth | ||||||||||||
| 4207 | memset( _dom_depth, 0, _idom_size * sizeof(uint) ); | ||||||||||||
| 4208 | |||||||||||||
| 4209 | Dominators(); | ||||||||||||
| 4210 | |||||||||||||
| 4211 | if (!_verify_only) { | ||||||||||||
| 4212 | // As a side effect, Dominators removed any unreachable CFG paths | ||||||||||||
| 4213 | // into RegionNodes. It doesn't do this test against Root, so | ||||||||||||
| 4214 | // we do it here. | ||||||||||||
| 4215 | for( uint i = 1; i < C->root()->req(); i++ ) { | ||||||||||||
| 4216 | if( !_nodes[C->root()->in(i)->_idx] ) { // Dead path into Root? | ||||||||||||
| 4217 | _igvn.delete_input_of(C->root(), i); | ||||||||||||
| 4218 | i--; // Rerun same iteration on compressed edges | ||||||||||||
| 4219 | } | ||||||||||||
| 4220 | } | ||||||||||||
| 4221 | |||||||||||||
| 4222 | // Given dominators, try to find inner loops with calls that must | ||||||||||||
| 4223 | // always be executed (call dominates loop tail). These loops do | ||||||||||||
| 4224 | // not need a separate safepoint. | ||||||||||||
| 4225 | Node_List cisstack; | ||||||||||||
| 4226 | _ltree_root->check_safepts(visited, cisstack); | ||||||||||||
| 4227 | } | ||||||||||||
| 4228 | |||||||||||||
| 4229 | // Walk the DATA nodes and place into loops. Find earliest control | ||||||||||||
| 4230 | // node. For CFG nodes, the _nodes array starts out and remains | ||||||||||||
| 4231 | // holding the associated IdealLoopTree pointer. For DATA nodes, the | ||||||||||||
| 4232 | // _nodes array holds the earliest legal controlling CFG node. | ||||||||||||
| 4233 | |||||||||||||
| 4234 | // Allocate stack with enough space to avoid frequent realloc | ||||||||||||
| 4235 | int stack_size = (C->live_nodes() >> 1) + 16; // (live_nodes>>1)+16 from Java2D stats | ||||||||||||
| 4236 | Node_Stack nstack(stack_size); | ||||||||||||
| 4237 | |||||||||||||
| 4238 | visited.clear(); | ||||||||||||
| 4239 | Node_List worklist; | ||||||||||||
| 4240 | // Don't need C->root() on worklist since | ||||||||||||
| 4241 | // it will be processed among C->top() inputs | ||||||||||||
| 4242 | worklist.push(C->top()); | ||||||||||||
| 4243 | visited.set(C->top()->_idx); // Set C->top() as visited now | ||||||||||||
| 4244 | build_loop_early( visited, worklist, nstack ); | ||||||||||||
| 4245 | |||||||||||||
| 4246 | // Given early legal placement, try finding counted loops. This placement | ||||||||||||
| 4247 | // is good enough to discover most loop invariants. | ||||||||||||
| 4248 | if (!_verify_me && !_verify_only && !strip_mined_loops_expanded) { | ||||||||||||
| 4249 | _ltree_root->counted_loop( this ); | ||||||||||||
| 4250 | } | ||||||||||||
| 4251 | |||||||||||||
| 4252 | // Find latest loop placement. Find ideal loop placement. | ||||||||||||
| 4253 | visited.clear(); | ||||||||||||
| 4254 | init_dom_lca_tags(); | ||||||||||||
| 4255 | // Need C->root() on worklist when processing outs | ||||||||||||
| 4256 | worklist.push(C->root()); | ||||||||||||
| 4257 | NOT_PRODUCT( C->verify_graph_edges(); )C->verify_graph_edges(); | ||||||||||||
| 4258 | worklist.push(C->top()); | ||||||||||||
| 4259 | build_loop_late( visited, worklist, nstack ); | ||||||||||||
| 4260 | |||||||||||||
| 4261 | if (_verify_only) { | ||||||||||||
| 4262 | C->restore_major_progress(old_progress); | ||||||||||||
| 4263 | assert(C->unique() == unique, "verification mode made Nodes? ? ?")do { if (!(C->unique() == unique)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4263, "assert(" "C->unique() == unique" ") failed", "verification mode made Nodes? ? ?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4264 | assert(_igvn._worklist.size() == orig_worklist_size, "shouldn't push anything")do { if (!(_igvn._worklist.size() == orig_worklist_size)) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4264, "assert(" "_igvn._worklist.size() == orig_worklist_size" ") failed", "shouldn't push anything"); ::breakpoint(); } } while (0); | ||||||||||||
| 4265 | return; | ||||||||||||
| 4266 | } | ||||||||||||
| 4267 | |||||||||||||
| 4268 | // clear out the dead code after build_loop_late | ||||||||||||
| 4269 | while (_deadlist.size()) { | ||||||||||||
| 4270 | _igvn.remove_globally_dead_node(_deadlist.pop()); | ||||||||||||
| 4271 | } | ||||||||||||
| 4272 | |||||||||||||
| 4273 | if (stop_early) { | ||||||||||||
| 4274 | assert(do_expensive_nodes, "why are we here?")do { if (!(do_expensive_nodes)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4274, "assert(" "do_expensive_nodes" ") failed", "why are we here?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4275 | if (process_expensive_nodes()) { | ||||||||||||
| 4276 | // If we made some progress when processing expensive nodes then | ||||||||||||
| 4277 | // the IGVN may modify the graph in a way that will allow us to | ||||||||||||
| 4278 | // make some more progress: we need to try processing expensive | ||||||||||||
| 4279 | // nodes again. | ||||||||||||
| 4280 | C->set_major_progress(); | ||||||||||||
| 4281 | } | ||||||||||||
| 4282 | return; | ||||||||||||
| 4283 | } | ||||||||||||
| 4284 | |||||||||||||
| 4285 | // Some parser-inserted loop predicates could never be used by loop | ||||||||||||
| 4286 | // predication or they were moved away from loop during some optimizations. | ||||||||||||
| 4287 | // For example, peeling. Eliminate them before next loop optimizations. | ||||||||||||
| 4288 | eliminate_useless_predicates(); | ||||||||||||
| 4289 | |||||||||||||
| 4290 | #ifndef PRODUCT | ||||||||||||
| 4291 | C->verify_graph_edges(); | ||||||||||||
| 4292 | if (_verify_me) { // Nested verify pass? | ||||||||||||
| 4293 | // Check to see if the verify mode is broken | ||||||||||||
| 4294 | assert(C->unique() == unique, "non-optimize mode made Nodes? ? ?")do { if (!(C->unique() == unique)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4294, "assert(" "C->unique() == unique" ") failed", "non-optimize mode made Nodes? ? ?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4295 | return; | ||||||||||||
| 4296 | } | ||||||||||||
| 4297 | if (VerifyLoopOptimizations) verify(); | ||||||||||||
| 4298 | if (TraceLoopOpts && C->has_loops()) { | ||||||||||||
| 4299 | _ltree_root->dump(); | ||||||||||||
| 4300 | } | ||||||||||||
| 4301 | #endif | ||||||||||||
| 4302 | |||||||||||||
| 4303 | if (skip_loop_opts) { | ||||||||||||
| 4304 | C->restore_major_progress(old_progress); | ||||||||||||
| 4305 | return; | ||||||||||||
| 4306 | } | ||||||||||||
| 4307 | |||||||||||||
| 4308 | if (mode == LoopOptsMaxUnroll) { | ||||||||||||
| 4309 | for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { | ||||||||||||
| 4310 | IdealLoopTree* lpt = iter.current(); | ||||||||||||
| 4311 | if (lpt->is_innermost() && lpt->_allow_optimizations && !lpt->_has_call && lpt->is_counted()) { | ||||||||||||
| 4312 | lpt->compute_trip_count(this); | ||||||||||||
| 4313 | if (!lpt->do_one_iteration_loop(this) && | ||||||||||||
| 4314 | !lpt->do_remove_empty_loop(this)) { | ||||||||||||
| 4315 | AutoNodeBudget node_budget(this); | ||||||||||||
| 4316 | if (lpt->_head->as_CountedLoop()->is_normal_loop() && | ||||||||||||
| 4317 | lpt->policy_maximally_unroll(this)) { | ||||||||||||
| 4318 | memset( worklist.adr(), 0, worklist.Size()*sizeof(Node*) ); | ||||||||||||
| 4319 | do_maximally_unroll(lpt, worklist); | ||||||||||||
| 4320 | } | ||||||||||||
| 4321 | } | ||||||||||||
| 4322 | } | ||||||||||||
| 4323 | } | ||||||||||||
| 4324 | |||||||||||||
| 4325 | C->restore_major_progress(old_progress); | ||||||||||||
| 4326 | return; | ||||||||||||
| 4327 | } | ||||||||||||
| 4328 | |||||||||||||
| 4329 | if (bs->optimize_loops(this, mode, visited, nstack, worklist)) { | ||||||||||||
| 4330 | return; | ||||||||||||
| 4331 | } | ||||||||||||
| 4332 | |||||||||||||
| 4333 | if (ReassociateInvariants && !C->major_progress()) { | ||||||||||||
| 4334 | // Reassociate invariants and prep for split_thru_phi | ||||||||||||
| 4335 | for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { | ||||||||||||
| 4336 | IdealLoopTree* lpt = iter.current(); | ||||||||||||
| 4337 | if (!lpt->is_loop()) { | ||||||||||||
| 4338 | continue; | ||||||||||||
| 4339 | } | ||||||||||||
| 4340 | Node* head = lpt->_head; | ||||||||||||
| 4341 | if (!head->is_BaseCountedLoop() || !lpt->is_innermost()) continue; | ||||||||||||
| 4342 | |||||||||||||
| 4343 | // check for vectorized loops, any reassociation of invariants was already done | ||||||||||||
| 4344 | if (head->is_CountedLoop()) { | ||||||||||||
| 4345 | if (head->as_CountedLoop()->is_unroll_only()) { | ||||||||||||
| 4346 | continue; | ||||||||||||
| 4347 | } else { | ||||||||||||
| 4348 | AutoNodeBudget node_budget(this); | ||||||||||||
| 4349 | lpt->reassociate_invariants(this); | ||||||||||||
| 4350 | } | ||||||||||||
| 4351 | } | ||||||||||||
| 4352 | // Because RCE opportunities can be masked by split_thru_phi, | ||||||||||||
| 4353 | // look for RCE candidates and inhibit split_thru_phi | ||||||||||||
| 4354 | // on just their loop-phi's for this pass of loop opts | ||||||||||||
| 4355 | if (SplitIfBlocks && do_split_ifs && | ||||||||||||
| 4356 | (lpt->policy_range_check(this, true, T_LONG) || | ||||||||||||
| 4357 | (head->is_CountedLoop() && lpt->policy_range_check(this, true, T_INT)))) { | ||||||||||||
| 4358 | lpt->_rce_candidate = 1; // = true | ||||||||||||
| 4359 | } | ||||||||||||
| 4360 | } | ||||||||||||
| 4361 | } | ||||||||||||
| 4362 | |||||||||||||
| 4363 | // Check for aggressive application of split-if and other transforms | ||||||||||||
| 4364 | // that require basic-block info (like cloning through Phi's) | ||||||||||||
| 4365 | if (!C->major_progress() && SplitIfBlocks && do_split_ifs) { | ||||||||||||
| 4366 | visited.clear(); | ||||||||||||
| 4367 | split_if_with_blocks( visited, nstack); | ||||||||||||
| 4368 | NOT_PRODUCT( if( VerifyLoopOptimizations ) verify(); )if( VerifyLoopOptimizations ) verify();; | ||||||||||||
| 4369 | } | ||||||||||||
| 4370 | |||||||||||||
| 4371 | if (!C->major_progress() && do_expensive_nodes && process_expensive_nodes()) { | ||||||||||||
| 4372 | C->set_major_progress(); | ||||||||||||
| 4373 | } | ||||||||||||
| 4374 | |||||||||||||
| 4375 | // Perform loop predication before iteration splitting | ||||||||||||
| 4376 | if (C->has_loops() && !C->major_progress() && (C->predicate_count() > 0)) { | ||||||||||||
| 4377 | _ltree_root->_child->loop_predication(this); | ||||||||||||
| 4378 | } | ||||||||||||
| 4379 | |||||||||||||
| 4380 | if (OptimizeFill && UseLoopPredicate && C->has_loops() && !C->major_progress()) { | ||||||||||||
| 4381 | if (do_intrinsify_fill()) { | ||||||||||||
| 4382 | C->set_major_progress(); | ||||||||||||
| 4383 | } | ||||||||||||
| 4384 | } | ||||||||||||
| 4385 | |||||||||||||
| 4386 | // Perform iteration-splitting on inner loops. Split iterations to avoid | ||||||||||||
| 4387 | // range checks or one-shot null checks. | ||||||||||||
| 4388 | |||||||||||||
| 4389 | // If split-if's didn't hack the graph too bad (no CFG changes) | ||||||||||||
| 4390 | // then do loop opts. | ||||||||||||
| 4391 | if (C->has_loops() && !C->major_progress()) { | ||||||||||||
| 4392 | memset( worklist.adr(), 0, worklist.Size()*sizeof(Node*) ); | ||||||||||||
| 4393 | _ltree_root->_child->iteration_split( this, worklist ); | ||||||||||||
| 4394 | // No verify after peeling! GCM has hoisted code out of the loop. | ||||||||||||
| 4395 | // After peeling, the hoisted code could sink inside the peeled area. | ||||||||||||
| 4396 | // The peeling code does not try to recompute the best location for | ||||||||||||
| 4397 | // all the code before the peeled area, so the verify pass will always | ||||||||||||
| 4398 | // complain about it. | ||||||||||||
| 4399 | } | ||||||||||||
| 4400 | |||||||||||||
| 4401 | // Check for bailout, and return | ||||||||||||
| 4402 | if (C->failing()) { | ||||||||||||
| 4403 | return; | ||||||||||||
| 4404 | } | ||||||||||||
| 4405 | |||||||||||||
| 4406 | // Do verify graph edges in any case | ||||||||||||
| 4407 | NOT_PRODUCT( C->verify_graph_edges(); )C->verify_graph_edges();; | ||||||||||||
| 4408 | |||||||||||||
| 4409 | if (!do_split_ifs) { | ||||||||||||
| 4410 | // We saw major progress in Split-If to get here. We forced a | ||||||||||||
| 4411 | // pass with unrolling and not split-if, however more split-if's | ||||||||||||
| 4412 | // might make progress. If the unrolling didn't make progress | ||||||||||||
| 4413 | // then the major-progress flag got cleared and we won't try | ||||||||||||
| 4414 | // another round of Split-If. In particular the ever-common | ||||||||||||
| 4415 | // instance-of/check-cast pattern requires at least 2 rounds of | ||||||||||||
| 4416 | // Split-If to clear out. | ||||||||||||
| 4417 | C->set_major_progress(); | ||||||||||||
| 4418 | } | ||||||||||||
| 4419 | |||||||||||||
| 4420 | // Repeat loop optimizations if new loops were seen | ||||||||||||
| 4421 | if (created_loop_node()) { | ||||||||||||
| 4422 | C->set_major_progress(); | ||||||||||||
| 4423 | } | ||||||||||||
| 4424 | |||||||||||||
| 4425 | // Keep loop predicates and perform optimizations with them | ||||||||||||
| 4426 | // until no more loop optimizations could be done. | ||||||||||||
| 4427 | // After that switch predicates off and do more loop optimizations. | ||||||||||||
| 4428 | if (!C->major_progress() && (C->predicate_count() > 0)) { | ||||||||||||
| 4429 | C->cleanup_loop_predicates(_igvn); | ||||||||||||
| 4430 | if (TraceLoopOpts) { | ||||||||||||
| 4431 | tty->print_cr("PredicatesOff"); | ||||||||||||
| 4432 | } | ||||||||||||
| 4433 | C->set_major_progress(); | ||||||||||||
| 4434 | } | ||||||||||||
| 4435 | |||||||||||||
| 4436 | // Convert scalar to superword operations at the end of all loop opts. | ||||||||||||
| 4437 | if (UseSuperWord && C->has_loops() && !C->major_progress()) { | ||||||||||||
| 4438 | // SuperWord transform | ||||||||||||
| 4439 | SuperWord sw(this); | ||||||||||||
| 4440 | for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { | ||||||||||||
| 4441 | IdealLoopTree* lpt = iter.current(); | ||||||||||||
| 4442 | if (lpt->is_counted()) { | ||||||||||||
| 4443 | CountedLoopNode *cl = lpt->_head->as_CountedLoop(); | ||||||||||||
| 4444 | |||||||||||||
| 4445 | if (PostLoopMultiversioning && cl->is_rce_post_loop() && !cl->is_vectorized_loop()) { | ||||||||||||
| 4446 | // Check that the rce'd post loop is encountered first, multiversion after all | ||||||||||||
| 4447 | // major main loop optimization are concluded | ||||||||||||
| 4448 | if (!C->major_progress()) { | ||||||||||||
| 4449 | IdealLoopTree *lpt_next = lpt->_next; | ||||||||||||
| 4450 | if (lpt_next && lpt_next->is_counted()) { | ||||||||||||
| 4451 | CountedLoopNode *cl = lpt_next->_head->as_CountedLoop(); | ||||||||||||
| 4452 | has_range_checks(lpt_next); | ||||||||||||
| 4453 | if (cl->is_post_loop() && cl->range_checks_present()) { | ||||||||||||
| 4454 | if (!cl->is_multiversioned()) { | ||||||||||||
| 4455 | if (multi_version_post_loops(lpt, lpt_next) == false) { | ||||||||||||
| 4456 | // Cause the rce loop to be optimized away if we fail | ||||||||||||
| 4457 | cl->mark_is_multiversioned(); | ||||||||||||
| 4458 | cl->set_slp_max_unroll(0); | ||||||||||||
| 4459 | poison_rce_post_loop(lpt); | ||||||||||||
| 4460 | } | ||||||||||||
| 4461 | } | ||||||||||||
| 4462 | } | ||||||||||||
| 4463 | } | ||||||||||||
| 4464 | sw.transform_loop(lpt, true); | ||||||||||||
| 4465 | } | ||||||||||||
| 4466 | } else if (cl->is_main_loop()) { | ||||||||||||
| 4467 | sw.transform_loop(lpt, true); | ||||||||||||
| 4468 | } | ||||||||||||
| 4469 | } | ||||||||||||
| 4470 | } | ||||||||||||
| 4471 | } | ||||||||||||
| 4472 | |||||||||||||
| 4473 | // disable assert until issue with split_flow_path is resolved (6742111) | ||||||||||||
| 4474 | // assert(!_has_irreducible_loops || C->parsed_irreducible_loop() || C->is_osr_compilation(), | ||||||||||||
| 4475 | // "shouldn't introduce irreducible loops"); | ||||||||||||
| 4476 | } | ||||||||||||
| 4477 | |||||||||||||
| 4478 | #ifndef PRODUCT | ||||||||||||
| 4479 | //------------------------------print_statistics------------------------------- | ||||||||||||
| 4480 | int PhaseIdealLoop::_loop_invokes=0;// Count of PhaseIdealLoop invokes | ||||||||||||
| 4481 | int PhaseIdealLoop::_loop_work=0; // Sum of PhaseIdealLoop x unique | ||||||||||||
| 4482 | volatile int PhaseIdealLoop::_long_loop_candidates=0; // Number of long loops seen | ||||||||||||
| 4483 | volatile int PhaseIdealLoop::_long_loop_nests=0; // Number of long loops successfully transformed to a nest | ||||||||||||
| 4484 | volatile int PhaseIdealLoop::_long_loop_counted_loops=0; // Number of long loops successfully transformed to a counted loop | ||||||||||||
| 4485 | void PhaseIdealLoop::print_statistics() { | ||||||||||||
| 4486 | tty->print_cr("PhaseIdealLoop=%d, sum _unique=%d, long loops=%d/%d/%d", _loop_invokes, _loop_work, _long_loop_counted_loops, _long_loop_nests, _long_loop_candidates); | ||||||||||||
| 4487 | } | ||||||||||||
| 4488 | |||||||||||||
| 4489 | //------------------------------verify----------------------------------------- | ||||||||||||
| 4490 | // Build a verify-only PhaseIdealLoop, and see that it agrees with me. | ||||||||||||
| 4491 | static int fail; // debug only, so its multi-thread dont care | ||||||||||||
| 4492 | void PhaseIdealLoop::verify() const { | ||||||||||||
| 4493 | int old_progress = C->major_progress(); | ||||||||||||
| 4494 | ResourceMark rm; | ||||||||||||
| 4495 | PhaseIdealLoop loop_verify(_igvn, this); | ||||||||||||
| 4496 | VectorSet visited; | ||||||||||||
| 4497 | |||||||||||||
| 4498 | fail = 0; | ||||||||||||
| 4499 | verify_compare(C->root(), &loop_verify, visited); | ||||||||||||
| 4500 | assert(fail == 0, "verify loops failed")do { if (!(fail == 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4500, "assert(" "fail == 0" ") failed", "verify loops failed" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4501 | // Verify loop structure is the same | ||||||||||||
| 4502 | _ltree_root->verify_tree(loop_verify._ltree_root, NULL__null); | ||||||||||||
| 4503 | // Reset major-progress. It was cleared by creating a verify version of | ||||||||||||
| 4504 | // PhaseIdealLoop. | ||||||||||||
| 4505 | C->restore_major_progress(old_progress); | ||||||||||||
| 4506 | } | ||||||||||||
| 4507 | |||||||||||||
| 4508 | //------------------------------verify_compare--------------------------------- | ||||||||||||
| 4509 | // Make sure me and the given PhaseIdealLoop agree on key data structures | ||||||||||||
| 4510 | void PhaseIdealLoop::verify_compare( Node *n, const PhaseIdealLoop *loop_verify, VectorSet &visited ) const { | ||||||||||||
| 4511 | if( !n ) return; | ||||||||||||
| 4512 | if( visited.test_set( n->_idx ) ) return; | ||||||||||||
| 4513 | if( !_nodes[n->_idx] ) { // Unreachable | ||||||||||||
| 4514 | assert( !loop_verify->_nodes[n->_idx], "both should be unreachable" )do { if (!(!loop_verify->_nodes[n->_idx])) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4514, "assert(" "!loop_verify->_nodes[n->_idx]" ") failed" , "both should be unreachable"); ::breakpoint(); } } while (0 ); | ||||||||||||
| 4515 | return; | ||||||||||||
| 4516 | } | ||||||||||||
| 4517 | |||||||||||||
| 4518 | uint i; | ||||||||||||
| 4519 | for( i = 0; i < n->req(); i++ ) | ||||||||||||
| 4520 | verify_compare( n->in(i), loop_verify, visited ); | ||||||||||||
| 4521 | |||||||||||||
| 4522 | // Check the '_nodes' block/loop structure | ||||||||||||
| 4523 | i = n->_idx; | ||||||||||||
| 4524 | if( has_ctrl(n) ) { // We have control; verify has loop or ctrl | ||||||||||||
| 4525 | if( _nodes[i] != loop_verify->_nodes[i] && | ||||||||||||
| 4526 | get_ctrl_no_update(n) != loop_verify->get_ctrl_no_update(n) ) { | ||||||||||||
| 4527 | tty->print("Mismatched control setting for: "); | ||||||||||||
| 4528 | n->dump(); | ||||||||||||
| 4529 | if( fail++ > 10 ) return; | ||||||||||||
| 4530 | Node *c = get_ctrl_no_update(n); | ||||||||||||
| 4531 | tty->print("We have it as: "); | ||||||||||||
| 4532 | if( c->in(0) ) c->dump(); | ||||||||||||
| 4533 | else tty->print_cr("N%d",c->_idx); | ||||||||||||
| 4534 | tty->print("Verify thinks: "); | ||||||||||||
| 4535 | if( loop_verify->has_ctrl(n) ) | ||||||||||||
| 4536 | loop_verify->get_ctrl_no_update(n)->dump(); | ||||||||||||
| 4537 | else | ||||||||||||
| 4538 | loop_verify->get_loop_idx(n)->dump(); | ||||||||||||
| 4539 | tty->cr(); | ||||||||||||
| 4540 | } | ||||||||||||
| 4541 | } else { // We have a loop | ||||||||||||
| 4542 | IdealLoopTree *us = get_loop_idx(n); | ||||||||||||
| 4543 | if( loop_verify->has_ctrl(n) ) { | ||||||||||||
| 4544 | tty->print("Mismatched loop setting for: "); | ||||||||||||
| 4545 | n->dump(); | ||||||||||||
| 4546 | if( fail++ > 10 ) return; | ||||||||||||
| 4547 | tty->print("We have it as: "); | ||||||||||||
| 4548 | us->dump(); | ||||||||||||
| 4549 | tty->print("Verify thinks: "); | ||||||||||||
| 4550 | loop_verify->get_ctrl_no_update(n)->dump(); | ||||||||||||
| 4551 | tty->cr(); | ||||||||||||
| 4552 | } else if (!C->major_progress()) { | ||||||||||||
| 4553 | // Loop selection can be messed up if we did a major progress | ||||||||||||
| 4554 | // operation, like split-if. Do not verify in that case. | ||||||||||||
| 4555 | IdealLoopTree *them = loop_verify->get_loop_idx(n); | ||||||||||||
| 4556 | if( us->_head != them->_head || us->_tail != them->_tail ) { | ||||||||||||
| 4557 | tty->print("Unequals loops for: "); | ||||||||||||
| 4558 | n->dump(); | ||||||||||||
| 4559 | if( fail++ > 10 ) return; | ||||||||||||
| 4560 | tty->print("We have it as: "); | ||||||||||||
| 4561 | us->dump(); | ||||||||||||
| 4562 | tty->print("Verify thinks: "); | ||||||||||||
| 4563 | them->dump(); | ||||||||||||
| 4564 | tty->cr(); | ||||||||||||
| 4565 | } | ||||||||||||
| 4566 | } | ||||||||||||
| 4567 | } | ||||||||||||
| 4568 | |||||||||||||
| 4569 | // Check for immediate dominators being equal | ||||||||||||
| 4570 | if( i >= _idom_size ) { | ||||||||||||
| 4571 | if( !n->is_CFG() ) return; | ||||||||||||
| 4572 | tty->print("CFG Node with no idom: "); | ||||||||||||
| 4573 | n->dump(); | ||||||||||||
| 4574 | return; | ||||||||||||
| 4575 | } | ||||||||||||
| 4576 | if( !n->is_CFG() ) return; | ||||||||||||
| 4577 | if( n == C->root() ) return; // No IDOM here | ||||||||||||
| 4578 | |||||||||||||
| 4579 | assert(n->_idx == i, "sanity")do { if (!(n->_idx == i)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4579, "assert(" "n->_idx == i" ") failed", "sanity"); :: breakpoint(); } } while (0); | ||||||||||||
| 4580 | Node *id = idom_no_update(n); | ||||||||||||
| 4581 | if( id != loop_verify->idom_no_update(n) ) { | ||||||||||||
| 4582 | tty->print("Unequals idoms for: "); | ||||||||||||
| 4583 | n->dump(); | ||||||||||||
| 4584 | if( fail++ > 10 ) return; | ||||||||||||
| 4585 | tty->print("We have it as: "); | ||||||||||||
| 4586 | id->dump(); | ||||||||||||
| 4587 | tty->print("Verify thinks: "); | ||||||||||||
| 4588 | loop_verify->idom_no_update(n)->dump(); | ||||||||||||
| 4589 | tty->cr(); | ||||||||||||
| 4590 | } | ||||||||||||
| 4591 | |||||||||||||
| 4592 | } | ||||||||||||
| 4593 | |||||||||||||
| 4594 | //------------------------------verify_tree------------------------------------ | ||||||||||||
| 4595 | // Verify that tree structures match. Because the CFG can change, siblings | ||||||||||||
| 4596 | // within the loop tree can be reordered. We attempt to deal with that by | ||||||||||||
| 4597 | // reordering the verify's loop tree if possible. | ||||||||||||
| 4598 | void IdealLoopTree::verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const { | ||||||||||||
| 4599 | assert( _parent == parent, "Badly formed loop tree" )do { if (!(_parent == parent)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4599, "assert(" "_parent == parent" ") failed", "Badly formed loop tree" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4600 | |||||||||||||
| 4601 | // Siblings not in same order? Attempt to re-order. | ||||||||||||
| 4602 | if( _head != loop->_head ) { | ||||||||||||
| 4603 | // Find _next pointer to update | ||||||||||||
| 4604 | IdealLoopTree **pp = &loop->_parent->_child; | ||||||||||||
| 4605 | while( *pp != loop ) | ||||||||||||
| 4606 | pp = &((*pp)->_next); | ||||||||||||
| 4607 | // Find proper sibling to be next | ||||||||||||
| 4608 | IdealLoopTree **nn = &loop->_next; | ||||||||||||
| 4609 | while( (*nn) && (*nn)->_head != _head ) | ||||||||||||
| 4610 | nn = &((*nn)->_next); | ||||||||||||
| 4611 | |||||||||||||
| 4612 | // Check for no match. | ||||||||||||
| 4613 | if( !(*nn) ) { | ||||||||||||
| 4614 | // Annoyingly, irreducible loops can pick different headers | ||||||||||||
| 4615 | // after a major_progress operation, so the rest of the loop | ||||||||||||
| 4616 | // tree cannot be matched. | ||||||||||||
| 4617 | if (_irreducible && Compile::current()->major_progress()) return; | ||||||||||||
| 4618 | assert( 0, "failed to match loop tree" )do { if (!(0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4618, "assert(" "0" ") failed", "failed to match loop tree" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4619 | } | ||||||||||||
| 4620 | |||||||||||||
| 4621 | // Move (*nn) to (*pp) | ||||||||||||
| 4622 | IdealLoopTree *hit = *nn; | ||||||||||||
| 4623 | *nn = hit->_next; | ||||||||||||
| 4624 | hit->_next = loop; | ||||||||||||
| 4625 | *pp = loop; | ||||||||||||
| 4626 | loop = hit; | ||||||||||||
| 4627 | // Now try again to verify | ||||||||||||
| 4628 | } | ||||||||||||
| 4629 | |||||||||||||
| 4630 | assert( _head == loop->_head , "mismatched loop head" )do { if (!(_head == loop->_head)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4630, "assert(" "_head == loop->_head" ") failed", "mismatched loop head" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4631 | Node *tail = _tail; // Inline a non-updating version of | ||||||||||||
| 4632 | while( !tail->in(0) ) // the 'tail()' call. | ||||||||||||
| 4633 | tail = tail->in(1); | ||||||||||||
| 4634 | assert( tail == loop->_tail, "mismatched loop tail" )do { if (!(tail == loop->_tail)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4634, "assert(" "tail == loop->_tail" ") failed", "mismatched loop tail" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4635 | |||||||||||||
| 4636 | // Counted loops that are guarded should be able to find their guards | ||||||||||||
| 4637 | if( _head->is_CountedLoop() && _head->as_CountedLoop()->is_main_loop() ) { | ||||||||||||
| 4638 | CountedLoopNode *cl = _head->as_CountedLoop(); | ||||||||||||
| 4639 | Node *init = cl->init_trip(); | ||||||||||||
| 4640 | Node *ctrl = cl->in(LoopNode::EntryControl); | ||||||||||||
| 4641 | assert( ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse, "" )do { if (!(ctrl->Opcode() == Op_IfTrue || ctrl->Opcode( ) == Op_IfFalse)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4641, "assert(" "ctrl->Opcode() == Op_IfTrue || ctrl->Opcode() == Op_IfFalse" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 4642 | Node *iff = ctrl->in(0); | ||||||||||||
| 4643 | assert( iff->Opcode() == Op_If, "" )do { if (!(iff->Opcode() == Op_If)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4643, "assert(" "iff->Opcode() == Op_If" ") failed", "") ; ::breakpoint(); } } while (0); | ||||||||||||
| 4644 | Node *bol = iff->in(1); | ||||||||||||
| 4645 | assert( bol->Opcode() == Op_Bool, "" )do { if (!(bol->Opcode() == Op_Bool)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4645, "assert(" "bol->Opcode() == Op_Bool" ") failed", "" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4646 | Node *cmp = bol->in(1); | ||||||||||||
| 4647 | assert( cmp->Opcode() == Op_CmpI, "" )do { if (!(cmp->Opcode() == Op_CmpI)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4647, "assert(" "cmp->Opcode() == Op_CmpI" ") failed", "" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4648 | Node *add = cmp->in(1); | ||||||||||||
| 4649 | Node *opaq; | ||||||||||||
| 4650 | if( add->Opcode() == Op_Opaque1 ) { | ||||||||||||
| 4651 | opaq = add; | ||||||||||||
| 4652 | } else { | ||||||||||||
| 4653 | assert( add->Opcode() == Op_AddI || add->Opcode() == Op_ConI , "" )do { if (!(add->Opcode() == Op_AddI || add->Opcode() == Op_ConI)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4653, "assert(" "add->Opcode() == Op_AddI || add->Opcode() == Op_ConI" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 4654 | assert( add == init, "" )do { if (!(add == init)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4654, "assert(" "add == init" ") failed", ""); ::breakpoint (); } } while (0); | ||||||||||||
| 4655 | opaq = cmp->in(2); | ||||||||||||
| 4656 | } | ||||||||||||
| 4657 | assert( opaq->Opcode() == Op_Opaque1, "" )do { if (!(opaq->Opcode() == Op_Opaque1)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4657, "assert(" "opaq->Opcode() == Op_Opaque1" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 4658 | |||||||||||||
| 4659 | } | ||||||||||||
| 4660 | |||||||||||||
| 4661 | if (_child != NULL__null) _child->verify_tree(loop->_child, this); | ||||||||||||
| 4662 | if (_next != NULL__null) _next ->verify_tree(loop->_next, parent); | ||||||||||||
| 4663 | // Innermost loops need to verify loop bodies, | ||||||||||||
| 4664 | // but only if no 'major_progress' | ||||||||||||
| 4665 | int fail = 0; | ||||||||||||
| 4666 | if (!Compile::current()->major_progress() && _child == NULL__null) { | ||||||||||||
| 4667 | for( uint i = 0; i < _body.size(); i++ ) { | ||||||||||||
| 4668 | Node *n = _body.at(i); | ||||||||||||
| 4669 | if (n->outcnt() == 0) continue; // Ignore dead | ||||||||||||
| 4670 | uint j; | ||||||||||||
| 4671 | for( j = 0; j < loop->_body.size(); j++ ) | ||||||||||||
| 4672 | if( loop->_body.at(j) == n ) | ||||||||||||
| 4673 | break; | ||||||||||||
| 4674 | if( j == loop->_body.size() ) { // Not found in loop body | ||||||||||||
| 4675 | // Last ditch effort to avoid assertion: Its possible that we | ||||||||||||
| 4676 | // have some users (so outcnt not zero) but are still dead. | ||||||||||||
| 4677 | // Try to find from root. | ||||||||||||
| 4678 | if (Compile::current()->root()->find(n->_idx)) { | ||||||||||||
| 4679 | fail++; | ||||||||||||
| 4680 | tty->print("We have that verify does not: "); | ||||||||||||
| 4681 | n->dump(); | ||||||||||||
| 4682 | } | ||||||||||||
| 4683 | } | ||||||||||||
| 4684 | } | ||||||||||||
| 4685 | for( uint i2 = 0; i2 < loop->_body.size(); i2++ ) { | ||||||||||||
| 4686 | Node *n = loop->_body.at(i2); | ||||||||||||
| 4687 | if (n->outcnt() == 0) continue; // Ignore dead | ||||||||||||
| 4688 | uint j; | ||||||||||||
| 4689 | for( j = 0; j < _body.size(); j++ ) | ||||||||||||
| 4690 | if( _body.at(j) == n ) | ||||||||||||
| 4691 | break; | ||||||||||||
| 4692 | if( j == _body.size() ) { // Not found in loop body | ||||||||||||
| 4693 | // Last ditch effort to avoid assertion: Its possible that we | ||||||||||||
| 4694 | // have some users (so outcnt not zero) but are still dead. | ||||||||||||
| 4695 | // Try to find from root. | ||||||||||||
| 4696 | if (Compile::current()->root()->find(n->_idx)) { | ||||||||||||
| 4697 | fail++; | ||||||||||||
| 4698 | tty->print("Verify has that we do not: "); | ||||||||||||
| 4699 | n->dump(); | ||||||||||||
| 4700 | } | ||||||||||||
| 4701 | } | ||||||||||||
| 4702 | } | ||||||||||||
| 4703 | assert( !fail, "loop body mismatch" )do { if (!(!fail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4703, "assert(" "!fail" ") failed", "loop body mismatch"); :: breakpoint(); } } while (0); | ||||||||||||
| 4704 | } | ||||||||||||
| 4705 | } | ||||||||||||
| 4706 | |||||||||||||
| 4707 | #endif | ||||||||||||
| 4708 | |||||||||||||
| 4709 | //------------------------------set_idom--------------------------------------- | ||||||||||||
| 4710 | void PhaseIdealLoop::set_idom(Node* d, Node* n, uint dom_depth) { | ||||||||||||
| 4711 | uint idx = d->_idx; | ||||||||||||
| 4712 | if (idx >= _idom_size) { | ||||||||||||
| 4713 | uint newsize = next_power_of_2(idx); | ||||||||||||
| 4714 | _idom = REALLOC_RESOURCE_ARRAY( Node*, _idom,_idom_size,newsize)(Node**) resource_reallocate_bytes((char*)(_idom), (_idom_size ) * sizeof(Node*), (newsize) * sizeof(Node*)); | ||||||||||||
| 4715 | _dom_depth = REALLOC_RESOURCE_ARRAY( uint, _dom_depth,_idom_size,newsize)(uint*) resource_reallocate_bytes((char*)(_dom_depth), (_idom_size ) * sizeof(uint), (newsize) * sizeof(uint)); | ||||||||||||
| 4716 | memset( _dom_depth + _idom_size, 0, (newsize - _idom_size) * sizeof(uint) ); | ||||||||||||
| 4717 | _idom_size = newsize; | ||||||||||||
| 4718 | } | ||||||||||||
| 4719 | _idom[idx] = n; | ||||||||||||
| 4720 | _dom_depth[idx] = dom_depth; | ||||||||||||
| 4721 | } | ||||||||||||
| 4722 | |||||||||||||
| 4723 | //------------------------------recompute_dom_depth--------------------------------------- | ||||||||||||
| 4724 | // The dominator tree is constructed with only parent pointers. | ||||||||||||
| 4725 | // This recomputes the depth in the tree by first tagging all | ||||||||||||
| 4726 | // nodes as "no depth yet" marker. The next pass then runs up | ||||||||||||
| 4727 | // the dom tree from each node marked "no depth yet", and computes | ||||||||||||
| 4728 | // the depth on the way back down. | ||||||||||||
| 4729 | void PhaseIdealLoop::recompute_dom_depth() { | ||||||||||||
| 4730 | uint no_depth_marker = C->unique(); | ||||||||||||
| 4731 | uint i; | ||||||||||||
| 4732 | // Initialize depth to "no depth yet" and realize all lazy updates | ||||||||||||
| 4733 | for (i = 0; i < _idom_size; i++) { | ||||||||||||
| 4734 | // Only indices with a _dom_depth has a Node* or NULL (otherwise uninitalized). | ||||||||||||
| 4735 | if (_dom_depth[i] > 0 && _idom[i] != NULL__null) { | ||||||||||||
| 4736 | _dom_depth[i] = no_depth_marker; | ||||||||||||
| 4737 | |||||||||||||
| 4738 | // heal _idom if it has a fwd mapping in _nodes | ||||||||||||
| 4739 | if (_idom[i]->in(0) == NULL__null) { | ||||||||||||
| 4740 | idom(i); | ||||||||||||
| 4741 | } | ||||||||||||
| 4742 | } | ||||||||||||
| 4743 | } | ||||||||||||
| 4744 | if (_dom_stk == NULL__null) { | ||||||||||||
| 4745 | uint init_size = C->live_nodes() / 100; // Guess that 1/100 is a reasonable initial size. | ||||||||||||
| 4746 | if (init_size < 10) init_size = 10; | ||||||||||||
| 4747 | _dom_stk = new GrowableArray<uint>(init_size); | ||||||||||||
| 4748 | } | ||||||||||||
| 4749 | // Compute new depth for each node. | ||||||||||||
| 4750 | for (i = 0; i < _idom_size; i++) { | ||||||||||||
| 4751 | uint j = i; | ||||||||||||
| 4752 | // Run up the dom tree to find a node with a depth | ||||||||||||
| 4753 | while (_dom_depth[j] == no_depth_marker) { | ||||||||||||
| 4754 | _dom_stk->push(j); | ||||||||||||
| 4755 | j = _idom[j]->_idx; | ||||||||||||
| 4756 | } | ||||||||||||
| 4757 | // Compute the depth on the way back down this tree branch | ||||||||||||
| 4758 | uint dd = _dom_depth[j] + 1; | ||||||||||||
| 4759 | while (_dom_stk->length() > 0) { | ||||||||||||
| 4760 | uint j = _dom_stk->pop(); | ||||||||||||
| 4761 | _dom_depth[j] = dd; | ||||||||||||
| 4762 | dd++; | ||||||||||||
| 4763 | } | ||||||||||||
| 4764 | } | ||||||||||||
| 4765 | } | ||||||||||||
| 4766 | |||||||||||||
| 4767 | //------------------------------sort------------------------------------------- | ||||||||||||
| 4768 | // Insert 'loop' into the existing loop tree. 'innermost' is a leaf of the | ||||||||||||
| 4769 | // loop tree, not the root. | ||||||||||||
| 4770 | IdealLoopTree *PhaseIdealLoop::sort( IdealLoopTree *loop, IdealLoopTree *innermost ) { | ||||||||||||
| 4771 | if( !innermost ) return loop; // New innermost loop | ||||||||||||
| 4772 | |||||||||||||
| 4773 | int loop_preorder = get_preorder(loop->_head); // Cache pre-order number | ||||||||||||
| 4774 | assert( loop_preorder, "not yet post-walked loop" )do { if (!(loop_preorder)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4774, "assert(" "loop_preorder" ") failed", "not yet post-walked loop" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4775 | IdealLoopTree **pp = &innermost; // Pointer to previous next-pointer | ||||||||||||
| 4776 | IdealLoopTree *l = *pp; // Do I go before or after 'l'? | ||||||||||||
| 4777 | |||||||||||||
| 4778 | // Insert at start of list | ||||||||||||
| 4779 | while( l ) { // Insertion sort based on pre-order | ||||||||||||
| 4780 | if( l == loop ) return innermost; // Already on list! | ||||||||||||
| 4781 | int l_preorder = get_preorder(l->_head); // Cache pre-order number | ||||||||||||
| 4782 | assert( l_preorder, "not yet post-walked l" )do { if (!(l_preorder)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4782, "assert(" "l_preorder" ") failed", "not yet post-walked l" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4783 | // Check header pre-order number to figure proper nesting | ||||||||||||
| 4784 | if( loop_preorder > l_preorder ) | ||||||||||||
| 4785 | break; // End of insertion | ||||||||||||
| 4786 | // If headers tie (e.g., shared headers) check tail pre-order numbers. | ||||||||||||
| 4787 | // Since I split shared headers, you'd think this could not happen. | ||||||||||||
| 4788 | // BUT: I must first do the preorder numbering before I can discover I | ||||||||||||
| 4789 | // have shared headers, so the split headers all get the same preorder | ||||||||||||
| 4790 | // number as the RegionNode they split from. | ||||||||||||
| 4791 | if( loop_preorder == l_preorder && | ||||||||||||
| 4792 | get_preorder(loop->_tail) < get_preorder(l->_tail) ) | ||||||||||||
| 4793 | break; // Also check for shared headers (same pre#) | ||||||||||||
| 4794 | pp = &l->_parent; // Chain up list | ||||||||||||
| 4795 | l = *pp; | ||||||||||||
| 4796 | } | ||||||||||||
| 4797 | // Link into list | ||||||||||||
| 4798 | // Point predecessor to me | ||||||||||||
| 4799 | *pp = loop; | ||||||||||||
| 4800 | // Point me to successor | ||||||||||||
| 4801 | IdealLoopTree *p = loop->_parent; | ||||||||||||
| 4802 | loop->_parent = l; // Point me to successor | ||||||||||||
| 4803 | if( p ) sort( p, innermost ); // Insert my parents into list as well | ||||||||||||
| 4804 | return innermost; | ||||||||||||
| 4805 | } | ||||||||||||
| 4806 | |||||||||||||
| 4807 | //------------------------------build_loop_tree-------------------------------- | ||||||||||||
| 4808 | // I use a modified Vick/Tarjan algorithm. I need pre- and a post- visit | ||||||||||||
| 4809 | // bits. The _nodes[] array is mapped by Node index and holds a NULL for | ||||||||||||
| 4810 | // not-yet-pre-walked, pre-order # for pre-but-not-post-walked and holds the | ||||||||||||
| 4811 | // tightest enclosing IdealLoopTree for post-walked. | ||||||||||||
| 4812 | // | ||||||||||||
| 4813 | // During my forward walk I do a short 1-layer lookahead to see if I can find | ||||||||||||
| 4814 | // a loop backedge with that doesn't have any work on the backedge. This | ||||||||||||
| 4815 | // helps me construct nested loops with shared headers better. | ||||||||||||
| 4816 | // | ||||||||||||
| 4817 | // Once I've done the forward recursion, I do the post-work. For each child | ||||||||||||
| 4818 | // I check to see if there is a backedge. Backedges define a loop! I | ||||||||||||
| 4819 | // insert an IdealLoopTree at the target of the backedge. | ||||||||||||
| 4820 | // | ||||||||||||
| 4821 | // During the post-work I also check to see if I have several children | ||||||||||||
| 4822 | // belonging to different loops. If so, then this Node is a decision point | ||||||||||||
| 4823 | // where control flow can choose to change loop nests. It is at this | ||||||||||||
| 4824 | // decision point where I can figure out how loops are nested. At this | ||||||||||||
| 4825 | // time I can properly order the different loop nests from my children. | ||||||||||||
| 4826 | // Note that there may not be any backedges at the decision point! | ||||||||||||
| 4827 | // | ||||||||||||
| 4828 | // Since the decision point can be far removed from the backedges, I can't | ||||||||||||
| 4829 | // order my loops at the time I discover them. Thus at the decision point | ||||||||||||
| 4830 | // I need to inspect loop header pre-order numbers to properly nest my | ||||||||||||
| 4831 | // loops. This means I need to sort my childrens' loops by pre-order. | ||||||||||||
| 4832 | // The sort is of size number-of-control-children, which generally limits | ||||||||||||
| 4833 | // it to size 2 (i.e., I just choose between my 2 target loops). | ||||||||||||
| 4834 | void PhaseIdealLoop::build_loop_tree() { | ||||||||||||
| 4835 | // Allocate stack of size C->live_nodes()/2 to avoid frequent realloc | ||||||||||||
| 4836 | GrowableArray <Node *> bltstack(C->live_nodes() >> 1); | ||||||||||||
| 4837 | Node *n = C->root(); | ||||||||||||
| 4838 | bltstack.push(n); | ||||||||||||
| 4839 | int pre_order = 1; | ||||||||||||
| 4840 | int stack_size; | ||||||||||||
| 4841 | |||||||||||||
| 4842 | while ( ( stack_size = bltstack.length() ) != 0 ) { | ||||||||||||
| 4843 | n = bltstack.top(); // Leave node on stack | ||||||||||||
| 4844 | if ( !is_visited(n) ) { | ||||||||||||
| 4845 | // ---- Pre-pass Work ---- | ||||||||||||
| 4846 | // Pre-walked but not post-walked nodes need a pre_order number. | ||||||||||||
| 4847 | |||||||||||||
| 4848 | set_preorder_visited( n, pre_order ); // set as visited | ||||||||||||
| 4849 | |||||||||||||
| 4850 | // ---- Scan over children ---- | ||||||||||||
| 4851 | // Scan first over control projections that lead to loop headers. | ||||||||||||
| 4852 | // This helps us find inner-to-outer loops with shared headers better. | ||||||||||||
| 4853 | |||||||||||||
| 4854 | // Scan children's children for loop headers. | ||||||||||||
| 4855 | for ( int i = n->outcnt() - 1; i >= 0; --i ) { | ||||||||||||
| 4856 | Node* m = n->raw_out(i); // Child | ||||||||||||
| 4857 | if( m->is_CFG() && !is_visited(m) ) { // Only for CFG children | ||||||||||||
| 4858 | // Scan over children's children to find loop | ||||||||||||
| 4859 | for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 4860 | Node* l = m->fast_out(j); | ||||||||||||
| 4861 | if( is_visited(l) && // Been visited? | ||||||||||||
| 4862 | !is_postvisited(l) && // But not post-visited | ||||||||||||
| 4863 | get_preorder(l) < pre_order ) { // And smaller pre-order | ||||||||||||
| 4864 | // Found! Scan the DFS down this path before doing other paths | ||||||||||||
| 4865 | bltstack.push(m); | ||||||||||||
| 4866 | break; | ||||||||||||
| 4867 | } | ||||||||||||
| 4868 | } | ||||||||||||
| 4869 | } | ||||||||||||
| 4870 | } | ||||||||||||
| 4871 | pre_order++; | ||||||||||||
| 4872 | } | ||||||||||||
| 4873 | else if ( !is_postvisited(n) ) { | ||||||||||||
| 4874 | // Note: build_loop_tree_impl() adds out edges on rare occasions, | ||||||||||||
| 4875 | // such as com.sun.rsasign.am::a. | ||||||||||||
| 4876 | // For non-recursive version, first, process current children. | ||||||||||||
| 4877 | // On next iteration, check if additional children were added. | ||||||||||||
| 4878 | for ( int k = n->outcnt() - 1; k >= 0; --k ) { | ||||||||||||
| 4879 | Node* u = n->raw_out(k); | ||||||||||||
| 4880 | if ( u->is_CFG() && !is_visited(u) ) { | ||||||||||||
| 4881 | bltstack.push(u); | ||||||||||||
| 4882 | } | ||||||||||||
| 4883 | } | ||||||||||||
| 4884 | if ( bltstack.length() == stack_size ) { | ||||||||||||
| 4885 | // There were no additional children, post visit node now | ||||||||||||
| 4886 | (void)bltstack.pop(); // Remove node from stack | ||||||||||||
| 4887 | pre_order = build_loop_tree_impl( n, pre_order ); | ||||||||||||
| 4888 | // Check for bailout | ||||||||||||
| 4889 | if (C->failing()) { | ||||||||||||
| 4890 | return; | ||||||||||||
| 4891 | } | ||||||||||||
| 4892 | // Check to grow _preorders[] array for the case when | ||||||||||||
| 4893 | // build_loop_tree_impl() adds new nodes. | ||||||||||||
| 4894 | check_grow_preorders(); | ||||||||||||
| 4895 | } | ||||||||||||
| 4896 | } | ||||||||||||
| 4897 | else { | ||||||||||||
| 4898 | (void)bltstack.pop(); // Remove post-visited node from stack | ||||||||||||
| 4899 | } | ||||||||||||
| 4900 | } | ||||||||||||
| 4901 | } | ||||||||||||
| 4902 | |||||||||||||
| 4903 | //------------------------------build_loop_tree_impl--------------------------- | ||||||||||||
| 4904 | int PhaseIdealLoop::build_loop_tree_impl( Node *n, int pre_order ) { | ||||||||||||
| 4905 | // ---- Post-pass Work ---- | ||||||||||||
| 4906 | // Pre-walked but not post-walked nodes need a pre_order number. | ||||||||||||
| 4907 | |||||||||||||
| 4908 | // Tightest enclosing loop for this Node | ||||||||||||
| 4909 | IdealLoopTree *innermost = NULL__null; | ||||||||||||
| 4910 | |||||||||||||
| 4911 | // For all children, see if any edge is a backedge. If so, make a loop | ||||||||||||
| 4912 | // for it. Then find the tightest enclosing loop for the self Node. | ||||||||||||
| 4913 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 4914 | Node* m = n->fast_out(i); // Child | ||||||||||||
| 4915 | if( n == m ) continue; // Ignore control self-cycles | ||||||||||||
| 4916 | if( !m->is_CFG() ) continue;// Ignore non-CFG edges | ||||||||||||
| 4917 | |||||||||||||
| 4918 | IdealLoopTree *l; // Child's loop | ||||||||||||
| 4919 | if( !is_postvisited(m) ) { // Child visited but not post-visited? | ||||||||||||
| 4920 | // Found a backedge | ||||||||||||
| 4921 | assert( get_preorder(m) < pre_order, "should be backedge" )do { if (!(get_preorder(m) < pre_order)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4921, "assert(" "get_preorder(m) < pre_order" ") failed" , "should be backedge"); ::breakpoint(); } } while (0); | ||||||||||||
| 4922 | // Check for the RootNode, which is already a LoopNode and is allowed | ||||||||||||
| 4923 | // to have multiple "backedges". | ||||||||||||
| 4924 | if( m == C->root()) { // Found the root? | ||||||||||||
| 4925 | l = _ltree_root; // Root is the outermost LoopNode | ||||||||||||
| 4926 | } else { // Else found a nested loop | ||||||||||||
| 4927 | // Insert a LoopNode to mark this loop. | ||||||||||||
| 4928 | l = new IdealLoopTree(this, m, n); | ||||||||||||
| 4929 | } // End of Else found a nested loop | ||||||||||||
| 4930 | if( !has_loop(m) ) // If 'm' does not already have a loop set | ||||||||||||
| 4931 | set_loop(m, l); // Set loop header to loop now | ||||||||||||
| 4932 | |||||||||||||
| 4933 | } else { // Else not a nested loop | ||||||||||||
| 4934 | if( !_nodes[m->_idx] ) continue; // Dead code has no loop | ||||||||||||
| 4935 | l = get_loop(m); // Get previously determined loop | ||||||||||||
| 4936 | // If successor is header of a loop (nest), move up-loop till it | ||||||||||||
| 4937 | // is a member of some outer enclosing loop. Since there are no | ||||||||||||
| 4938 | // shared headers (I've split them already) I only need to go up | ||||||||||||
| 4939 | // at most 1 level. | ||||||||||||
| 4940 | while( l && l->_head == m ) // Successor heads loop? | ||||||||||||
| 4941 | l = l->_parent; // Move up 1 for me | ||||||||||||
| 4942 | // If this loop is not properly parented, then this loop | ||||||||||||
| 4943 | // has no exit path out, i.e. its an infinite loop. | ||||||||||||
| 4944 | if( !l ) { | ||||||||||||
| 4945 | // Make loop "reachable" from root so the CFG is reachable. Basically | ||||||||||||
| 4946 | // insert a bogus loop exit that is never taken. 'm', the loop head, | ||||||||||||
| 4947 | // points to 'n', one (of possibly many) fall-in paths. There may be | ||||||||||||
| 4948 | // many backedges as well. | ||||||||||||
| 4949 | |||||||||||||
| 4950 | // Here I set the loop to be the root loop. I could have, after | ||||||||||||
| 4951 | // inserting a bogus loop exit, restarted the recursion and found my | ||||||||||||
| 4952 | // new loop exit. This would make the infinite loop a first-class | ||||||||||||
| 4953 | // loop and it would then get properly optimized. What's the use of | ||||||||||||
| 4954 | // optimizing an infinite loop? | ||||||||||||
| 4955 | l = _ltree_root; // Oops, found infinite loop | ||||||||||||
| 4956 | |||||||||||||
| 4957 | if (!_verify_only) { | ||||||||||||
| 4958 | // Insert the NeverBranch between 'm' and it's control user. | ||||||||||||
| 4959 | NeverBranchNode *iff = new NeverBranchNode( m ); | ||||||||||||
| 4960 | _igvn.register_new_node_with_optimizer(iff); | ||||||||||||
| 4961 | set_loop(iff, l); | ||||||||||||
| 4962 | Node *if_t = new CProjNode( iff, 0 ); | ||||||||||||
| 4963 | _igvn.register_new_node_with_optimizer(if_t); | ||||||||||||
| 4964 | set_loop(if_t, l); | ||||||||||||
| 4965 | |||||||||||||
| 4966 | Node* cfg = NULL__null; // Find the One True Control User of m | ||||||||||||
| 4967 | for (DUIterator_Fast jmax, j = m->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 4968 | Node* x = m->fast_out(j); | ||||||||||||
| 4969 | if (x->is_CFG() && x != m && x != iff) | ||||||||||||
| 4970 | { cfg = x; break; } | ||||||||||||
| 4971 | } | ||||||||||||
| 4972 | assert(cfg != NULL, "must find the control user of m")do { if (!(cfg != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 4972, "assert(" "cfg != __null" ") failed", "must find the control user of m" ); ::breakpoint(); } } while (0); | ||||||||||||
| 4973 | uint k = 0; // Probably cfg->in(0) | ||||||||||||
| 4974 | while( cfg->in(k) != m ) k++; // But check incase cfg is a Region | ||||||||||||
| 4975 | cfg->set_req( k, if_t ); // Now point to NeverBranch | ||||||||||||
| 4976 | _igvn._worklist.push(cfg); | ||||||||||||
| 4977 | |||||||||||||
| 4978 | // Now create the never-taken loop exit | ||||||||||||
| 4979 | Node *if_f = new CProjNode( iff, 1 ); | ||||||||||||
| 4980 | _igvn.register_new_node_with_optimizer(if_f); | ||||||||||||
| 4981 | set_loop(if_f, l); | ||||||||||||
| 4982 | // Find frame ptr for Halt. Relies on the optimizer | ||||||||||||
| 4983 | // V-N'ing. Easier and quicker than searching through | ||||||||||||
| 4984 | // the program structure. | ||||||||||||
| 4985 | Node *frame = new ParmNode( C->start(), TypeFunc::FramePtr ); | ||||||||||||
| 4986 | _igvn.register_new_node_with_optimizer(frame); | ||||||||||||
| 4987 | // Halt & Catch Fire | ||||||||||||
| 4988 | Node* halt = new HaltNode(if_f, frame, "never-taken loop exit reached"); | ||||||||||||
| 4989 | _igvn.register_new_node_with_optimizer(halt); | ||||||||||||
| 4990 | set_loop(halt, l); | ||||||||||||
| 4991 | C->root()->add_req(halt); | ||||||||||||
| 4992 | } | ||||||||||||
| 4993 | set_loop(C->root(), _ltree_root); | ||||||||||||
| 4994 | } | ||||||||||||
| 4995 | } | ||||||||||||
| 4996 | // Weeny check for irreducible. This child was already visited (this | ||||||||||||
| 4997 | // IS the post-work phase). Is this child's loop header post-visited | ||||||||||||
| 4998 | // as well? If so, then I found another entry into the loop. | ||||||||||||
| 4999 | if (!_verify_only) { | ||||||||||||
| 5000 | while( is_postvisited(l->_head) ) { | ||||||||||||
| 5001 | // found irreducible | ||||||||||||
| 5002 | l->_irreducible = 1; // = true | ||||||||||||
| 5003 | l = l->_parent; | ||||||||||||
| 5004 | _has_irreducible_loops = true; | ||||||||||||
| 5005 | // Check for bad CFG here to prevent crash, and bailout of compile | ||||||||||||
| 5006 | if (l == NULL__null) { | ||||||||||||
| 5007 | C->record_method_not_compilable("unhandled CFG detected during loop optimization"); | ||||||||||||
| 5008 | return pre_order; | ||||||||||||
| 5009 | } | ||||||||||||
| 5010 | } | ||||||||||||
| 5011 | C->set_has_irreducible_loop(_has_irreducible_loops); | ||||||||||||
| 5012 | } | ||||||||||||
| 5013 | |||||||||||||
| 5014 | // This Node might be a decision point for loops. It is only if | ||||||||||||
| 5015 | // it's children belong to several different loops. The sort call | ||||||||||||
| 5016 | // does a trivial amount of work if there is only 1 child or all | ||||||||||||
| 5017 | // children belong to the same loop. If however, the children | ||||||||||||
| 5018 | // belong to different loops, the sort call will properly set the | ||||||||||||
| 5019 | // _parent pointers to show how the loops nest. | ||||||||||||
| 5020 | // | ||||||||||||
| 5021 | // In any case, it returns the tightest enclosing loop. | ||||||||||||
| 5022 | innermost = sort( l, innermost ); | ||||||||||||
| 5023 | } | ||||||||||||
| 5024 | |||||||||||||
| 5025 | // Def-use info will have some dead stuff; dead stuff will have no | ||||||||||||
| 5026 | // loop decided on. | ||||||||||||
| 5027 | |||||||||||||
| 5028 | // Am I a loop header? If so fix up my parent's child and next ptrs. | ||||||||||||
| 5029 | if( innermost && innermost->_head == n ) { | ||||||||||||
| 5030 | assert( get_loop(n) == innermost, "" )do { if (!(get_loop(n) == innermost)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5030, "assert(" "get_loop(n) == innermost" ") failed", ""); ::breakpoint(); } } while (0); | ||||||||||||
| 5031 | IdealLoopTree *p = innermost->_parent; | ||||||||||||
| 5032 | IdealLoopTree *l = innermost; | ||||||||||||
| 5033 | while( p && l->_head == n ) { | ||||||||||||
| 5034 | l->_next = p->_child; // Put self on parents 'next child' | ||||||||||||
| 5035 | p->_child = l; // Make self as first child of parent | ||||||||||||
| 5036 | l = p; // Now walk up the parent chain | ||||||||||||
| 5037 | p = l->_parent; | ||||||||||||
| 5038 | } | ||||||||||||
| 5039 | } else { | ||||||||||||
| 5040 | // Note that it is possible for a LoopNode to reach here, if the | ||||||||||||
| 5041 | // backedge has been made unreachable (hence the LoopNode no longer | ||||||||||||
| 5042 | // denotes a Loop, and will eventually be removed). | ||||||||||||
| 5043 | |||||||||||||
| 5044 | // Record tightest enclosing loop for self. Mark as post-visited. | ||||||||||||
| 5045 | set_loop(n, innermost); | ||||||||||||
| 5046 | // Also record has_call flag early on | ||||||||||||
| 5047 | if( innermost ) { | ||||||||||||
| 5048 | if( n->is_Call() && !n->is_CallLeaf() && !n->is_macro() ) { | ||||||||||||
| 5049 | // Do not count uncommon calls | ||||||||||||
| 5050 | if( !n->is_CallStaticJava() || !n->as_CallStaticJava()->_name ) { | ||||||||||||
| 5051 | Node *iff = n->in(0)->in(0); | ||||||||||||
| 5052 | // No any calls for vectorized loops. | ||||||||||||
| 5053 | if( UseSuperWord || !iff->is_If() || | ||||||||||||
| 5054 | (n->in(0)->Opcode() == Op_IfFalse && | ||||||||||||
| 5055 | (1.0 - iff->as_If()->_prob) >= 0.01) || | ||||||||||||
| 5056 | (iff->as_If()->_prob >= 0.01) ) | ||||||||||||
| 5057 | innermost->_has_call = 1; | ||||||||||||
| 5058 | } | ||||||||||||
| 5059 | } else if( n->is_Allocate() && n->as_Allocate()->_is_scalar_replaceable ) { | ||||||||||||
| 5060 | // Disable loop optimizations if the loop has a scalar replaceable | ||||||||||||
| 5061 | // allocation. This disabling may cause a potential performance lost | ||||||||||||
| 5062 | // if the allocation is not eliminated for some reason. | ||||||||||||
| 5063 | innermost->_allow_optimizations = false; | ||||||||||||
| 5064 | innermost->_has_call = 1; // = true | ||||||||||||
| 5065 | } else if (n->Opcode() == Op_SafePoint) { | ||||||||||||
| 5066 | // Record all safepoints in this loop. | ||||||||||||
| 5067 | if (innermost->_safepts == NULL__null) innermost->_safepts = new Node_List(); | ||||||||||||
| 5068 | innermost->_safepts->push(n); | ||||||||||||
| 5069 | } | ||||||||||||
| 5070 | } | ||||||||||||
| 5071 | } | ||||||||||||
| 5072 | |||||||||||||
| 5073 | // Flag as post-visited now | ||||||||||||
| 5074 | set_postvisited(n); | ||||||||||||
| 5075 | return pre_order; | ||||||||||||
| 5076 | } | ||||||||||||
| 5077 | |||||||||||||
| 5078 | |||||||||||||
| 5079 | //------------------------------build_loop_early------------------------------- | ||||||||||||
| 5080 | // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. | ||||||||||||
| 5081 | // First pass computes the earliest controlling node possible. This is the | ||||||||||||
| 5082 | // controlling input with the deepest dominating depth. | ||||||||||||
| 5083 | void PhaseIdealLoop::build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { | ||||||||||||
| 5084 | while (worklist.size() != 0) { | ||||||||||||
| 5085 | // Use local variables nstack_top_n & nstack_top_i to cache values | ||||||||||||
| 5086 | // on nstack's top. | ||||||||||||
| 5087 | Node *nstack_top_n = worklist.pop(); | ||||||||||||
| 5088 | uint nstack_top_i = 0; | ||||||||||||
| 5089 | //while_nstack_nonempty: | ||||||||||||
| 5090 | while (true) { | ||||||||||||
| 5091 | // Get parent node and next input's index from stack's top. | ||||||||||||
| 5092 | Node *n = nstack_top_n; | ||||||||||||
| 5093 | uint i = nstack_top_i; | ||||||||||||
| 5094 | uint cnt = n->req(); // Count of inputs | ||||||||||||
| 5095 | if (i == 0) { // Pre-process the node. | ||||||||||||
| 5096 | if( has_node(n) && // Have either loop or control already? | ||||||||||||
| 5097 | !has_ctrl(n) ) { // Have loop picked out already? | ||||||||||||
| 5098 | // During "merge_many_backedges" we fold up several nested loops | ||||||||||||
| 5099 | // into a single loop. This makes the members of the original | ||||||||||||
| 5100 | // loop bodies pointing to dead loops; they need to move up | ||||||||||||
| 5101 | // to the new UNION'd larger loop. I set the _head field of these | ||||||||||||
| 5102 | // dead loops to NULL and the _parent field points to the owning | ||||||||||||
| 5103 | // loop. Shades of UNION-FIND algorithm. | ||||||||||||
| 5104 | IdealLoopTree *ilt; | ||||||||||||
| 5105 | while( !(ilt = get_loop(n))->_head ) { | ||||||||||||
| 5106 | // Normally I would use a set_loop here. But in this one special | ||||||||||||
| 5107 | // case, it is legal (and expected) to change what loop a Node | ||||||||||||
| 5108 | // belongs to. | ||||||||||||
| 5109 | _nodes.map(n->_idx, (Node*)(ilt->_parent) ); | ||||||||||||
| 5110 | } | ||||||||||||
| 5111 | // Remove safepoints ONLY if I've already seen I don't need one. | ||||||||||||
| 5112 | // (the old code here would yank a 2nd safepoint after seeing a | ||||||||||||
| 5113 | // first one, even though the 1st did not dominate in the loop body | ||||||||||||
| 5114 | // and thus could be avoided indefinitely) | ||||||||||||
| 5115 | if( !_verify_only && !_verify_me && ilt->_has_sfpt && n->Opcode() == Op_SafePoint && | ||||||||||||
| 5116 | is_deleteable_safept(n)) { | ||||||||||||
| 5117 | Node *in = n->in(TypeFunc::Control); | ||||||||||||
| 5118 | lazy_replace(n,in); // Pull safepoint now | ||||||||||||
| 5119 | if (ilt->_safepts != NULL__null) { | ||||||||||||
| 5120 | ilt->_safepts->yank(n); | ||||||||||||
| 5121 | } | ||||||||||||
| 5122 | // Carry on with the recursion "as if" we are walking | ||||||||||||
| 5123 | // only the control input | ||||||||||||
| 5124 | if( !visited.test_set( in->_idx ) ) { | ||||||||||||
| 5125 | worklist.push(in); // Visit this guy later, using worklist | ||||||||||||
| 5126 | } | ||||||||||||
| 5127 | // Get next node from nstack: | ||||||||||||
| 5128 | // - skip n's inputs processing by setting i > cnt; | ||||||||||||
| 5129 | // - we also will not call set_early_ctrl(n) since | ||||||||||||
| 5130 | // has_node(n) == true (see the condition above). | ||||||||||||
| 5131 | i = cnt + 1; | ||||||||||||
| 5132 | } | ||||||||||||
| 5133 | } | ||||||||||||
| 5134 | } // if (i == 0) | ||||||||||||
| 5135 | |||||||||||||
| 5136 | // Visit all inputs | ||||||||||||
| 5137 | bool done = true; // Assume all n's inputs will be processed | ||||||||||||
| 5138 | while (i < cnt) { | ||||||||||||
| 5139 | Node *in = n->in(i); | ||||||||||||
| 5140 | ++i; | ||||||||||||
| 5141 | if (in == NULL__null) continue; | ||||||||||||
| 5142 | if (in->pinned() && !in->is_CFG()) | ||||||||||||
| 5143 | set_ctrl(in, in->in(0)); | ||||||||||||
| 5144 | int is_visited = visited.test_set( in->_idx ); | ||||||||||||
| 5145 | if (!has_node(in)) { // No controlling input yet? | ||||||||||||
| 5146 | assert( !in->is_CFG(), "CFG Node with no controlling input?" )do { if (!(!in->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5146, "assert(" "!in->is_CFG()" ") failed", "CFG Node with no controlling input?" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5147 | assert( !is_visited, "visit only once" )do { if (!(!is_visited)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5147, "assert(" "!is_visited" ") failed", "visit only once" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5148 | nstack.push(n, i); // Save parent node and next input's index. | ||||||||||||
| 5149 | nstack_top_n = in; // Process current input now. | ||||||||||||
| 5150 | nstack_top_i = 0; | ||||||||||||
| 5151 | done = false; // Not all n's inputs processed. | ||||||||||||
| 5152 | break; // continue while_nstack_nonempty; | ||||||||||||
| 5153 | } else if (!is_visited) { | ||||||||||||
| 5154 | // This guy has a location picked out for him, but has not yet | ||||||||||||
| 5155 | // been visited. Happens to all CFG nodes, for instance. | ||||||||||||
| 5156 | // Visit him using the worklist instead of recursion, to break | ||||||||||||
| 5157 | // cycles. Since he has a location already we do not need to | ||||||||||||
| 5158 | // find his location before proceeding with the current Node. | ||||||||||||
| 5159 | worklist.push(in); // Visit this guy later, using worklist | ||||||||||||
| 5160 | } | ||||||||||||
| 5161 | } | ||||||||||||
| 5162 | if (done) { | ||||||||||||
| 5163 | // All of n's inputs have been processed, complete post-processing. | ||||||||||||
| 5164 | |||||||||||||
| 5165 | // Compute earliest point this Node can go. | ||||||||||||
| 5166 | // CFG, Phi, pinned nodes already know their controlling input. | ||||||||||||
| 5167 | if (!has_node(n)) { | ||||||||||||
| 5168 | // Record earliest legal location | ||||||||||||
| 5169 | set_early_ctrl(n, false); | ||||||||||||
| 5170 | } | ||||||||||||
| 5171 | if (nstack.is_empty()) { | ||||||||||||
| 5172 | // Finished all nodes on stack. | ||||||||||||
| 5173 | // Process next node on the worklist. | ||||||||||||
| 5174 | break; | ||||||||||||
| 5175 | } | ||||||||||||
| 5176 | // Get saved parent node and next input's index. | ||||||||||||
| 5177 | nstack_top_n = nstack.node(); | ||||||||||||
| 5178 | nstack_top_i = nstack.index(); | ||||||||||||
| 5179 | nstack.pop(); | ||||||||||||
| 5180 | } | ||||||||||||
| 5181 | } // while (true) | ||||||||||||
| 5182 | } | ||||||||||||
| 5183 | } | ||||||||||||
| 5184 | |||||||||||||
| 5185 | //------------------------------dom_lca_internal-------------------------------- | ||||||||||||
| 5186 | // Pair-wise LCA | ||||||||||||
| 5187 | Node *PhaseIdealLoop::dom_lca_internal( Node *n1, Node *n2 ) const { | ||||||||||||
| 5188 | if( !n1 ) return n2; // Handle NULL original LCA | ||||||||||||
| 5189 | assert( n1->is_CFG(), "" )do { if (!(n1->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5189, "assert(" "n1->is_CFG()" ") failed", ""); ::breakpoint (); } } while (0); | ||||||||||||
| 5190 | assert( n2->is_CFG(), "" )do { if (!(n2->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5190, "assert(" "n2->is_CFG()" ") failed", ""); ::breakpoint (); } } while (0); | ||||||||||||
| 5191 | // find LCA of all uses | ||||||||||||
| 5192 | uint d1 = dom_depth(n1); | ||||||||||||
| 5193 | uint d2 = dom_depth(n2); | ||||||||||||
| 5194 | while (n1 != n2) { | ||||||||||||
| 5195 | if (d1 > d2) { | ||||||||||||
| 5196 | n1 = idom(n1); | ||||||||||||
| 5197 | d1 = dom_depth(n1); | ||||||||||||
| 5198 | } else if (d1 < d2) { | ||||||||||||
| 5199 | n2 = idom(n2); | ||||||||||||
| 5200 | d2 = dom_depth(n2); | ||||||||||||
| 5201 | } else { | ||||||||||||
| 5202 | // Here d1 == d2. Due to edits of the dominator-tree, sections | ||||||||||||
| 5203 | // of the tree might have the same depth. These sections have | ||||||||||||
| 5204 | // to be searched more carefully. | ||||||||||||
| 5205 | |||||||||||||
| 5206 | // Scan up all the n1's with equal depth, looking for n2. | ||||||||||||
| 5207 | Node *t1 = idom(n1); | ||||||||||||
| 5208 | while (dom_depth(t1) == d1) { | ||||||||||||
| 5209 | if (t1 == n2) return n2; | ||||||||||||
| 5210 | t1 = idom(t1); | ||||||||||||
| 5211 | } | ||||||||||||
| 5212 | // Scan up all the n2's with equal depth, looking for n1. | ||||||||||||
| 5213 | Node *t2 = idom(n2); | ||||||||||||
| 5214 | while (dom_depth(t2) == d2) { | ||||||||||||
| 5215 | if (t2 == n1) return n1; | ||||||||||||
| 5216 | t2 = idom(t2); | ||||||||||||
| 5217 | } | ||||||||||||
| 5218 | // Move up to a new dominator-depth value as well as up the dom-tree. | ||||||||||||
| 5219 | n1 = t1; | ||||||||||||
| 5220 | n2 = t2; | ||||||||||||
| 5221 | d1 = dom_depth(n1); | ||||||||||||
| 5222 | d2 = dom_depth(n2); | ||||||||||||
| 5223 | } | ||||||||||||
| 5224 | } | ||||||||||||
| 5225 | return n1; | ||||||||||||
| 5226 | } | ||||||||||||
| 5227 | |||||||||||||
| 5228 | //------------------------------compute_idom----------------------------------- | ||||||||||||
| 5229 | // Locally compute IDOM using dom_lca call. Correct only if the incoming | ||||||||||||
| 5230 | // IDOMs are correct. | ||||||||||||
| 5231 | Node *PhaseIdealLoop::compute_idom( Node *region ) const { | ||||||||||||
| 5232 | assert( region->is_Region(), "" )do { if (!(region->is_Region())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5232, "assert(" "region->is_Region()" ") failed", ""); :: breakpoint(); } } while (0); | ||||||||||||
| 5233 | Node *LCA = NULL__null; | ||||||||||||
| 5234 | for( uint i = 1; i < region->req(); i++ ) { | ||||||||||||
| 5235 | if( region->in(i) != C->top() ) | ||||||||||||
| 5236 | LCA = dom_lca( LCA, region->in(i) ); | ||||||||||||
| 5237 | } | ||||||||||||
| 5238 | return LCA; | ||||||||||||
| 5239 | } | ||||||||||||
| 5240 | |||||||||||||
| 5241 | bool PhaseIdealLoop::verify_dominance(Node* n, Node* use, Node* LCA, Node* early) { | ||||||||||||
| 5242 | bool had_error = false; | ||||||||||||
| 5243 | #ifdef ASSERT1 | ||||||||||||
| 5244 | if (early != C->root()) { | ||||||||||||
| 5245 | // Make sure that there's a dominance path from LCA to early | ||||||||||||
| 5246 | Node* d = LCA; | ||||||||||||
| 5247 | while (d != early) { | ||||||||||||
| 5248 | if (d == C->root()) { | ||||||||||||
| 5249 | dump_bad_graph("Bad graph detected in compute_lca_of_uses", n, early, LCA); | ||||||||||||
| 5250 | tty->print_cr("*** Use %d isn't dominated by def %d ***", use->_idx, n->_idx); | ||||||||||||
| 5251 | had_error = true; | ||||||||||||
| 5252 | break; | ||||||||||||
| 5253 | } | ||||||||||||
| 5254 | d = idom(d); | ||||||||||||
| 5255 | } | ||||||||||||
| 5256 | } | ||||||||||||
| 5257 | #endif | ||||||||||||
| 5258 | return had_error; | ||||||||||||
| 5259 | } | ||||||||||||
| 5260 | |||||||||||||
| 5261 | |||||||||||||
| 5262 | Node* PhaseIdealLoop::compute_lca_of_uses(Node* n, Node* early, bool verify) { | ||||||||||||
| 5263 | // Compute LCA over list of uses | ||||||||||||
| 5264 | bool had_error = false; | ||||||||||||
| 5265 | Node *LCA = NULL__null; | ||||||||||||
| 5266 | for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax && LCA != early; i++) { | ||||||||||||
| 5267 | Node* c = n->fast_out(i); | ||||||||||||
| 5268 | if (_nodes[c->_idx] == NULL__null) | ||||||||||||
| 5269 | continue; // Skip the occasional dead node | ||||||||||||
| 5270 | if( c->is_Phi() ) { // For Phis, we must land above on the path | ||||||||||||
| 5271 | for( uint j=1; j<c->req(); j++ ) {// For all inputs | ||||||||||||
| 5272 | if( c->in(j) == n ) { // Found matching input? | ||||||||||||
| 5273 | Node *use = c->in(0)->in(j); | ||||||||||||
| 5274 | if (_verify_only && use->is_top()) continue; | ||||||||||||
| 5275 | LCA = dom_lca_for_get_late_ctrl( LCA, use, n ); | ||||||||||||
| 5276 | if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error; | ||||||||||||
| 5277 | } | ||||||||||||
| 5278 | } | ||||||||||||
| 5279 | } else { | ||||||||||||
| 5280 | // For CFG data-users, use is in the block just prior | ||||||||||||
| 5281 | Node *use = has_ctrl(c) ? get_ctrl(c) : c->in(0); | ||||||||||||
| 5282 | LCA = dom_lca_for_get_late_ctrl( LCA, use, n ); | ||||||||||||
| 5283 | if (verify) had_error = verify_dominance(n, use, LCA, early) || had_error; | ||||||||||||
| 5284 | } | ||||||||||||
| 5285 | } | ||||||||||||
| 5286 | assert(!had_error, "bad dominance")do { if (!(!had_error)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5286, "assert(" "!had_error" ") failed", "bad dominance"); :: breakpoint(); } } while (0); | ||||||||||||
| 5287 | return LCA; | ||||||||||||
| 5288 | } | ||||||||||||
| 5289 | |||||||||||||
| 5290 | // Check the shape of the graph at the loop entry. In some cases, | ||||||||||||
| 5291 | // the shape of the graph does not match the shape outlined below. | ||||||||||||
| 5292 | // That is caused by the Opaque1 node "protecting" the shape of | ||||||||||||
| 5293 | // the graph being removed by, for example, the IGVN performed | ||||||||||||
| 5294 | // in PhaseIdealLoop::build_and_optimize(). | ||||||||||||
| 5295 | // | ||||||||||||
| 5296 | // After the Opaque1 node has been removed, optimizations (e.g., split-if, | ||||||||||||
| 5297 | // loop unswitching, and IGVN, or a combination of them) can freely change | ||||||||||||
| 5298 | // the graph's shape. As a result, the graph shape outlined below cannot | ||||||||||||
| 5299 | // be guaranteed anymore. | ||||||||||||
| 5300 | Node* CountedLoopNode::is_canonical_loop_entry() { | ||||||||||||
| 5301 | if (!is_main_loop() && !is_post_loop()) { | ||||||||||||
| 5302 | return NULL__null; | ||||||||||||
| 5303 | } | ||||||||||||
| 5304 | Node* ctrl = skip_predicates(); | ||||||||||||
| 5305 | |||||||||||||
| 5306 | if (ctrl == NULL__null || (!ctrl->is_IfTrue() && !ctrl->is_IfFalse())) { | ||||||||||||
| 5307 | return NULL__null; | ||||||||||||
| 5308 | } | ||||||||||||
| 5309 | Node* iffm = ctrl->in(0); | ||||||||||||
| 5310 | if (iffm == NULL__null || !iffm->is_If()) { | ||||||||||||
| 5311 | return NULL__null; | ||||||||||||
| 5312 | } | ||||||||||||
| 5313 | Node* bolzm = iffm->in(1); | ||||||||||||
| 5314 | if (bolzm == NULL__null || !bolzm->is_Bool()) { | ||||||||||||
| 5315 | return NULL__null; | ||||||||||||
| 5316 | } | ||||||||||||
| 5317 | Node* cmpzm = bolzm->in(1); | ||||||||||||
| 5318 | if (cmpzm == NULL__null || !cmpzm->is_Cmp()) { | ||||||||||||
| 5319 | return NULL__null; | ||||||||||||
| 5320 | } | ||||||||||||
| 5321 | |||||||||||||
| 5322 | uint input = is_main_loop() ? 2 : 1; | ||||||||||||
| 5323 | if (input >= cmpzm->req() || cmpzm->in(input) == NULL__null) { | ||||||||||||
| 5324 | return NULL__null; | ||||||||||||
| 5325 | } | ||||||||||||
| 5326 | bool res = cmpzm->in(input)->Opcode() == Op_Opaque1; | ||||||||||||
| 5327 | #ifdef ASSERT1 | ||||||||||||
| 5328 | bool found_opaque = false; | ||||||||||||
| 5329 | for (uint i = 1; i < cmpzm->req(); i++) { | ||||||||||||
| 5330 | Node* opnd = cmpzm->in(i); | ||||||||||||
| 5331 | if (opnd && opnd->Opcode() == Op_Opaque1) { | ||||||||||||
| 5332 | found_opaque = true; | ||||||||||||
| 5333 | break; | ||||||||||||
| 5334 | } | ||||||||||||
| 5335 | } | ||||||||||||
| 5336 | assert(found_opaque == res, "wrong pattern")do { if (!(found_opaque == res)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5336, "assert(" "found_opaque == res" ") failed", "wrong pattern" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5337 | #endif | ||||||||||||
| 5338 | return res ? cmpzm->in(input) : NULL__null; | ||||||||||||
| 5339 | } | ||||||||||||
| 5340 | |||||||||||||
| 5341 | //------------------------------get_late_ctrl---------------------------------- | ||||||||||||
| 5342 | // Compute latest legal control. | ||||||||||||
| 5343 | Node *PhaseIdealLoop::get_late_ctrl( Node *n, Node *early ) { | ||||||||||||
| 5344 | assert(early != NULL, "early control should not be NULL")do { if (!(early != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5344, "assert(" "early != __null" ") failed", "early control should not be NULL" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5345 | |||||||||||||
| 5346 | Node* LCA = compute_lca_of_uses(n, early); | ||||||||||||
| 5347 | #ifdef ASSERT1 | ||||||||||||
| 5348 | if (LCA == C->root() && LCA != early) { | ||||||||||||
| 5349 | // def doesn't dominate uses so print some useful debugging output | ||||||||||||
| 5350 | compute_lca_of_uses(n, early, true); | ||||||||||||
| 5351 | } | ||||||||||||
| 5352 | #endif | ||||||||||||
| 5353 | |||||||||||||
| 5354 | if (n->is_Load() && LCA != early) { | ||||||||||||
| 5355 | LCA = get_late_ctrl_with_anti_dep(n->as_Load(), early, LCA); | ||||||||||||
| 5356 | } | ||||||||||||
| 5357 | |||||||||||||
| 5358 | assert(LCA == find_non_split_ctrl(LCA), "unexpected late control")do { if (!(LCA == find_non_split_ctrl(LCA))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5358, "assert(" "LCA == find_non_split_ctrl(LCA)" ") failed" , "unexpected late control"); ::breakpoint(); } } while (0); | ||||||||||||
| 5359 | return LCA; | ||||||||||||
| 5360 | } | ||||||||||||
| 5361 | |||||||||||||
| 5362 | // if this is a load, check for anti-dependent stores | ||||||||||||
| 5363 | // We use a conservative algorithm to identify potential interfering | ||||||||||||
| 5364 | // instructions and for rescheduling the load. The users of the memory | ||||||||||||
| 5365 | // input of this load are examined. Any use which is not a load and is | ||||||||||||
| 5366 | // dominated by early is considered a potentially interfering store. | ||||||||||||
| 5367 | // This can produce false positives. | ||||||||||||
| 5368 | Node* PhaseIdealLoop::get_late_ctrl_with_anti_dep(LoadNode* n, Node* early, Node* LCA) { | ||||||||||||
| 5369 | int load_alias_idx = C->get_alias_index(n->adr_type()); | ||||||||||||
| 5370 | if (C->alias_type(load_alias_idx)->is_rewritable()) { | ||||||||||||
| 5371 | Unique_Node_List worklist; | ||||||||||||
| 5372 | |||||||||||||
| 5373 | Node* mem = n->in(MemNode::Memory); | ||||||||||||
| 5374 | for (DUIterator_Fast imax, i = mem->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 5375 | Node* s = mem->fast_out(i); | ||||||||||||
| 5376 | worklist.push(s); | ||||||||||||
| 5377 | } | ||||||||||||
| 5378 | for (uint i = 0; i < worklist.size() && LCA != early; i++) { | ||||||||||||
| 5379 | Node* s = worklist.at(i); | ||||||||||||
| 5380 | if (s->is_Load() || s->Opcode() == Op_SafePoint || | ||||||||||||
| 5381 | (s->is_CallStaticJava() && s->as_CallStaticJava()->uncommon_trap_request() != 0) || | ||||||||||||
| 5382 | s->is_Phi()) { | ||||||||||||
| 5383 | continue; | ||||||||||||
| 5384 | } else if (s->is_MergeMem()) { | ||||||||||||
| 5385 | for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 5386 | Node* s1 = s->fast_out(i); | ||||||||||||
| 5387 | worklist.push(s1); | ||||||||||||
| 5388 | } | ||||||||||||
| 5389 | } else { | ||||||||||||
| 5390 | Node* sctrl = has_ctrl(s) ? get_ctrl(s) : s->in(0); | ||||||||||||
| 5391 | assert(sctrl != NULL || !s->is_reachable_from_root(), "must have control")do { if (!(sctrl != __null || !s->is_reachable_from_root() )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5391, "assert(" "sctrl != __null || !s->is_reachable_from_root()" ") failed", "must have control"); ::breakpoint(); } } while ( 0); | ||||||||||||
| 5392 | if (sctrl != NULL__null && !sctrl->is_top() && is_dominator(early, sctrl)) { | ||||||||||||
| 5393 | const TypePtr* adr_type = s->adr_type(); | ||||||||||||
| 5394 | if (s->is_ArrayCopy()) { | ||||||||||||
| 5395 | // Copy to known instance needs destination type to test for aliasing | ||||||||||||
| 5396 | const TypePtr* dest_type = s->as_ArrayCopy()->_dest_type; | ||||||||||||
| 5397 | if (dest_type != TypeOopPtr::BOTTOM) { | ||||||||||||
| 5398 | adr_type = dest_type; | ||||||||||||
| 5399 | } | ||||||||||||
| 5400 | } | ||||||||||||
| 5401 | if (C->can_alias(adr_type, load_alias_idx)) { | ||||||||||||
| 5402 | LCA = dom_lca_for_get_late_ctrl(LCA, sctrl, n); | ||||||||||||
| 5403 | } else if (s->is_CFG() && s->is_Multi()) { | ||||||||||||
| 5404 | // Look for the memory use of s (that is the use of its memory projection) | ||||||||||||
| 5405 | for (DUIterator_Fast imax, i = s->fast_outs(imax); i < imax; i++) { | ||||||||||||
| 5406 | Node* s1 = s->fast_out(i); | ||||||||||||
| 5407 | assert(s1->is_Proj(), "projection expected")do { if (!(s1->is_Proj())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5407, "assert(" "s1->is_Proj()" ") failed", "projection expected" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5408 | if (_igvn.type(s1) == Type::MEMORY) { | ||||||||||||
| 5409 | for (DUIterator_Fast jmax, j = s1->fast_outs(jmax); j < jmax; j++) { | ||||||||||||
| 5410 | Node* s2 = s1->fast_out(j); | ||||||||||||
| 5411 | worklist.push(s2); | ||||||||||||
| 5412 | } | ||||||||||||
| 5413 | } | ||||||||||||
| 5414 | } | ||||||||||||
| 5415 | } | ||||||||||||
| 5416 | } | ||||||||||||
| 5417 | } | ||||||||||||
| 5418 | } | ||||||||||||
| 5419 | // For Phis only consider Region's inputs that were reached by following the memory edges | ||||||||||||
| 5420 | if (LCA != early) { | ||||||||||||
| 5421 | for (uint i = 0; i < worklist.size(); i++) { | ||||||||||||
| 5422 | Node* s = worklist.at(i); | ||||||||||||
| 5423 | if (s->is_Phi() && C->can_alias(s->adr_type(), load_alias_idx)) { | ||||||||||||
| 5424 | Node* r = s->in(0); | ||||||||||||
| 5425 | for (uint j = 1; j < s->req(); j++) { | ||||||||||||
| 5426 | Node* in = s->in(j); | ||||||||||||
| 5427 | Node* r_in = r->in(j); | ||||||||||||
| 5428 | // We can't reach any node from a Phi because we don't enqueue Phi's uses above | ||||||||||||
| 5429 | if (((worklist.member(in) && !in->is_Phi()) || in == mem) && is_dominator(early, r_in)) { | ||||||||||||
| 5430 | LCA = dom_lca_for_get_late_ctrl(LCA, r_in, n); | ||||||||||||
| 5431 | } | ||||||||||||
| 5432 | } | ||||||||||||
| 5433 | } | ||||||||||||
| 5434 | } | ||||||||||||
| 5435 | } | ||||||||||||
| 5436 | } | ||||||||||||
| 5437 | return LCA; | ||||||||||||
| 5438 | } | ||||||||||||
| 5439 | |||||||||||||
| 5440 | // true if CFG node d dominates CFG node n | ||||||||||||
| 5441 | bool PhaseIdealLoop::is_dominator(Node *d, Node *n) { | ||||||||||||
| 5442 | if (d == n) | ||||||||||||
| 5443 | return true; | ||||||||||||
| 5444 | assert(d->is_CFG() && n->is_CFG(), "must have CFG nodes")do { if (!(d->is_CFG() && n->is_CFG())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5444, "assert(" "d->is_CFG() && n->is_CFG()" ") failed" , "must have CFG nodes"); ::breakpoint(); } } while (0); | ||||||||||||
| 5445 | uint dd = dom_depth(d); | ||||||||||||
| 5446 | while (dom_depth(n) >= dd) { | ||||||||||||
| 5447 | if (n == d) | ||||||||||||
| 5448 | return true; | ||||||||||||
| 5449 | n = idom(n); | ||||||||||||
| 5450 | } | ||||||||||||
| 5451 | return false; | ||||||||||||
| 5452 | } | ||||||||||||
| 5453 | |||||||||||||
| 5454 | //------------------------------dom_lca_for_get_late_ctrl_internal------------- | ||||||||||||
| 5455 | // Pair-wise LCA with tags. | ||||||||||||
| 5456 | // Tag each index with the node 'tag' currently being processed | ||||||||||||
| 5457 | // before advancing up the dominator chain using idom(). | ||||||||||||
| 5458 | // Later calls that find a match to 'tag' know that this path has already | ||||||||||||
| 5459 | // been considered in the current LCA (which is input 'n1' by convention). | ||||||||||||
| 5460 | // Since get_late_ctrl() is only called once for each node, the tag array | ||||||||||||
| 5461 | // does not need to be cleared between calls to get_late_ctrl(). | ||||||||||||
| 5462 | // Algorithm trades a larger constant factor for better asymptotic behavior | ||||||||||||
| 5463 | // | ||||||||||||
| 5464 | Node *PhaseIdealLoop::dom_lca_for_get_late_ctrl_internal(Node *n1, Node *n2, Node *tag_node) { | ||||||||||||
| 5465 | uint d1 = dom_depth(n1); | ||||||||||||
| 5466 | uint d2 = dom_depth(n2); | ||||||||||||
| 5467 | jlong tag = tag_node->_idx | (((jlong)_dom_lca_tags_round) << 32); | ||||||||||||
| 5468 | |||||||||||||
| 5469 | do { | ||||||||||||
| 5470 | if (d1 > d2) { | ||||||||||||
| 5471 | // current lca is deeper than n2 | ||||||||||||
| 5472 | _dom_lca_tags.at_put_grow(n1->_idx, tag); | ||||||||||||
| 5473 | n1 = idom(n1); | ||||||||||||
| 5474 | d1 = dom_depth(n1); | ||||||||||||
| 5475 | } else if (d1 < d2) { | ||||||||||||
| 5476 | // n2 is deeper than current lca | ||||||||||||
| 5477 | jlong memo = _dom_lca_tags.at_grow(n2->_idx, 0); | ||||||||||||
| 5478 | if (memo == tag) { | ||||||||||||
| 5479 | return n1; // Return the current LCA | ||||||||||||
| 5480 | } | ||||||||||||
| 5481 | _dom_lca_tags.at_put_grow(n2->_idx, tag); | ||||||||||||
| 5482 | n2 = idom(n2); | ||||||||||||
| 5483 | d2 = dom_depth(n2); | ||||||||||||
| 5484 | } else { | ||||||||||||
| 5485 | // Here d1 == d2. Due to edits of the dominator-tree, sections | ||||||||||||
| 5486 | // of the tree might have the same depth. These sections have | ||||||||||||
| 5487 | // to be searched more carefully. | ||||||||||||
| 5488 | |||||||||||||
| 5489 | // Scan up all the n1's with equal depth, looking for n2. | ||||||||||||
| 5490 | _dom_lca_tags.at_put_grow(n1->_idx, tag); | ||||||||||||
| 5491 | Node *t1 = idom(n1); | ||||||||||||
| 5492 | while (dom_depth(t1) == d1) { | ||||||||||||
| 5493 | if (t1 == n2) return n2; | ||||||||||||
| 5494 | _dom_lca_tags.at_put_grow(t1->_idx, tag); | ||||||||||||
| 5495 | t1 = idom(t1); | ||||||||||||
| 5496 | } | ||||||||||||
| 5497 | // Scan up all the n2's with equal depth, looking for n1. | ||||||||||||
| 5498 | _dom_lca_tags.at_put_grow(n2->_idx, tag); | ||||||||||||
| 5499 | Node *t2 = idom(n2); | ||||||||||||
| 5500 | while (dom_depth(t2) == d2) { | ||||||||||||
| 5501 | if (t2 == n1) return n1; | ||||||||||||
| 5502 | _dom_lca_tags.at_put_grow(t2->_idx, tag); | ||||||||||||
| 5503 | t2 = idom(t2); | ||||||||||||
| 5504 | } | ||||||||||||
| 5505 | // Move up to a new dominator-depth value as well as up the dom-tree. | ||||||||||||
| 5506 | n1 = t1; | ||||||||||||
| 5507 | n2 = t2; | ||||||||||||
| 5508 | d1 = dom_depth(n1); | ||||||||||||
| 5509 | d2 = dom_depth(n2); | ||||||||||||
| 5510 | } | ||||||||||||
| 5511 | } while (n1 != n2); | ||||||||||||
| 5512 | return n1; | ||||||||||||
| 5513 | } | ||||||||||||
| 5514 | |||||||||||||
| 5515 | //------------------------------init_dom_lca_tags------------------------------ | ||||||||||||
| 5516 | // Tag could be a node's integer index, 32bits instead of 64bits in some cases | ||||||||||||
| 5517 | // Intended use does not involve any growth for the array, so it could | ||||||||||||
| 5518 | // be of fixed size. | ||||||||||||
| 5519 | void PhaseIdealLoop::init_dom_lca_tags() { | ||||||||||||
| 5520 | uint limit = C->unique() + 1; | ||||||||||||
| 5521 | _dom_lca_tags.at_grow(limit, 0); | ||||||||||||
| 5522 | _dom_lca_tags_round = 0; | ||||||||||||
| 5523 | #ifdef ASSERT1 | ||||||||||||
| 5524 | for (uint i = 0; i < limit; ++i) { | ||||||||||||
| 5525 | assert(_dom_lca_tags.at(i) == 0, "Must be distinct from each node pointer")do { if (!(_dom_lca_tags.at(i) == 0)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5525, "assert(" "_dom_lca_tags.at(i) == 0" ") failed", "Must be distinct from each node pointer" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5526 | } | ||||||||||||
| 5527 | #endif // ASSERT | ||||||||||||
| 5528 | } | ||||||||||||
| 5529 | |||||||||||||
| 5530 | //------------------------------build_loop_late-------------------------------- | ||||||||||||
| 5531 | // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. | ||||||||||||
| 5532 | // Second pass finds latest legal placement, and ideal loop placement. | ||||||||||||
| 5533 | void PhaseIdealLoop::build_loop_late( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ) { | ||||||||||||
| 5534 | while (worklist.size() != 0) { | ||||||||||||
| 5535 | Node *n = worklist.pop(); | ||||||||||||
| 5536 | // Only visit once | ||||||||||||
| 5537 | if (visited.test_set(n->_idx)) continue; | ||||||||||||
| 5538 | uint cnt = n->outcnt(); | ||||||||||||
| 5539 | uint i = 0; | ||||||||||||
| 5540 | while (true) { | ||||||||||||
| 5541 | assert( _nodes[n->_idx], "no dead nodes" )do { if (!(_nodes[n->_idx])) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5541, "assert(" "_nodes[n->_idx]" ") failed", "no dead nodes" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5542 | // Visit all children | ||||||||||||
| 5543 | if (i < cnt) { | ||||||||||||
| 5544 | Node* use = n->raw_out(i); | ||||||||||||
| 5545 | ++i; | ||||||||||||
| 5546 | // Check for dead uses. Aggressively prune such junk. It might be | ||||||||||||
| 5547 | // dead in the global sense, but still have local uses so I cannot | ||||||||||||
| 5548 | // easily call 'remove_dead_node'. | ||||||||||||
| 5549 | if( _nodes[use->_idx] != NULL__null || use->is_top() ) { // Not dead? | ||||||||||||
| 5550 | // Due to cycles, we might not hit the same fixed point in the verify | ||||||||||||
| 5551 | // pass as we do in the regular pass. Instead, visit such phis as | ||||||||||||
| 5552 | // simple uses of the loop head. | ||||||||||||
| 5553 | if( use->in(0) && (use->is_CFG() || use->is_Phi()) ) { | ||||||||||||
| 5554 | if( !visited.test(use->_idx) ) | ||||||||||||
| 5555 | worklist.push(use); | ||||||||||||
| 5556 | } else if( !visited.test_set(use->_idx) ) { | ||||||||||||
| 5557 | nstack.push(n, i); // Save parent and next use's index. | ||||||||||||
| 5558 | n = use; // Process all children of current use. | ||||||||||||
| 5559 | cnt = use->outcnt(); | ||||||||||||
| 5560 | i = 0; | ||||||||||||
| 5561 | } | ||||||||||||
| 5562 | } else { | ||||||||||||
| 5563 | // Do not visit around the backedge of loops via data edges. | ||||||||||||
| 5564 | // push dead code onto a worklist | ||||||||||||
| 5565 | _deadlist.push(use); | ||||||||||||
| 5566 | } | ||||||||||||
| 5567 | } else { | ||||||||||||
| 5568 | // All of n's children have been processed, complete post-processing. | ||||||||||||
| 5569 | build_loop_late_post(n); | ||||||||||||
| 5570 | if (nstack.is_empty()) { | ||||||||||||
| 5571 | // Finished all nodes on stack. | ||||||||||||
| 5572 | // Process next node on the worklist. | ||||||||||||
| 5573 | break; | ||||||||||||
| 5574 | } | ||||||||||||
| 5575 | // Get saved parent node and next use's index. Visit the rest of uses. | ||||||||||||
| 5576 | n = nstack.node(); | ||||||||||||
| 5577 | cnt = n->outcnt(); | ||||||||||||
| 5578 | i = nstack.index(); | ||||||||||||
| 5579 | nstack.pop(); | ||||||||||||
| 5580 | } | ||||||||||||
| 5581 | } | ||||||||||||
| 5582 | } | ||||||||||||
| 5583 | } | ||||||||||||
| 5584 | |||||||||||||
| 5585 | // Verify that no data node is scheduled in the outer loop of a strip | ||||||||||||
| 5586 | // mined loop. | ||||||||||||
| 5587 | void PhaseIdealLoop::verify_strip_mined_scheduling(Node *n, Node* least) { | ||||||||||||
| 5588 | #ifdef ASSERT1 | ||||||||||||
| 5589 | if (get_loop(least)->_nest == 0) { | ||||||||||||
| 5590 | return; | ||||||||||||
| 5591 | } | ||||||||||||
| 5592 | IdealLoopTree* loop = get_loop(least); | ||||||||||||
| 5593 | Node* head = loop->_head; | ||||||||||||
| 5594 | if (head->is_OuterStripMinedLoop() && | ||||||||||||
| 5595 | // Verification can't be applied to fully built strip mined loops | ||||||||||||
| 5596 | head->as_Loop()->outer_loop_end()->in(1)->find_int_con(-1) == 0) { | ||||||||||||
| 5597 | Node* sfpt = head->as_Loop()->outer_safepoint(); | ||||||||||||
| 5598 | ResourceMark rm; | ||||||||||||
| 5599 | Unique_Node_List wq; | ||||||||||||
| 5600 | wq.push(sfpt); | ||||||||||||
| 5601 | for (uint i = 0; i < wq.size(); i++) { | ||||||||||||
| 5602 | Node *m = wq.at(i); | ||||||||||||
| 5603 | for (uint i = 1; i < m->req(); i++) { | ||||||||||||
| 5604 | Node* nn = m->in(i); | ||||||||||||
| 5605 | if (nn == n) { | ||||||||||||
| 5606 | return; | ||||||||||||
| 5607 | } | ||||||||||||
| 5608 | if (nn != NULL__null && has_ctrl(nn) && get_loop(get_ctrl(nn)) == loop) { | ||||||||||||
| 5609 | wq.push(nn); | ||||||||||||
| 5610 | } | ||||||||||||
| 5611 | } | ||||||||||||
| 5612 | } | ||||||||||||
| 5613 | ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5613); ::breakpoint(); } while (0); | ||||||||||||
| 5614 | } | ||||||||||||
| 5615 | #endif | ||||||||||||
| 5616 | } | ||||||||||||
| 5617 | |||||||||||||
| 5618 | |||||||||||||
| 5619 | //------------------------------build_loop_late_post--------------------------- | ||||||||||||
| 5620 | // Put Data nodes into some loop nest, by setting the _nodes[]->loop mapping. | ||||||||||||
| 5621 | // Second pass finds latest legal placement, and ideal loop placement. | ||||||||||||
| 5622 | void PhaseIdealLoop::build_loop_late_post(Node *n) { | ||||||||||||
| 5623 | build_loop_late_post_work(n, true); | ||||||||||||
| 5624 | } | ||||||||||||
| 5625 | |||||||||||||
| 5626 | void PhaseIdealLoop::build_loop_late_post_work(Node *n, bool pinned) { | ||||||||||||
| 5627 | |||||||||||||
| 5628 | if (n->req() == 2 && (n->Opcode() == Op_ConvI2L || n->Opcode() == Op_CastII) && !C->major_progress() && !_verify_only) { | ||||||||||||
| 5629 | _igvn._worklist.push(n); // Maybe we'll normalize it, if no more loops. | ||||||||||||
| 5630 | } | ||||||||||||
| 5631 | |||||||||||||
| 5632 | #ifdef ASSERT1 | ||||||||||||
| 5633 | if (_verify_only && !n->is_CFG()) { | ||||||||||||
| 5634 | // Check def-use domination. | ||||||||||||
| 5635 | compute_lca_of_uses(n, get_ctrl(n), true /* verify */); | ||||||||||||
| 5636 | } | ||||||||||||
| 5637 | #endif | ||||||||||||
| 5638 | |||||||||||||
| 5639 | // CFG and pinned nodes already handled | ||||||||||||
| 5640 | if( n->in(0) ) { | ||||||||||||
| 5641 | if( n->in(0)->is_top() ) return; // Dead? | ||||||||||||
| 5642 | |||||||||||||
| 5643 | // We'd like +VerifyLoopOptimizations to not believe that Mod's/Loads | ||||||||||||
| 5644 | // _must_ be pinned (they have to observe their control edge of course). | ||||||||||||
| 5645 | // Unlike Stores (which modify an unallocable resource, the memory | ||||||||||||
| 5646 | // state), Mods/Loads can float around. So free them up. | ||||||||||||
| 5647 | switch( n->Opcode() ) { | ||||||||||||
| 5648 | case Op_DivI: | ||||||||||||
| 5649 | case Op_DivF: | ||||||||||||
| 5650 | case Op_DivD: | ||||||||||||
| 5651 | case Op_ModI: | ||||||||||||
| 5652 | case Op_ModF: | ||||||||||||
| 5653 | case Op_ModD: | ||||||||||||
| 5654 | case Op_LoadB: // Same with Loads; they can sink | ||||||||||||
| 5655 | case Op_LoadUB: // during loop optimizations. | ||||||||||||
| 5656 | case Op_LoadUS: | ||||||||||||
| 5657 | case Op_LoadD: | ||||||||||||
| 5658 | case Op_LoadF: | ||||||||||||
| 5659 | case Op_LoadI: | ||||||||||||
| 5660 | case Op_LoadKlass: | ||||||||||||
| 5661 | case Op_LoadNKlass: | ||||||||||||
| 5662 | case Op_LoadL: | ||||||||||||
| 5663 | case Op_LoadS: | ||||||||||||
| 5664 | case Op_LoadP: | ||||||||||||
| 5665 | case Op_LoadN: | ||||||||||||
| 5666 | case Op_LoadRange: | ||||||||||||
| 5667 | case Op_LoadD_unaligned: | ||||||||||||
| 5668 | case Op_LoadL_unaligned: | ||||||||||||
| 5669 | case Op_StrComp: // Does a bunch of load-like effects | ||||||||||||
| 5670 | case Op_StrEquals: | ||||||||||||
| 5671 | case Op_StrIndexOf: | ||||||||||||
| 5672 | case Op_StrIndexOfChar: | ||||||||||||
| 5673 | case Op_AryEq: | ||||||||||||
| 5674 | case Op_HasNegatives: | ||||||||||||
| 5675 | pinned = false; | ||||||||||||
| 5676 | } | ||||||||||||
| 5677 | if (n->is_CMove() || n->is_ConstraintCast()) { | ||||||||||||
| 5678 | pinned = false; | ||||||||||||
| 5679 | } | ||||||||||||
| 5680 | if( pinned ) { | ||||||||||||
| 5681 | IdealLoopTree *chosen_loop = get_loop(n->is_CFG() ? n : get_ctrl(n)); | ||||||||||||
| 5682 | if( !chosen_loop->_child ) // Inner loop? | ||||||||||||
| 5683 | chosen_loop->_body.push(n); // Collect inner loops | ||||||||||||
| 5684 | return; | ||||||||||||
| 5685 | } | ||||||||||||
| 5686 | } else { // No slot zero | ||||||||||||
| 5687 | if( n->is_CFG() ) { // CFG with no slot 0 is dead | ||||||||||||
| 5688 | _nodes.map(n->_idx,0); // No block setting, it's globally dead | ||||||||||||
| 5689 | return; | ||||||||||||
| 5690 | } | ||||||||||||
| 5691 | assert(!n->is_CFG() || n->outcnt() == 0, "")do { if (!(!n->is_CFG() || n->outcnt() == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5691, "assert(" "!n->is_CFG() || n->outcnt() == 0" ") failed" , ""); ::breakpoint(); } } while (0); | ||||||||||||
| 5692 | } | ||||||||||||
| 5693 | |||||||||||||
| 5694 | // Do I have a "safe range" I can select over? | ||||||||||||
| 5695 | Node *early = get_ctrl(n);// Early location already computed | ||||||||||||
| 5696 | |||||||||||||
| 5697 | // Compute latest point this Node can go | ||||||||||||
| 5698 | Node *LCA = get_late_ctrl( n, early ); | ||||||||||||
| 5699 | // LCA is NULL due to uses being dead | ||||||||||||
| 5700 | if( LCA == NULL__null ) { | ||||||||||||
| 5701 | #ifdef ASSERT1 | ||||||||||||
| 5702 | for (DUIterator i1 = n->outs(); n->has_out(i1); i1++) { | ||||||||||||
| 5703 | assert( _nodes[n->out(i1)->_idx] == NULL, "all uses must also be dead")do { if (!(_nodes[n->out(i1)->_idx] == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5703, "assert(" "_nodes[n->out(i1)->_idx] == __null" ") failed" , "all uses must also be dead"); ::breakpoint(); } } while (0 ); | ||||||||||||
| 5704 | } | ||||||||||||
| 5705 | #endif | ||||||||||||
| 5706 | _nodes.map(n->_idx, 0); // This node is useless | ||||||||||||
| 5707 | _deadlist.push(n); | ||||||||||||
| 5708 | return; | ||||||||||||
| 5709 | } | ||||||||||||
| 5710 | assert(LCA != NULL && !LCA->is_top(), "no dead nodes")do { if (!(LCA != __null && !LCA->is_top())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5710, "assert(" "LCA != __null && !LCA->is_top()" ") failed", "no dead nodes"); ::breakpoint(); } } while (0); | ||||||||||||
| 5711 | |||||||||||||
| 5712 | Node *legal = LCA; // Walk 'legal' up the IDOM chain | ||||||||||||
| 5713 | Node *least = legal; // Best legal position so far | ||||||||||||
| 5714 | while( early != legal ) { // While not at earliest legal | ||||||||||||
| 5715 | #ifdef ASSERT1 | ||||||||||||
| 5716 | if (legal->is_Start() && !early->is_Root()) { | ||||||||||||
| 5717 | // Bad graph. Print idom path and fail. | ||||||||||||
| 5718 | dump_bad_graph("Bad graph detected in build_loop_late", n, early, LCA); | ||||||||||||
| 5719 | assert(false, "Bad graph detected in build_loop_late")do { if (!(false)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5719, "assert(" "false" ") failed", "Bad graph detected in build_loop_late" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5720 | } | ||||||||||||
| 5721 | #endif | ||||||||||||
| 5722 | // Find least loop nesting depth | ||||||||||||
| 5723 | legal = idom(legal); // Bump up the IDOM tree | ||||||||||||
| 5724 | // Check for lower nesting depth | ||||||||||||
| 5725 | if( get_loop(legal)->_nest < get_loop(least)->_nest ) | ||||||||||||
| 5726 | least = legal; | ||||||||||||
| 5727 | } | ||||||||||||
| 5728 | assert(early == legal || legal != C->root(), "bad dominance of inputs")do { if (!(early == legal || legal != C->root())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5728, "assert(" "early == legal || legal != C->root()" ") failed" , "bad dominance of inputs"); ::breakpoint(); } } while (0); | ||||||||||||
| 5729 | |||||||||||||
| 5730 | // Try not to place code on a loop entry projection | ||||||||||||
| 5731 | // which can inhibit range check elimination. | ||||||||||||
| 5732 | if (least != early) { | ||||||||||||
| 5733 | Node* ctrl_out = least->unique_ctrl_out(); | ||||||||||||
| 5734 | if (ctrl_out && ctrl_out->is_Loop() && | ||||||||||||
| 5735 | least == ctrl_out->in(LoopNode::EntryControl)) { | ||||||||||||
| 5736 | // Move the node above predicates as far up as possible so a | ||||||||||||
| 5737 | // following pass of loop predication doesn't hoist a predicate | ||||||||||||
| 5738 | // that depends on it above that node. | ||||||||||||
| 5739 | Node* new_ctrl = least; | ||||||||||||
| 5740 | for (;;) { | ||||||||||||
| 5741 | if (!new_ctrl->is_Proj()) { | ||||||||||||
| 5742 | break; | ||||||||||||
| 5743 | } | ||||||||||||
| 5744 | CallStaticJavaNode* call = new_ctrl->as_Proj()->is_uncommon_trap_if_pattern(Deoptimization::Reason_none); | ||||||||||||
| 5745 | if (call == NULL__null) { | ||||||||||||
| 5746 | break; | ||||||||||||
| 5747 | } | ||||||||||||
| 5748 | int req = call->uncommon_trap_request(); | ||||||||||||
| 5749 | Deoptimization::DeoptReason trap_reason = Deoptimization::trap_request_reason(req); | ||||||||||||
| 5750 | if (trap_reason != Deoptimization::Reason_loop_limit_check && | ||||||||||||
| 5751 | trap_reason != Deoptimization::Reason_predicate && | ||||||||||||
| 5752 | trap_reason != Deoptimization::Reason_profile_predicate) { | ||||||||||||
| 5753 | break; | ||||||||||||
| 5754 | } | ||||||||||||
| 5755 | Node* c = new_ctrl->in(0)->in(0); | ||||||||||||
| 5756 | if (is_dominator(c, early) && c != early) { | ||||||||||||
| 5757 | break; | ||||||||||||
| 5758 | } | ||||||||||||
| 5759 | new_ctrl = c; | ||||||||||||
| 5760 | } | ||||||||||||
| 5761 | least = new_ctrl; | ||||||||||||
| 5762 | } | ||||||||||||
| 5763 | } | ||||||||||||
| 5764 | |||||||||||||
| 5765 | #ifdef ASSERT1 | ||||||||||||
| 5766 | // If verifying, verify that 'verify_me' has a legal location | ||||||||||||
| 5767 | // and choose it as our location. | ||||||||||||
| 5768 | if( _verify_me ) { | ||||||||||||
| 5769 | Node *v_ctrl = _verify_me->get_ctrl_no_update(n); | ||||||||||||
| 5770 | Node *legal = LCA; | ||||||||||||
| 5771 | while( early != legal ) { // While not at earliest legal | ||||||||||||
| 5772 | if( legal == v_ctrl ) break; // Check for prior good location | ||||||||||||
| 5773 | legal = idom(legal) ;// Bump up the IDOM tree | ||||||||||||
| 5774 | } | ||||||||||||
| 5775 | // Check for prior good location | ||||||||||||
| 5776 | if( legal == v_ctrl ) least = legal; // Keep prior if found | ||||||||||||
| 5777 | } | ||||||||||||
| 5778 | #endif | ||||||||||||
| 5779 | |||||||||||||
| 5780 | // Assign discovered "here or above" point | ||||||||||||
| 5781 | least = find_non_split_ctrl(least); | ||||||||||||
| 5782 | verify_strip_mined_scheduling(n, least); | ||||||||||||
| 5783 | set_ctrl(n, least); | ||||||||||||
| 5784 | |||||||||||||
| 5785 | // Collect inner loop bodies | ||||||||||||
| 5786 | IdealLoopTree *chosen_loop = get_loop(least); | ||||||||||||
| 5787 | if( !chosen_loop->_child ) // Inner loop? | ||||||||||||
| 5788 | chosen_loop->_body.push(n);// Collect inner loops | ||||||||||||
| 5789 | } | ||||||||||||
| 5790 | |||||||||||||
| 5791 | #ifdef ASSERT1 | ||||||||||||
| 5792 | void PhaseIdealLoop::dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA) { | ||||||||||||
| 5793 | tty->print_cr("%s", msg); | ||||||||||||
| 5794 | tty->print("n: "); n->dump(); | ||||||||||||
| 5795 | tty->print("early(n): "); early->dump(); | ||||||||||||
| 5796 | if (n->in(0) != NULL__null && !n->in(0)->is_top() && | ||||||||||||
| 5797 | n->in(0) != early && !n->in(0)->is_Root()) { | ||||||||||||
| 5798 | tty->print("n->in(0): "); n->in(0)->dump(); | ||||||||||||
| 5799 | } | ||||||||||||
| 5800 | for (uint i = 1; i < n->req(); i++) { | ||||||||||||
| 5801 | Node* in1 = n->in(i); | ||||||||||||
| 5802 | if (in1 != NULL__null && in1 != n && !in1->is_top()) { | ||||||||||||
| 5803 | tty->print("n->in(%d): ", i); in1->dump(); | ||||||||||||
| 5804 | Node* in1_early = get_ctrl(in1); | ||||||||||||
| 5805 | tty->print("early(n->in(%d)): ", i); in1_early->dump(); | ||||||||||||
| 5806 | if (in1->in(0) != NULL__null && !in1->in(0)->is_top() && | ||||||||||||
| 5807 | in1->in(0) != in1_early && !in1->in(0)->is_Root()) { | ||||||||||||
| 5808 | tty->print("n->in(%d)->in(0): ", i); in1->in(0)->dump(); | ||||||||||||
| 5809 | } | ||||||||||||
| 5810 | for (uint j = 1; j < in1->req(); j++) { | ||||||||||||
| 5811 | Node* in2 = in1->in(j); | ||||||||||||
| 5812 | if (in2 != NULL__null && in2 != n && in2 != in1 && !in2->is_top()) { | ||||||||||||
| 5813 | tty->print("n->in(%d)->in(%d): ", i, j); in2->dump(); | ||||||||||||
| 5814 | Node* in2_early = get_ctrl(in2); | ||||||||||||
| 5815 | tty->print("early(n->in(%d)->in(%d)): ", i, j); in2_early->dump(); | ||||||||||||
| 5816 | if (in2->in(0) != NULL__null && !in2->in(0)->is_top() && | ||||||||||||
| 5817 | in2->in(0) != in2_early && !in2->in(0)->is_Root()) { | ||||||||||||
| 5818 | tty->print("n->in(%d)->in(%d)->in(0): ", i, j); in2->in(0)->dump(); | ||||||||||||
| 5819 | } | ||||||||||||
| 5820 | } | ||||||||||||
| 5821 | } | ||||||||||||
| 5822 | } | ||||||||||||
| 5823 | } | ||||||||||||
| 5824 | tty->cr(); | ||||||||||||
| 5825 | tty->print("LCA(n): "); LCA->dump(); | ||||||||||||
| 5826 | for (uint i = 0; i < n->outcnt(); i++) { | ||||||||||||
| 5827 | Node* u1 = n->raw_out(i); | ||||||||||||
| 5828 | if (u1 == n) | ||||||||||||
| 5829 | continue; | ||||||||||||
| 5830 | tty->print("n->out(%d): ", i); u1->dump(); | ||||||||||||
| 5831 | if (u1->is_CFG()) { | ||||||||||||
| 5832 | for (uint j = 0; j < u1->outcnt(); j++) { | ||||||||||||
| 5833 | Node* u2 = u1->raw_out(j); | ||||||||||||
| 5834 | if (u2 != u1 && u2 != n && u2->is_CFG()) { | ||||||||||||
| 5835 | tty->print("n->out(%d)->out(%d): ", i, j); u2->dump(); | ||||||||||||
| 5836 | } | ||||||||||||
| 5837 | } | ||||||||||||
| 5838 | } else { | ||||||||||||
| 5839 | Node* u1_later = get_ctrl(u1); | ||||||||||||
| 5840 | tty->print("later(n->out(%d)): ", i); u1_later->dump(); | ||||||||||||
| 5841 | if (u1->in(0) != NULL__null && !u1->in(0)->is_top() && | ||||||||||||
| 5842 | u1->in(0) != u1_later && !u1->in(0)->is_Root()) { | ||||||||||||
| 5843 | tty->print("n->out(%d)->in(0): ", i); u1->in(0)->dump(); | ||||||||||||
| 5844 | } | ||||||||||||
| 5845 | for (uint j = 0; j < u1->outcnt(); j++) { | ||||||||||||
| 5846 | Node* u2 = u1->raw_out(j); | ||||||||||||
| 5847 | if (u2 == n || u2 == u1) | ||||||||||||
| 5848 | continue; | ||||||||||||
| 5849 | tty->print("n->out(%d)->out(%d): ", i, j); u2->dump(); | ||||||||||||
| 5850 | if (!u2->is_CFG()) { | ||||||||||||
| 5851 | Node* u2_later = get_ctrl(u2); | ||||||||||||
| 5852 | tty->print("later(n->out(%d)->out(%d)): ", i, j); u2_later->dump(); | ||||||||||||
| 5853 | if (u2->in(0) != NULL__null && !u2->in(0)->is_top() && | ||||||||||||
| 5854 | u2->in(0) != u2_later && !u2->in(0)->is_Root()) { | ||||||||||||
| 5855 | tty->print("n->out(%d)->in(0): ", i); u2->in(0)->dump(); | ||||||||||||
| 5856 | } | ||||||||||||
| 5857 | } | ||||||||||||
| 5858 | } | ||||||||||||
| 5859 | } | ||||||||||||
| 5860 | } | ||||||||||||
| 5861 | tty->cr(); | ||||||||||||
| 5862 | tty->print_cr("idoms of early %d:", early->_idx); | ||||||||||||
| 5863 | dump_idom(early); | ||||||||||||
| 5864 | tty->cr(); | ||||||||||||
| 5865 | tty->print_cr("idoms of (wrong) LCA %d:", LCA->_idx); | ||||||||||||
| 5866 | dump_idom(LCA); | ||||||||||||
| 5867 | tty->cr(); | ||||||||||||
| 5868 | dump_real_LCA(early, LCA); | ||||||||||||
| 5869 | tty->cr(); | ||||||||||||
| 5870 | } | ||||||||||||
| 5871 | |||||||||||||
| 5872 | // Find the real LCA of early and the wrongly assumed LCA. | ||||||||||||
| 5873 | void PhaseIdealLoop::dump_real_LCA(Node* early, Node* wrong_lca) { | ||||||||||||
| 5874 | assert(!is_dominator(early, wrong_lca) && !is_dominator(early, wrong_lca),do { if (!(!is_dominator(early, wrong_lca) && !is_dominator (early, wrong_lca))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5875, "assert(" "!is_dominator(early, wrong_lca) && !is_dominator(early, wrong_lca)" ") failed", "sanity check that one node does not dominate the other" ); ::breakpoint(); } } while (0) | ||||||||||||
| 5875 | "sanity check that one node does not dominate the other")do { if (!(!is_dominator(early, wrong_lca) && !is_dominator (early, wrong_lca))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5875, "assert(" "!is_dominator(early, wrong_lca) && !is_dominator(early, wrong_lca)" ") failed", "sanity check that one node does not dominate the other" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5876 | assert(!has_ctrl(early) && !has_ctrl(wrong_lca), "sanity check, no data nodes")do { if (!(!has_ctrl(early) && !has_ctrl(wrong_lca))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5876, "assert(" "!has_ctrl(early) && !has_ctrl(wrong_lca)" ") failed", "sanity check, no data nodes"); ::breakpoint(); } } while (0); | ||||||||||||
| 5877 | |||||||||||||
| 5878 | ResourceMark rm; | ||||||||||||
| 5879 | Node_List nodes_seen; | ||||||||||||
| 5880 | Node* real_LCA = NULL__null; | ||||||||||||
| 5881 | Node* n1 = wrong_lca; | ||||||||||||
| 5882 | Node* n2 = early; | ||||||||||||
| 5883 | uint count_1 = 0; | ||||||||||||
| 5884 | uint count_2 = 0; | ||||||||||||
| 5885 | // Add early and wrong_lca to simplify calculation of idom indices | ||||||||||||
| 5886 | nodes_seen.push(n1); | ||||||||||||
| 5887 | nodes_seen.push(n2); | ||||||||||||
| 5888 | |||||||||||||
| 5889 | // Walk the idom chain up from early and wrong_lca and stop when they intersect. | ||||||||||||
| 5890 | while (!n1->is_Start() && !n2->is_Start()) { | ||||||||||||
| 5891 | n1 = idom(n1); | ||||||||||||
| 5892 | n2 = idom(n2); | ||||||||||||
| 5893 | if (n1 == n2) { | ||||||||||||
| 5894 | // Both idom chains intersect at the same index | ||||||||||||
| 5895 | real_LCA = n1; | ||||||||||||
| 5896 | count_1 = nodes_seen.size() / 2; | ||||||||||||
| 5897 | count_2 = count_1; | ||||||||||||
| 5898 | break; | ||||||||||||
| 5899 | } | ||||||||||||
| 5900 | if (check_idom_chains_intersection(n1, count_1, count_2, &nodes_seen)) { | ||||||||||||
| 5901 | real_LCA = n1; | ||||||||||||
| 5902 | break; | ||||||||||||
| 5903 | } | ||||||||||||
| 5904 | if (check_idom_chains_intersection(n2, count_2, count_1, &nodes_seen)) { | ||||||||||||
| 5905 | real_LCA = n2; | ||||||||||||
| 5906 | break; | ||||||||||||
| 5907 | } | ||||||||||||
| 5908 | nodes_seen.push(n1); | ||||||||||||
| 5909 | nodes_seen.push(n2); | ||||||||||||
| 5910 | } | ||||||||||||
| 5911 | |||||||||||||
| 5912 | assert(real_LCA != NULL, "must always find an LCA")do { if (!(real_LCA != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 5912, "assert(" "real_LCA != __null" ") failed", "must always find an LCA" ); ::breakpoint(); } } while (0); | ||||||||||||
| 5913 | tty->print_cr("Real LCA of early %d (idom[%d]) and (wrong) LCA %d (idom[%d]):", early->_idx, count_2, wrong_lca->_idx, count_1); | ||||||||||||
| 5914 | real_LCA->dump(); | ||||||||||||
| 5915 | } | ||||||||||||
| 5916 | |||||||||||||
| 5917 | // Check if n is already on nodes_seen (i.e. idom chains of early and wrong_lca intersect at n). Determine the idom index of n | ||||||||||||
| 5918 | // on both idom chains and return them in idom_idx_new and idom_idx_other, respectively. | ||||||||||||
| 5919 | bool PhaseIdealLoop::check_idom_chains_intersection(const Node* n, uint& idom_idx_new, uint& idom_idx_other, const Node_List* nodes_seen) const { | ||||||||||||
| 5920 | if (nodes_seen->contains(n)) { | ||||||||||||
| 5921 | // The idom chain has just discovered n. | ||||||||||||
| 5922 | // Divide by 2 because nodes_seen contains the same amount of nodes from both chains. | ||||||||||||
| 5923 | idom_idx_new = nodes_seen->size() / 2; | ||||||||||||
| 5924 | |||||||||||||
| 5925 | // The other chain already contained n. Search the index. | ||||||||||||
| 5926 | for (uint i = 0; i < nodes_seen->size(); i++) { | ||||||||||||
| 5927 | if (nodes_seen->at(i) == n) { | ||||||||||||
| 5928 | // Divide by 2 because nodes_seen contains the same amount of nodes from both chains. | ||||||||||||
| 5929 | idom_idx_other = i / 2; | ||||||||||||
| 5930 | } | ||||||||||||
| 5931 | } | ||||||||||||
| 5932 | return true; | ||||||||||||
| 5933 | } | ||||||||||||
| 5934 | return false; | ||||||||||||
| 5935 | } | ||||||||||||
| 5936 | #endif // ASSERT | ||||||||||||
| 5937 | |||||||||||||
| 5938 | #ifndef PRODUCT | ||||||||||||
| 5939 | //------------------------------dump------------------------------------------- | ||||||||||||
| 5940 | void PhaseIdealLoop::dump() const { | ||||||||||||
| 5941 | ResourceMark rm; | ||||||||||||
| 5942 | Node_Stack stack(C->live_nodes() >> 2); | ||||||||||||
| 5943 | Node_List rpo_list; | ||||||||||||
| 5944 | VectorSet visited; | ||||||||||||
| 5945 | visited.set(C->top()->_idx); | ||||||||||||
| 5946 | rpo(C->root(), stack, visited, rpo_list); | ||||||||||||
| 5947 | // Dump root loop indexed by last element in PO order | ||||||||||||
| 5948 | dump(_ltree_root, rpo_list.size(), rpo_list); | ||||||||||||
| 5949 | } | ||||||||||||
| 5950 | |||||||||||||
| 5951 | void PhaseIdealLoop::dump(IdealLoopTree* loop, uint idx, Node_List &rpo_list) const { | ||||||||||||
| 5952 | loop->dump_head(); | ||||||||||||
| 5953 | |||||||||||||
| 5954 | // Now scan for CFG nodes in the same loop | ||||||||||||
| 5955 | for (uint j = idx; j > 0; j--) { | ||||||||||||
| 5956 | Node* n = rpo_list[j-1]; | ||||||||||||
| 5957 | if (!_nodes[n->_idx]) // Skip dead nodes | ||||||||||||
| 5958 | continue; | ||||||||||||
| 5959 | |||||||||||||
| 5960 | if (get_loop(n) != loop) { // Wrong loop nest | ||||||||||||
| 5961 | if (get_loop(n)->_head == n && // Found nested loop? | ||||||||||||
| 5962 | get_loop(n)->_parent == loop) | ||||||||||||
| 5963 | dump(get_loop(n), rpo_list.size(), rpo_list); // Print it nested-ly | ||||||||||||
| 5964 | continue; | ||||||||||||
| 5965 | } | ||||||||||||
| 5966 | |||||||||||||
| 5967 | // Dump controlling node | ||||||||||||
| 5968 | tty->sp(2 * loop->_nest); | ||||||||||||
| 5969 | tty->print("C"); | ||||||||||||
| 5970 | if (n == C->root()) { | ||||||||||||
| 5971 | n->dump(); | ||||||||||||
| 5972 | } else { | ||||||||||||
| 5973 | Node* cached_idom = idom_no_update(n); | ||||||||||||
| 5974 | Node* computed_idom = n->in(0); | ||||||||||||
| 5975 | if (n->is_Region()) { | ||||||||||||
| 5976 | computed_idom = compute_idom(n); | ||||||||||||
| 5977 | // computed_idom() will return n->in(0) when idom(n) is an IfNode (or | ||||||||||||
| 5978 | // any MultiBranch ctrl node), so apply a similar transform to | ||||||||||||
| 5979 | // the cached idom returned from idom_no_update. | ||||||||||||
| 5980 | cached_idom = find_non_split_ctrl(cached_idom); | ||||||||||||
| 5981 | } | ||||||||||||
| 5982 | tty->print(" ID:%d", computed_idom->_idx); | ||||||||||||
| 5983 | n->dump(); | ||||||||||||
| 5984 | if (cached_idom != computed_idom) { | ||||||||||||
| 5985 | tty->print_cr("*** BROKEN IDOM! Computed as: %d, cached as: %d", | ||||||||||||
| 5986 | computed_idom->_idx, cached_idom->_idx); | ||||||||||||
| 5987 | } | ||||||||||||
| 5988 | } | ||||||||||||
| 5989 | // Dump nodes it controls | ||||||||||||
| 5990 | for (uint k = 0; k < _nodes.Size(); k++) { | ||||||||||||
| 5991 | // (k < C->unique() && get_ctrl(find(k)) == n) | ||||||||||||
| 5992 | if (k < C->unique() && _nodes[k] == (Node*)((intptr_t)n + 1)) { | ||||||||||||
| 5993 | Node* m = C->root()->find(k); | ||||||||||||
| 5994 | if (m && m->outcnt() > 0) { | ||||||||||||
| 5995 | if (!(has_ctrl(m) && get_ctrl_no_update(m) == n)) { | ||||||||||||
| 5996 | tty->print_cr("*** BROKEN CTRL ACCESSOR! _nodes[k] is %p, ctrl is %p", | ||||||||||||
| 5997 | _nodes[k], has_ctrl(m) ? get_ctrl_no_update(m) : NULL__null); | ||||||||||||
| 5998 | } | ||||||||||||
| 5999 | tty->sp(2 * loop->_nest + 1); | ||||||||||||
| 6000 | m->dump(); | ||||||||||||
| 6001 | } | ||||||||||||
| 6002 | } | ||||||||||||
| 6003 | } | ||||||||||||
| 6004 | } | ||||||||||||
| 6005 | } | ||||||||||||
| 6006 | |||||||||||||
| 6007 | void PhaseIdealLoop::dump_idom(Node* n) const { | ||||||||||||
| 6008 | if (has_ctrl(n)) { | ||||||||||||
| 6009 | tty->print_cr("No idom for data nodes"); | ||||||||||||
| 6010 | } else { | ||||||||||||
| 6011 | for (int i = 0; i < 100 && !n->is_Start(); i++) { | ||||||||||||
| 6012 | tty->print("idom[%d] ", i); | ||||||||||||
| 6013 | n->dump(); | ||||||||||||
| 6014 | n = idom(n); | ||||||||||||
| 6015 | } | ||||||||||||
| 6016 | } | ||||||||||||
| 6017 | } | ||||||||||||
| 6018 | #endif // NOT PRODUCT | ||||||||||||
| 6019 | |||||||||||||
| 6020 | // Collect a R-P-O for the whole CFG. | ||||||||||||
| 6021 | // Result list is in post-order (scan backwards for RPO) | ||||||||||||
| 6022 | void PhaseIdealLoop::rpo(Node* start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list) const { | ||||||||||||
| 6023 | stk.push(start, 0); | ||||||||||||
| 6024 | visited.set(start->_idx); | ||||||||||||
| 6025 | |||||||||||||
| 6026 | while (stk.is_nonempty()) { | ||||||||||||
| 6027 | Node* m = stk.node(); | ||||||||||||
| 6028 | uint idx = stk.index(); | ||||||||||||
| 6029 | if (idx < m->outcnt()) { | ||||||||||||
| 6030 | stk.set_index(idx + 1); | ||||||||||||
| 6031 | Node* n = m->raw_out(idx); | ||||||||||||
| 6032 | if (n->is_CFG() && !visited.test_set(n->_idx)) { | ||||||||||||
| 6033 | stk.push(n, 0); | ||||||||||||
| 6034 | } | ||||||||||||
| 6035 | } else { | ||||||||||||
| 6036 | rpo_list.push(m); | ||||||||||||
| 6037 | stk.pop(); | ||||||||||||
| 6038 | } | ||||||||||||
| 6039 | } | ||||||||||||
| 6040 | } | ||||||||||||
| 6041 | |||||||||||||
| 6042 | |||||||||||||
| 6043 | //============================================================================= | ||||||||||||
| 6044 | //------------------------------LoopTreeIterator------------------------------- | ||||||||||||
| 6045 | |||||||||||||
| 6046 | // Advance to next loop tree using a preorder, left-to-right traversal. | ||||||||||||
| 6047 | void LoopTreeIterator::next() { | ||||||||||||
| 6048 | assert(!done(), "must not be done.")do { if (!(!done())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 6048, "assert(" "!done()" ") failed", "must not be done."); ::breakpoint(); } } while (0); | ||||||||||||
| 6049 | if (_curnt->_child != NULL__null) { | ||||||||||||
| 6050 | _curnt = _curnt->_child; | ||||||||||||
| 6051 | } else if (_curnt->_next != NULL__null) { | ||||||||||||
| 6052 | _curnt = _curnt->_next; | ||||||||||||
| 6053 | } else { | ||||||||||||
| 6054 | while (_curnt != _root && _curnt->_next == NULL__null) { | ||||||||||||
| 6055 | _curnt = _curnt->_parent; | ||||||||||||
| 6056 | } | ||||||||||||
| 6057 | if (_curnt == _root) { | ||||||||||||
| 6058 | _curnt = NULL__null; | ||||||||||||
| 6059 | assert(done(), "must be done.")do { if (!(done())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 6059, "assert(" "done()" ") failed", "must be done."); ::breakpoint (); } } while (0); | ||||||||||||
| 6060 | } else { | ||||||||||||
| 6061 | assert(_curnt->_next != NULL, "must be more to do")do { if (!(_curnt->_next != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.cpp" , 6061, "assert(" "_curnt->_next != __null" ") failed", "must be more to do" ); ::breakpoint(); } } while (0); | ||||||||||||
| 6062 | _curnt = _curnt->_next; | ||||||||||||
| 6063 | } | ||||||||||||
| 6064 | } | ||||||||||||
| 6065 | } |
| 1 | /* |
| 2 | * Copyright (c) 1997, 2021, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_OPTO_NODE_HPP |
| 26 | #define SHARE_OPTO_NODE_HPP |
| 27 | |
| 28 | #include "libadt/vectset.hpp" |
| 29 | #include "opto/compile.hpp" |
| 30 | #include "opto/type.hpp" |
| 31 | #include "utilities/copy.hpp" |
| 32 | |
| 33 | // Portions of code courtesy of Clifford Click |
| 34 | |
| 35 | // Optimization - Graph Style |
| 36 | |
| 37 | |
| 38 | class AbstractLockNode; |
| 39 | class AddNode; |
| 40 | class AddPNode; |
| 41 | class AliasInfo; |
| 42 | class AllocateArrayNode; |
| 43 | class AllocateNode; |
| 44 | class ArrayCopyNode; |
| 45 | class BaseCountedLoopNode; |
| 46 | class BaseCountedLoopEndNode; |
| 47 | class BlackholeNode; |
| 48 | class Block; |
| 49 | class BoolNode; |
| 50 | class BoxLockNode; |
| 51 | class CMoveNode; |
| 52 | class CallDynamicJavaNode; |
| 53 | class CallJavaNode; |
| 54 | class CallLeafNode; |
| 55 | class CallLeafNoFPNode; |
| 56 | class CallNode; |
| 57 | class CallRuntimeNode; |
| 58 | class CallNativeNode; |
| 59 | class CallStaticJavaNode; |
| 60 | class CastFFNode; |
| 61 | class CastDDNode; |
| 62 | class CastVVNode; |
| 63 | class CastIINode; |
| 64 | class CastLLNode; |
| 65 | class CatchNode; |
| 66 | class CatchProjNode; |
| 67 | class CheckCastPPNode; |
| 68 | class ClearArrayNode; |
| 69 | class CmpNode; |
| 70 | class CodeBuffer; |
| 71 | class ConstraintCastNode; |
| 72 | class ConNode; |
| 73 | class CompareAndSwapNode; |
| 74 | class CompareAndExchangeNode; |
| 75 | class CountedLoopNode; |
| 76 | class CountedLoopEndNode; |
| 77 | class DecodeNarrowPtrNode; |
| 78 | class DecodeNNode; |
| 79 | class DecodeNKlassNode; |
| 80 | class EncodeNarrowPtrNode; |
| 81 | class EncodePNode; |
| 82 | class EncodePKlassNode; |
| 83 | class FastLockNode; |
| 84 | class FastUnlockNode; |
| 85 | class HaltNode; |
| 86 | class IfNode; |
| 87 | class IfProjNode; |
| 88 | class IfFalseNode; |
| 89 | class IfTrueNode; |
| 90 | class InitializeNode; |
| 91 | class JVMState; |
| 92 | class JumpNode; |
| 93 | class JumpProjNode; |
| 94 | class LoadNode; |
| 95 | class LoadStoreNode; |
| 96 | class LoadStoreConditionalNode; |
| 97 | class LockNode; |
| 98 | class LongCountedLoopNode; |
| 99 | class LongCountedLoopEndNode; |
| 100 | class LoopNode; |
| 101 | class LShiftNode; |
| 102 | class MachBranchNode; |
| 103 | class MachCallDynamicJavaNode; |
| 104 | class MachCallJavaNode; |
| 105 | class MachCallLeafNode; |
| 106 | class MachCallNode; |
| 107 | class MachCallNativeNode; |
| 108 | class MachCallRuntimeNode; |
| 109 | class MachCallStaticJavaNode; |
| 110 | class MachConstantBaseNode; |
| 111 | class MachConstantNode; |
| 112 | class MachGotoNode; |
| 113 | class MachIfNode; |
| 114 | class MachJumpNode; |
| 115 | class MachNode; |
| 116 | class MachNullCheckNode; |
| 117 | class MachProjNode; |
| 118 | class MachReturnNode; |
| 119 | class MachSafePointNode; |
| 120 | class MachSpillCopyNode; |
| 121 | class MachTempNode; |
| 122 | class MachMergeNode; |
| 123 | class MachMemBarNode; |
| 124 | class Matcher; |
| 125 | class MemBarNode; |
| 126 | class MemBarStoreStoreNode; |
| 127 | class MemNode; |
| 128 | class MergeMemNode; |
| 129 | class MoveNode; |
| 130 | class MulNode; |
| 131 | class MultiNode; |
| 132 | class MultiBranchNode; |
| 133 | class NeverBranchNode; |
| 134 | class Opaque1Node; |
| 135 | class OuterStripMinedLoopNode; |
| 136 | class OuterStripMinedLoopEndNode; |
| 137 | class Node; |
| 138 | class Node_Array; |
| 139 | class Node_List; |
| 140 | class Node_Stack; |
| 141 | class OopMap; |
| 142 | class ParmNode; |
| 143 | class PCTableNode; |
| 144 | class PhaseCCP; |
| 145 | class PhaseGVN; |
| 146 | class PhaseIterGVN; |
| 147 | class PhaseRegAlloc; |
| 148 | class PhaseTransform; |
| 149 | class PhaseValues; |
| 150 | class PhiNode; |
| 151 | class Pipeline; |
| 152 | class ProjNode; |
| 153 | class RangeCheckNode; |
| 154 | class RegMask; |
| 155 | class RegionNode; |
| 156 | class RootNode; |
| 157 | class SafePointNode; |
| 158 | class SafePointScalarObjectNode; |
| 159 | class StartNode; |
| 160 | class State; |
| 161 | class StoreNode; |
| 162 | class SubNode; |
| 163 | class SubTypeCheckNode; |
| 164 | class Type; |
| 165 | class TypeNode; |
| 166 | class UnlockNode; |
| 167 | class VectorNode; |
| 168 | class LoadVectorNode; |
| 169 | class LoadVectorMaskedNode; |
| 170 | class StoreVectorMaskedNode; |
| 171 | class LoadVectorGatherNode; |
| 172 | class StoreVectorNode; |
| 173 | class StoreVectorScatterNode; |
| 174 | class VectorMaskCmpNode; |
| 175 | class VectorUnboxNode; |
| 176 | class VectorSet; |
| 177 | class VectorReinterpretNode; |
| 178 | class ShiftVNode; |
| 179 | |
| 180 | // The type of all node counts and indexes. |
| 181 | // It must hold at least 16 bits, but must also be fast to load and store. |
| 182 | // This type, if less than 32 bits, could limit the number of possible nodes. |
| 183 | // (To make this type platform-specific, move to globalDefinitions_xxx.hpp.) |
| 184 | typedef unsigned int node_idx_t; |
| 185 | |
| 186 | |
| 187 | #ifndef OPTO_DU_ITERATOR_ASSERT1 |
| 188 | #ifdef ASSERT1 |
| 189 | #define OPTO_DU_ITERATOR_ASSERT1 1 |
| 190 | #else |
| 191 | #define OPTO_DU_ITERATOR_ASSERT1 0 |
| 192 | #endif |
| 193 | #endif //OPTO_DU_ITERATOR_ASSERT |
| 194 | |
| 195 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 196 | class DUIterator; |
| 197 | class DUIterator_Fast; |
| 198 | class DUIterator_Last; |
| 199 | #else |
| 200 | typedef uint DUIterator; |
| 201 | typedef Node** DUIterator_Fast; |
| 202 | typedef Node** DUIterator_Last; |
| 203 | #endif |
| 204 | |
| 205 | // Node Sentinel |
| 206 | #define NodeSentinel(Node*)-1 (Node*)-1 |
| 207 | |
| 208 | // Unknown count frequency |
| 209 | #define COUNT_UNKNOWN(-1.0f) (-1.0f) |
| 210 | |
| 211 | //------------------------------Node------------------------------------------- |
| 212 | // Nodes define actions in the program. They create values, which have types. |
| 213 | // They are both vertices in a directed graph and program primitives. Nodes |
| 214 | // are labeled; the label is the "opcode", the primitive function in the lambda |
| 215 | // calculus sense that gives meaning to the Node. Node inputs are ordered (so |
| 216 | // that "a-b" is different from "b-a"). The inputs to a Node are the inputs to |
| 217 | // the Node's function. These inputs also define a Type equation for the Node. |
| 218 | // Solving these Type equations amounts to doing dataflow analysis. |
| 219 | // Control and data are uniformly represented in the graph. Finally, Nodes |
| 220 | // have a unique dense integer index which is used to index into side arrays |
| 221 | // whenever I have phase-specific information. |
| 222 | |
| 223 | class Node { |
| 224 | friend class VMStructs; |
| 225 | |
| 226 | // Lots of restrictions on cloning Nodes |
| 227 | NONCOPYABLE(Node)Node(Node const&) = delete; Node& operator=(Node const &) = delete; |
| 228 | |
| 229 | public: |
| 230 | friend class Compile; |
| 231 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 232 | friend class DUIterator_Common; |
| 233 | friend class DUIterator; |
| 234 | friend class DUIterator_Fast; |
| 235 | friend class DUIterator_Last; |
| 236 | #endif |
| 237 | |
| 238 | // Because Nodes come and go, I define an Arena of Node structures to pull |
| 239 | // from. This should allow fast access to node creation & deletion. This |
| 240 | // field is a local cache of a value defined in some "program fragment" for |
| 241 | // which these Nodes are just a part of. |
| 242 | |
| 243 | inline void* operator new(size_t x) throw() { |
| 244 | Compile* C = Compile::current(); |
| 245 | Node* n = (Node*)C->node_arena()->AmallocWords(x); |
| 246 | return (void*)n; |
| 247 | } |
| 248 | |
| 249 | // Delete is a NOP |
| 250 | void operator delete( void *ptr ) {} |
| 251 | // Fancy destructor; eagerly attempt to reclaim Node numberings and storage |
| 252 | void destruct(PhaseValues* phase); |
| 253 | |
| 254 | // Create a new Node. Required is the number is of inputs required for |
| 255 | // semantic correctness. |
| 256 | Node( uint required ); |
| 257 | |
| 258 | // Create a new Node with given input edges. |
| 259 | // This version requires use of the "edge-count" new. |
| 260 | // E.g. new (C,3) FooNode( C, NULL, left, right ); |
| 261 | Node( Node *n0 ); |
| 262 | Node( Node *n0, Node *n1 ); |
| 263 | Node( Node *n0, Node *n1, Node *n2 ); |
| 264 | Node( Node *n0, Node *n1, Node *n2, Node *n3 ); |
| 265 | Node( Node *n0, Node *n1, Node *n2, Node *n3, Node *n4 ); |
| 266 | Node( Node *n0, Node *n1, Node *n2, Node *n3, Node *n4, Node *n5 ); |
| 267 | Node( Node *n0, Node *n1, Node *n2, Node *n3, |
| 268 | Node *n4, Node *n5, Node *n6 ); |
| 269 | |
| 270 | // Clone an inherited Node given only the base Node type. |
| 271 | Node* clone() const; |
| 272 | |
| 273 | // Clone a Node, immediately supplying one or two new edges. |
| 274 | // The first and second arguments, if non-null, replace in(1) and in(2), |
| 275 | // respectively. |
| 276 | Node* clone_with_data_edge(Node* in1, Node* in2 = NULL__null) const { |
| 277 | Node* nn = clone(); |
| 278 | if (in1 != NULL__null) nn->set_req(1, in1); |
| 279 | if (in2 != NULL__null) nn->set_req(2, in2); |
| 280 | return nn; |
| 281 | } |
| 282 | |
| 283 | private: |
| 284 | // Shared setup for the above constructors. |
| 285 | // Handles all interactions with Compile::current. |
| 286 | // Puts initial values in all Node fields except _idx. |
| 287 | // Returns the initial value for _idx, which cannot |
| 288 | // be initialized by assignment. |
| 289 | inline int Init(int req); |
| 290 | |
| 291 | //----------------- input edge handling |
| 292 | protected: |
| 293 | friend class PhaseCFG; // Access to address of _in array elements |
| 294 | Node **_in; // Array of use-def references to Nodes |
| 295 | Node **_out; // Array of def-use references to Nodes |
| 296 | |
| 297 | // Input edges are split into two categories. Required edges are required |
| 298 | // for semantic correctness; order is important and NULLs are allowed. |
| 299 | // Precedence edges are used to help determine execution order and are |
| 300 | // added, e.g., for scheduling purposes. They are unordered and not |
| 301 | // duplicated; they have no embedded NULLs. Edges from 0 to _cnt-1 |
| 302 | // are required, from _cnt to _max-1 are precedence edges. |
| 303 | node_idx_t _cnt; // Total number of required Node inputs. |
| 304 | |
| 305 | node_idx_t _max; // Actual length of input array. |
| 306 | |
| 307 | // Output edges are an unordered list of def-use edges which exactly |
| 308 | // correspond to required input edges which point from other nodes |
| 309 | // to this one. Thus the count of the output edges is the number of |
| 310 | // users of this node. |
| 311 | node_idx_t _outcnt; // Total number of Node outputs. |
| 312 | |
| 313 | node_idx_t _outmax; // Actual length of output array. |
| 314 | |
| 315 | // Grow the actual input array to the next larger power-of-2 bigger than len. |
| 316 | void grow( uint len ); |
| 317 | // Grow the output array to the next larger power-of-2 bigger than len. |
| 318 | void out_grow( uint len ); |
| 319 | |
| 320 | public: |
| 321 | // Each Node is assigned a unique small/dense number. This number is used |
| 322 | // to index into auxiliary arrays of data and bit vectors. |
| 323 | // The field _idx is declared constant to defend against inadvertent assignments, |
| 324 | // since it is used by clients as a naked field. However, the field's value can be |
| 325 | // changed using the set_idx() method. |
| 326 | // |
| 327 | // The PhaseRenumberLive phase renumbers nodes based on liveness information. |
| 328 | // Therefore, it updates the value of the _idx field. The parse-time _idx is |
| 329 | // preserved in _parse_idx. |
| 330 | const node_idx_t _idx; |
| 331 | DEBUG_ONLY(const node_idx_t _parse_idx;)const node_idx_t _parse_idx; |
| 332 | // IGV node identifier. Two nodes, possibly in different compilation phases, |
| 333 | // have the same IGV identifier if (and only if) they are the very same node |
| 334 | // (same memory address) or one is "derived" from the other (by e.g. |
| 335 | // renumbering or matching). This identifier makes it possible to follow the |
| 336 | // entire lifetime of a node in IGV even if its C2 identifier (_idx) changes. |
| 337 | NOT_PRODUCT(node_idx_t _igv_idx;)node_idx_t _igv_idx; |
| 338 | |
| 339 | // Get the (read-only) number of input edges |
| 340 | uint req() const { return _cnt; } |
| 341 | uint len() const { return _max; } |
| 342 | // Get the (read-only) number of output edges |
| 343 | uint outcnt() const { return _outcnt; } |
| 344 | |
| 345 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 346 | // Iterate over the out-edges of this node. Deletions are illegal. |
| 347 | inline DUIterator outs() const; |
| 348 | // Use this when the out array might have changed to suppress asserts. |
| 349 | inline DUIterator& refresh_out_pos(DUIterator& i) const; |
| 350 | // Does the node have an out at this position? (Used for iteration.) |
| 351 | inline bool has_out(DUIterator& i) const; |
| 352 | inline Node* out(DUIterator& i) const; |
| 353 | // Iterate over the out-edges of this node. All changes are illegal. |
| 354 | inline DUIterator_Fast fast_outs(DUIterator_Fast& max) const; |
| 355 | inline Node* fast_out(DUIterator_Fast& i) const; |
| 356 | // Iterate over the out-edges of this node, deleting one at a time. |
| 357 | inline DUIterator_Last last_outs(DUIterator_Last& min) const; |
| 358 | inline Node* last_out(DUIterator_Last& i) const; |
| 359 | // The inline bodies of all these methods are after the iterator definitions. |
| 360 | #else |
| 361 | // Iterate over the out-edges of this node. Deletions are illegal. |
| 362 | // This iteration uses integral indexes, to decouple from array reallocations. |
| 363 | DUIterator outs() const { return 0; } |
| 364 | // Use this when the out array might have changed to suppress asserts. |
| 365 | DUIterator refresh_out_pos(DUIterator i) const { return i; } |
| 366 | |
| 367 | // Reference to the i'th output Node. Error if out of bounds. |
| 368 | Node* out(DUIterator i) const { assert(i < _outcnt, "oob")do { if (!(i < _outcnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 368, "assert(" "i < _outcnt" ") failed", "oob"); ::breakpoint (); } } while (0); return _out[i]; } |
| 369 | // Does the node have an out at this position? (Used for iteration.) |
| 370 | bool has_out(DUIterator i) const { return i < _outcnt; } |
| 371 | |
| 372 | // Iterate over the out-edges of this node. All changes are illegal. |
| 373 | // This iteration uses a pointer internal to the out array. |
| 374 | DUIterator_Fast fast_outs(DUIterator_Fast& max) const { |
| 375 | Node** out = _out; |
| 376 | // Assign a limit pointer to the reference argument: |
| 377 | max = out + (ptrdiff_t)_outcnt; |
| 378 | // Return the base pointer: |
| 379 | return out; |
| 380 | } |
| 381 | Node* fast_out(DUIterator_Fast i) const { return *i; } |
| 382 | // Iterate over the out-edges of this node, deleting one at a time. |
| 383 | // This iteration uses a pointer internal to the out array. |
| 384 | DUIterator_Last last_outs(DUIterator_Last& min) const { |
| 385 | Node** out = _out; |
| 386 | // Assign a limit pointer to the reference argument: |
| 387 | min = out; |
| 388 | // Return the pointer to the start of the iteration: |
| 389 | return out + (ptrdiff_t)_outcnt - 1; |
| 390 | } |
| 391 | Node* last_out(DUIterator_Last i) const { return *i; } |
| 392 | #endif |
| 393 | |
| 394 | // Reference to the i'th input Node. Error if out of bounds. |
| 395 | Node* in(uint i) const { assert(i < _max, "oob: i=%d, _max=%d", i, _max)do { if (!(i < _max)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 395, "assert(" "i < _max" ") failed", "oob: i=%d, _max=%d" , i, _max); ::breakpoint(); } } while (0); return _in[i]; } |
| 396 | // Reference to the i'th input Node. NULL if out of bounds. |
| 397 | Node* lookup(uint i) const { return ((i < _max) ? _in[i] : NULL__null); } |
| 398 | // Reference to the i'th output Node. Error if out of bounds. |
| 399 | // Use this accessor sparingly. We are going trying to use iterators instead. |
| 400 | Node* raw_out(uint i) const { assert(i < _outcnt,"oob")do { if (!(i < _outcnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 400, "assert(" "i < _outcnt" ") failed", "oob"); ::breakpoint (); } } while (0); return _out[i]; } |
| 401 | // Return the unique out edge. |
| 402 | Node* unique_out() const { assert(_outcnt==1,"not unique")do { if (!(_outcnt==1)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 402, "assert(" "_outcnt==1" ") failed", "not unique"); ::breakpoint (); } } while (0); return _out[0]; } |
| 403 | // Delete out edge at position 'i' by moving last out edge to position 'i' |
| 404 | void raw_del_out(uint i) { |
| 405 | assert(i < _outcnt,"oob")do { if (!(i < _outcnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 405, "assert(" "i < _outcnt" ") failed", "oob"); ::breakpoint (); } } while (0); |
| 406 | assert(_outcnt > 0,"oob")do { if (!(_outcnt > 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 406, "assert(" "_outcnt > 0" ") failed", "oob"); ::breakpoint (); } } while (0); |
| 407 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 408 | // Record that a change happened here. |
| 409 | debug_only(_last_del = _out[i]; ++_del_tick)_last_del = _out[i]; ++_del_tick; |
| 410 | #endif |
| 411 | _out[i] = _out[--_outcnt]; |
| 412 | // Smash the old edge so it can't be used accidentally. |
| 413 | debug_only(_out[_outcnt] = (Node *)(uintptr_t)0xdeadbeef)_out[_outcnt] = (Node *)(uintptr_t)0xdeadbeef; |
| 414 | } |
| 415 | |
| 416 | #ifdef ASSERT1 |
| 417 | bool is_dead() const; |
| 418 | #define is_not_dead(n)((n) == __null || !VerifyIterativeGVN || !((n)->is_dead()) ) ((n) == NULL__null || !VerifyIterativeGVN || !((n)->is_dead())) |
| 419 | bool is_reachable_from_root() const; |
| 420 | #endif |
| 421 | // Check whether node has become unreachable |
| 422 | bool is_unreachable(PhaseIterGVN &igvn) const; |
| 423 | |
| 424 | // Set a required input edge, also updates corresponding output edge |
| 425 | void add_req( Node *n ); // Append a NEW required input |
| 426 | void add_req( Node *n0, Node *n1 ) { |
| 427 | add_req(n0); add_req(n1); } |
| 428 | void add_req( Node *n0, Node *n1, Node *n2 ) { |
| 429 | add_req(n0); add_req(n1); add_req(n2); } |
| 430 | void add_req_batch( Node* n, uint m ); // Append m NEW required inputs (all n). |
| 431 | void del_req( uint idx ); // Delete required edge & compact |
| 432 | void del_req_ordered( uint idx ); // Delete required edge & compact with preserved order |
| 433 | void ins_req( uint i, Node *n ); // Insert a NEW required input |
| 434 | void set_req( uint i, Node *n ) { |
| 435 | assert( is_not_dead(n), "can not use dead node")do { if (!(((n) == __null || !VerifyIterativeGVN || !((n)-> is_dead())))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 435, "assert(" "((n) == __null || !VerifyIterativeGVN || !((n)->is_dead()))" ") failed", "can not use dead node"); ::breakpoint(); } } while (0); |
| 436 | assert( i < _cnt, "oob: i=%d, _cnt=%d", i, _cnt)do { if (!(i < _cnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 436, "assert(" "i < _cnt" ") failed", "oob: i=%d, _cnt=%d" , i, _cnt); ::breakpoint(); } } while (0); |
| 437 | assert( !VerifyHashTableKeys || _hash_lock == 0,do { if (!(!VerifyHashTableKeys || _hash_lock == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 438, "assert(" "!VerifyHashTableKeys || _hash_lock == 0" ") failed" , "remove node from hash table before modifying it"); ::breakpoint (); } } while (0) |
| 438 | "remove node from hash table before modifying it")do { if (!(!VerifyHashTableKeys || _hash_lock == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 438, "assert(" "!VerifyHashTableKeys || _hash_lock == 0" ") failed" , "remove node from hash table before modifying it"); ::breakpoint (); } } while (0); |
| 439 | Node** p = &_in[i]; // cache this._in, across the del_out call |
| 440 | if (*p != NULL__null) (*p)->del_out((Node *)this); |
| 441 | (*p) = n; |
| 442 | if (n != NULL__null) n->add_out((Node *)this); |
| 443 | Compile::current()->record_modified_node(this); |
| 444 | } |
| 445 | // Light version of set_req() to init inputs after node creation. |
| 446 | void init_req( uint i, Node *n ) { |
| 447 | assert( i == 0 && this == n ||do { if (!(i == 0 && this == n || ((n) == __null || ! VerifyIterativeGVN || !((n)->is_dead())))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 448, "assert(" "i == 0 && this == n || ((n) == __null || !VerifyIterativeGVN || !((n)->is_dead()))" ") failed", "can not use dead node"); ::breakpoint(); } } while (0) |
| 448 | is_not_dead(n), "can not use dead node")do { if (!(i == 0 && this == n || ((n) == __null || ! VerifyIterativeGVN || !((n)->is_dead())))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 448, "assert(" "i == 0 && this == n || ((n) == __null || !VerifyIterativeGVN || !((n)->is_dead()))" ") failed", "can not use dead node"); ::breakpoint(); } } while (0); |
| 449 | assert( i < _cnt, "oob")do { if (!(i < _cnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 449, "assert(" "i < _cnt" ") failed", "oob"); ::breakpoint (); } } while (0); |
| 450 | assert( !VerifyHashTableKeys || _hash_lock == 0,do { if (!(!VerifyHashTableKeys || _hash_lock == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 451, "assert(" "!VerifyHashTableKeys || _hash_lock == 0" ") failed" , "remove node from hash table before modifying it"); ::breakpoint (); } } while (0) |
| 451 | "remove node from hash table before modifying it")do { if (!(!VerifyHashTableKeys || _hash_lock == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 451, "assert(" "!VerifyHashTableKeys || _hash_lock == 0" ") failed" , "remove node from hash table before modifying it"); ::breakpoint (); } } while (0); |
| 452 | assert( _in[i] == NULL, "sanity")do { if (!(_in[i] == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 452, "assert(" "_in[i] == __null" ") failed", "sanity"); :: breakpoint(); } } while (0); |
| 453 | _in[i] = n; |
| 454 | if (n != NULL__null) n->add_out((Node *)this); |
| 455 | Compile::current()->record_modified_node(this); |
| 456 | } |
| 457 | // Find first occurrence of n among my edges: |
| 458 | int find_edge(Node* n); |
| 459 | int find_prec_edge(Node* n) { |
| 460 | for (uint i = req(); i < len(); i++) { |
| 461 | if (_in[i] == n) return i; |
| 462 | if (_in[i] == NULL__null) { |
| 463 | DEBUG_ONLY( while ((++i) < len()) assert(_in[i] == NULL, "Gap in prec edges!"); )while ((++i) < len()) do { if (!(_in[i] == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 463, "assert(" "_in[i] == __null" ") failed", "Gap in prec edges!" ); ::breakpoint(); } } while (0); |
| 464 | break; |
| 465 | } |
| 466 | } |
| 467 | return -1; |
| 468 | } |
| 469 | int replace_edge(Node* old, Node* neww, PhaseGVN* gvn = NULL__null); |
| 470 | int replace_edges_in_range(Node* old, Node* neww, int start, int end, PhaseGVN* gvn); |
| 471 | // NULL out all inputs to eliminate incoming Def-Use edges. |
| 472 | void disconnect_inputs(Compile* C); |
| 473 | |
| 474 | // Quickly, return true if and only if I am Compile::current()->top(). |
| 475 | bool is_top() const { |
| 476 | assert((this == (Node*) Compile::current()->top()) == (_out == NULL), "")do { if (!((this == (Node*) Compile::current()->top()) == ( _out == __null))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 476, "assert(" "(this == (Node*) Compile::current()->top()) == (_out == __null)" ") failed", ""); ::breakpoint(); } } while (0); |
| 477 | return (_out == NULL__null); |
| 478 | } |
| 479 | // Reaffirm invariants for is_top. (Only from Compile::set_cached_top_node.) |
| 480 | void setup_is_top(); |
| 481 | |
| 482 | // Strip away casting. (It is depth-limited.) |
| 483 | Node* uncast(bool keep_deps = false) const; |
| 484 | // Return whether two Nodes are equivalent, after stripping casting. |
| 485 | bool eqv_uncast(const Node* n, bool keep_deps = false) const { |
| 486 | return (this->uncast(keep_deps) == n->uncast(keep_deps)); |
| 487 | } |
| 488 | |
| 489 | // Find out of current node that matches opcode. |
| 490 | Node* find_out_with(int opcode); |
| 491 | // Return true if the current node has an out that matches opcode. |
| 492 | bool has_out_with(int opcode); |
| 493 | // Return true if the current node has an out that matches any of the opcodes. |
| 494 | bool has_out_with(int opcode1, int opcode2, int opcode3, int opcode4); |
| 495 | |
| 496 | private: |
| 497 | static Node* uncast_helper(const Node* n, bool keep_deps); |
| 498 | |
| 499 | // Add an output edge to the end of the list |
| 500 | void add_out( Node *n ) { |
| 501 | if (is_top()) return; |
| 502 | if( _outcnt == _outmax ) out_grow(_outcnt); |
| 503 | _out[_outcnt++] = n; |
| 504 | } |
| 505 | // Delete an output edge |
| 506 | void del_out( Node *n ) { |
| 507 | if (is_top()) return; |
| 508 | Node** outp = &_out[_outcnt]; |
| 509 | // Find and remove n |
| 510 | do { |
| 511 | assert(outp > _out, "Missing Def-Use edge")do { if (!(outp > _out)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 511, "assert(" "outp > _out" ") failed", "Missing Def-Use edge" ); ::breakpoint(); } } while (0); |
| 512 | } while (*--outp != n); |
| 513 | *outp = _out[--_outcnt]; |
| 514 | // Smash the old edge so it can't be used accidentally. |
| 515 | debug_only(_out[_outcnt] = (Node *)(uintptr_t)0xdeadbeef)_out[_outcnt] = (Node *)(uintptr_t)0xdeadbeef; |
| 516 | // Record that a change happened here. |
| 517 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 518 | debug_only(_last_del = n; ++_del_tick)_last_del = n; ++_del_tick; |
| 519 | #endif |
| 520 | } |
| 521 | // Close gap after removing edge. |
| 522 | void close_prec_gap_at(uint gap) { |
| 523 | assert(_cnt <= gap && gap < _max, "no valid prec edge")do { if (!(_cnt <= gap && gap < _max)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 523, "assert(" "_cnt <= gap && gap < _max" ") failed" , "no valid prec edge"); ::breakpoint(); } } while (0); |
| 524 | uint i = gap; |
| 525 | Node *last = NULL__null; |
| 526 | for (; i < _max-1; ++i) { |
| 527 | Node *next = _in[i+1]; |
| 528 | if (next == NULL__null) break; |
| 529 | last = next; |
| 530 | } |
| 531 | _in[gap] = last; // Move last slot to empty one. |
| 532 | _in[i] = NULL__null; // NULL out last slot. |
| 533 | } |
| 534 | |
| 535 | public: |
| 536 | // Globally replace this node by a given new node, updating all uses. |
| 537 | void replace_by(Node* new_node); |
| 538 | // Globally replace this node by a given new node, updating all uses |
| 539 | // and cutting input edges of old node. |
| 540 | void subsume_by(Node* new_node, Compile* c) { |
| 541 | replace_by(new_node); |
| 542 | disconnect_inputs(c); |
| 543 | } |
| 544 | void set_req_X(uint i, Node *n, PhaseIterGVN *igvn); |
| 545 | void set_req_X(uint i, Node *n, PhaseGVN *gvn); |
| 546 | // Find the one non-null required input. RegionNode only |
| 547 | Node *nonnull_req() const; |
| 548 | // Add or remove precedence edges |
| 549 | void add_prec( Node *n ); |
| 550 | void rm_prec( uint i ); |
| 551 | |
| 552 | // Note: prec(i) will not necessarily point to n if edge already exists. |
| 553 | void set_prec( uint i, Node *n ) { |
| 554 | assert(i < _max, "oob: i=%d, _max=%d", i, _max)do { if (!(i < _max)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 554, "assert(" "i < _max" ") failed", "oob: i=%d, _max=%d" , i, _max); ::breakpoint(); } } while (0); |
| 555 | assert(is_not_dead(n), "can not use dead node")do { if (!(((n) == __null || !VerifyIterativeGVN || !((n)-> is_dead())))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 555, "assert(" "((n) == __null || !VerifyIterativeGVN || !((n)->is_dead()))" ") failed", "can not use dead node"); ::breakpoint(); } } while (0); |
| 556 | assert(i >= _cnt, "not a precedence edge")do { if (!(i >= _cnt)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 556, "assert(" "i >= _cnt" ") failed", "not a precedence edge" ); ::breakpoint(); } } while (0); |
| 557 | // Avoid spec violation: duplicated prec edge. |
| 558 | if (_in[i] == n) return; |
| 559 | if (n == NULL__null || find_prec_edge(n) != -1) { |
| 560 | rm_prec(i); |
| 561 | return; |
| 562 | } |
| 563 | if (_in[i] != NULL__null) _in[i]->del_out((Node *)this); |
| 564 | _in[i] = n; |
| 565 | n->add_out((Node *)this); |
| 566 | } |
| 567 | |
| 568 | // Set this node's index, used by cisc_version to replace current node |
| 569 | void set_idx(uint new_idx) { |
| 570 | const node_idx_t* ref = &_idx; |
| 571 | *(node_idx_t*)ref = new_idx; |
| 572 | } |
| 573 | // Swap input edge order. (Edge indexes i1 and i2 are usually 1 and 2.) |
| 574 | void swap_edges(uint i1, uint i2) { |
| 575 | debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH)uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH; |
| 576 | // Def-Use info is unchanged |
| 577 | Node* n1 = in(i1); |
| 578 | Node* n2 = in(i2); |
| 579 | _in[i1] = n2; |
| 580 | _in[i2] = n1; |
| 581 | // If this node is in the hash table, make sure it doesn't need a rehash. |
| 582 | assert(check_hash == NO_HASH || check_hash == hash(), "edge swap must preserve hash code")do { if (!(check_hash == NO_HASH || check_hash == hash())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 582, "assert(" "check_hash == NO_HASH || check_hash == hash()" ") failed", "edge swap must preserve hash code"); ::breakpoint (); } } while (0); |
| 583 | } |
| 584 | |
| 585 | // Iterators over input Nodes for a Node X are written as: |
| 586 | // for( i = 0; i < X.req(); i++ ) ... X[i] ... |
| 587 | // NOTE: Required edges can contain embedded NULL pointers. |
| 588 | |
| 589 | //----------------- Other Node Properties |
| 590 | |
| 591 | // Generate class IDs for (some) ideal nodes so that it is possible to determine |
| 592 | // the type of a node using a non-virtual method call (the method is_<Node>() below). |
| 593 | // |
| 594 | // A class ID of an ideal node is a set of bits. In a class ID, a single bit determines |
| 595 | // the type of the node the ID represents; another subset of an ID's bits are reserved |
| 596 | // for the superclasses of the node represented by the ID. |
| 597 | // |
| 598 | // By design, if A is a supertype of B, A.is_B() returns true and B.is_A() |
| 599 | // returns false. A.is_A() returns true. |
| 600 | // |
| 601 | // If two classes, A and B, have the same superclass, a different bit of A's class id |
| 602 | // is reserved for A's type than for B's type. That bit is specified by the third |
| 603 | // parameter in the macro DEFINE_CLASS_ID. |
| 604 | // |
| 605 | // By convention, classes with deeper hierarchy are declared first. Moreover, |
| 606 | // classes with the same hierarchy depth are sorted by usage frequency. |
| 607 | // |
| 608 | // The query method masks the bits to cut off bits of subclasses and then compares |
| 609 | // the result with the class id (see the macro DEFINE_CLASS_QUERY below). |
| 610 | // |
| 611 | // Class_MachCall=30, ClassMask_MachCall=31 |
| 612 | // 12 8 4 0 |
| 613 | // 0 0 0 0 0 0 0 0 1 1 1 1 0 |
| 614 | // | | | | |
| 615 | // | | | Bit_Mach=2 |
| 616 | // | | Bit_MachReturn=4 |
| 617 | // | Bit_MachSafePoint=8 |
| 618 | // Bit_MachCall=16 |
| 619 | // |
| 620 | // Class_CountedLoop=56, ClassMask_CountedLoop=63 |
| 621 | // 12 8 4 0 |
| 622 | // 0 0 0 0 0 0 0 1 1 1 0 0 0 |
| 623 | // | | | |
| 624 | // | | Bit_Region=8 |
| 625 | // | Bit_Loop=16 |
| 626 | // Bit_CountedLoop=32 |
| 627 | |
| 628 | #define DEFINE_CLASS_ID(cl, supcl, subn) \ |
| 629 | Bit_##cl = (Class_##supcl == 0) ? 1 << subn : (Bit_##supcl) << (1 + subn) , \ |
| 630 | Class_##cl = Class_##supcl + Bit_##cl , \ |
| 631 | ClassMask_##cl = ((Bit_##cl << 1) - 1) , |
| 632 | |
| 633 | // This enum is used only for C2 ideal and mach nodes with is_<node>() methods |
| 634 | // so that its values fit into 32 bits. |
| 635 | enum NodeClasses { |
| 636 | Bit_Node = 0x00000000, |
| 637 | Class_Node = 0x00000000, |
| 638 | ClassMask_Node = 0xFFFFFFFF, |
| 639 | |
| 640 | DEFINE_CLASS_ID(Multi, Node, 0) |
| 641 | DEFINE_CLASS_ID(SafePoint, Multi, 0) |
| 642 | DEFINE_CLASS_ID(Call, SafePoint, 0) |
| 643 | DEFINE_CLASS_ID(CallJava, Call, 0) |
| 644 | DEFINE_CLASS_ID(CallStaticJava, CallJava, 0) |
| 645 | DEFINE_CLASS_ID(CallDynamicJava, CallJava, 1) |
| 646 | DEFINE_CLASS_ID(CallRuntime, Call, 1) |
| 647 | DEFINE_CLASS_ID(CallLeaf, CallRuntime, 0) |
| 648 | DEFINE_CLASS_ID(CallLeafNoFP, CallLeaf, 0) |
| 649 | DEFINE_CLASS_ID(Allocate, Call, 2) |
| 650 | DEFINE_CLASS_ID(AllocateArray, Allocate, 0) |
| 651 | DEFINE_CLASS_ID(AbstractLock, Call, 3) |
| 652 | DEFINE_CLASS_ID(Lock, AbstractLock, 0) |
| 653 | DEFINE_CLASS_ID(Unlock, AbstractLock, 1) |
| 654 | DEFINE_CLASS_ID(ArrayCopy, Call, 4) |
| 655 | DEFINE_CLASS_ID(CallNative, Call, 5) |
| 656 | DEFINE_CLASS_ID(MultiBranch, Multi, 1) |
| 657 | DEFINE_CLASS_ID(PCTable, MultiBranch, 0) |
| 658 | DEFINE_CLASS_ID(Catch, PCTable, 0) |
| 659 | DEFINE_CLASS_ID(Jump, PCTable, 1) |
| 660 | DEFINE_CLASS_ID(If, MultiBranch, 1) |
| 661 | DEFINE_CLASS_ID(BaseCountedLoopEnd, If, 0) |
| 662 | DEFINE_CLASS_ID(CountedLoopEnd, BaseCountedLoopEnd, 0) |
| 663 | DEFINE_CLASS_ID(LongCountedLoopEnd, BaseCountedLoopEnd, 1) |
| 664 | DEFINE_CLASS_ID(RangeCheck, If, 1) |
| 665 | DEFINE_CLASS_ID(OuterStripMinedLoopEnd, If, 2) |
| 666 | DEFINE_CLASS_ID(NeverBranch, MultiBranch, 2) |
| 667 | DEFINE_CLASS_ID(Start, Multi, 2) |
| 668 | DEFINE_CLASS_ID(MemBar, Multi, 3) |
| 669 | DEFINE_CLASS_ID(Initialize, MemBar, 0) |
| 670 | DEFINE_CLASS_ID(MemBarStoreStore, MemBar, 1) |
| 671 | |
| 672 | DEFINE_CLASS_ID(Mach, Node, 1) |
| 673 | DEFINE_CLASS_ID(MachReturn, Mach, 0) |
| 674 | DEFINE_CLASS_ID(MachSafePoint, MachReturn, 0) |
| 675 | DEFINE_CLASS_ID(MachCall, MachSafePoint, 0) |
| 676 | DEFINE_CLASS_ID(MachCallJava, MachCall, 0) |
| 677 | DEFINE_CLASS_ID(MachCallStaticJava, MachCallJava, 0) |
| 678 | DEFINE_CLASS_ID(MachCallDynamicJava, MachCallJava, 1) |
| 679 | DEFINE_CLASS_ID(MachCallRuntime, MachCall, 1) |
| 680 | DEFINE_CLASS_ID(MachCallLeaf, MachCallRuntime, 0) |
| 681 | DEFINE_CLASS_ID(MachCallNative, MachCall, 2) |
| 682 | DEFINE_CLASS_ID(MachBranch, Mach, 1) |
| 683 | DEFINE_CLASS_ID(MachIf, MachBranch, 0) |
| 684 | DEFINE_CLASS_ID(MachGoto, MachBranch, 1) |
| 685 | DEFINE_CLASS_ID(MachNullCheck, MachBranch, 2) |
| 686 | DEFINE_CLASS_ID(MachSpillCopy, Mach, 2) |
| 687 | DEFINE_CLASS_ID(MachTemp, Mach, 3) |
| 688 | DEFINE_CLASS_ID(MachConstantBase, Mach, 4) |
| 689 | DEFINE_CLASS_ID(MachConstant, Mach, 5) |
| 690 | DEFINE_CLASS_ID(MachJump, MachConstant, 0) |
| 691 | DEFINE_CLASS_ID(MachMerge, Mach, 6) |
| 692 | DEFINE_CLASS_ID(MachMemBar, Mach, 7) |
| 693 | |
| 694 | DEFINE_CLASS_ID(Type, Node, 2) |
| 695 | DEFINE_CLASS_ID(Phi, Type, 0) |
| 696 | DEFINE_CLASS_ID(ConstraintCast, Type, 1) |
| 697 | DEFINE_CLASS_ID(CastII, ConstraintCast, 0) |
| 698 | DEFINE_CLASS_ID(CheckCastPP, ConstraintCast, 1) |
| 699 | DEFINE_CLASS_ID(CastLL, ConstraintCast, 2) |
| 700 | DEFINE_CLASS_ID(CastFF, ConstraintCast, 3) |
| 701 | DEFINE_CLASS_ID(CastDD, ConstraintCast, 4) |
| 702 | DEFINE_CLASS_ID(CastVV, ConstraintCast, 5) |
| 703 | DEFINE_CLASS_ID(CMove, Type, 3) |
| 704 | DEFINE_CLASS_ID(SafePointScalarObject, Type, 4) |
| 705 | DEFINE_CLASS_ID(DecodeNarrowPtr, Type, 5) |
| 706 | DEFINE_CLASS_ID(DecodeN, DecodeNarrowPtr, 0) |
| 707 | DEFINE_CLASS_ID(DecodeNKlass, DecodeNarrowPtr, 1) |
| 708 | DEFINE_CLASS_ID(EncodeNarrowPtr, Type, 6) |
| 709 | DEFINE_CLASS_ID(EncodeP, EncodeNarrowPtr, 0) |
| 710 | DEFINE_CLASS_ID(EncodePKlass, EncodeNarrowPtr, 1) |
| 711 | DEFINE_CLASS_ID(Vector, Type, 7) |
| 712 | DEFINE_CLASS_ID(VectorMaskCmp, Vector, 0) |
| 713 | DEFINE_CLASS_ID(VectorUnbox, Vector, 1) |
| 714 | DEFINE_CLASS_ID(VectorReinterpret, Vector, 2) |
| 715 | DEFINE_CLASS_ID(ShiftV, Vector, 3) |
| 716 | |
| 717 | DEFINE_CLASS_ID(Proj, Node, 3) |
| 718 | DEFINE_CLASS_ID(CatchProj, Proj, 0) |
| 719 | DEFINE_CLASS_ID(JumpProj, Proj, 1) |
| 720 | DEFINE_CLASS_ID(IfProj, Proj, 2) |
| 721 | DEFINE_CLASS_ID(IfTrue, IfProj, 0) |
| 722 | DEFINE_CLASS_ID(IfFalse, IfProj, 1) |
| 723 | DEFINE_CLASS_ID(Parm, Proj, 4) |
| 724 | DEFINE_CLASS_ID(MachProj, Proj, 5) |
| 725 | |
| 726 | DEFINE_CLASS_ID(Mem, Node, 4) |
| 727 | DEFINE_CLASS_ID(Load, Mem, 0) |
| 728 | DEFINE_CLASS_ID(LoadVector, Load, 0) |
| 729 | DEFINE_CLASS_ID(LoadVectorGather, LoadVector, 0) |
| 730 | DEFINE_CLASS_ID(LoadVectorMasked, LoadVector, 1) |
| 731 | DEFINE_CLASS_ID(Store, Mem, 1) |
| 732 | DEFINE_CLASS_ID(StoreVector, Store, 0) |
| 733 | DEFINE_CLASS_ID(StoreVectorScatter, StoreVector, 0) |
| 734 | DEFINE_CLASS_ID(StoreVectorMasked, StoreVector, 1) |
| 735 | DEFINE_CLASS_ID(LoadStore, Mem, 2) |
| 736 | DEFINE_CLASS_ID(LoadStoreConditional, LoadStore, 0) |
| 737 | DEFINE_CLASS_ID(CompareAndSwap, LoadStoreConditional, 0) |
| 738 | DEFINE_CLASS_ID(CompareAndExchangeNode, LoadStore, 1) |
| 739 | |
| 740 | DEFINE_CLASS_ID(Region, Node, 5) |
| 741 | DEFINE_CLASS_ID(Loop, Region, 0) |
| 742 | DEFINE_CLASS_ID(Root, Loop, 0) |
| 743 | DEFINE_CLASS_ID(BaseCountedLoop, Loop, 1) |
| 744 | DEFINE_CLASS_ID(CountedLoop, BaseCountedLoop, 0) |
| 745 | DEFINE_CLASS_ID(LongCountedLoop, BaseCountedLoop, 1) |
| 746 | DEFINE_CLASS_ID(OuterStripMinedLoop, Loop, 2) |
| 747 | |
| 748 | DEFINE_CLASS_ID(Sub, Node, 6) |
| 749 | DEFINE_CLASS_ID(Cmp, Sub, 0) |
| 750 | DEFINE_CLASS_ID(FastLock, Cmp, 0) |
| 751 | DEFINE_CLASS_ID(FastUnlock, Cmp, 1) |
| 752 | DEFINE_CLASS_ID(SubTypeCheck,Cmp, 2) |
| 753 | |
| 754 | DEFINE_CLASS_ID(MergeMem, Node, 7) |
| 755 | DEFINE_CLASS_ID(Bool, Node, 8) |
| 756 | DEFINE_CLASS_ID(AddP, Node, 9) |
| 757 | DEFINE_CLASS_ID(BoxLock, Node, 10) |
| 758 | DEFINE_CLASS_ID(Add, Node, 11) |
| 759 | DEFINE_CLASS_ID(Mul, Node, 12) |
| 760 | DEFINE_CLASS_ID(ClearArray, Node, 14) |
| 761 | DEFINE_CLASS_ID(Halt, Node, 15) |
| 762 | DEFINE_CLASS_ID(Opaque1, Node, 16) |
| 763 | DEFINE_CLASS_ID(Move, Node, 17) |
| 764 | DEFINE_CLASS_ID(LShift, Node, 18) |
| 765 | |
| 766 | _max_classes = ClassMask_Move |
| 767 | }; |
| 768 | #undef DEFINE_CLASS_ID |
| 769 | |
| 770 | // Flags are sorted by usage frequency. |
| 771 | enum NodeFlags { |
| 772 | Flag_is_Copy = 1 << 0, // should be first bit to avoid shift |
| 773 | Flag_rematerialize = 1 << 1, |
| 774 | Flag_needs_anti_dependence_check = 1 << 2, |
| 775 | Flag_is_macro = 1 << 3, |
| 776 | Flag_is_Con = 1 << 4, |
| 777 | Flag_is_cisc_alternate = 1 << 5, |
| 778 | Flag_is_dead_loop_safe = 1 << 6, |
| 779 | Flag_may_be_short_branch = 1 << 7, |
| 780 | Flag_avoid_back_to_back_before = 1 << 8, |
| 781 | Flag_avoid_back_to_back_after = 1 << 9, |
| 782 | Flag_has_call = 1 << 10, |
| 783 | Flag_is_reduction = 1 << 11, |
| 784 | Flag_is_scheduled = 1 << 12, |
| 785 | Flag_has_vector_mask_set = 1 << 13, |
| 786 | Flag_is_expensive = 1 << 14, |
| 787 | Flag_is_predicated_vector = 1 << 15, |
| 788 | Flag_for_post_loop_opts_igvn = 1 << 16, |
| 789 | _last_flag = Flag_for_post_loop_opts_igvn |
| 790 | }; |
| 791 | |
| 792 | class PD; |
| 793 | |
| 794 | private: |
| 795 | juint _class_id; |
| 796 | juint _flags; |
| 797 | |
| 798 | static juint max_flags(); |
| 799 | |
| 800 | protected: |
| 801 | // These methods should be called from constructors only. |
| 802 | void init_class_id(juint c) { |
| 803 | _class_id = c; // cast out const |
| 804 | } |
| 805 | void init_flags(uint fl) { |
| 806 | assert(fl <= max_flags(), "invalid node flag")do { if (!(fl <= max_flags())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 806, "assert(" "fl <= max_flags()" ") failed", "invalid node flag" ); ::breakpoint(); } } while (0); |
| 807 | _flags |= fl; |
| 808 | } |
| 809 | void clear_flag(uint fl) { |
| 810 | assert(fl <= max_flags(), "invalid node flag")do { if (!(fl <= max_flags())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 810, "assert(" "fl <= max_flags()" ") failed", "invalid node flag" ); ::breakpoint(); } } while (0); |
| 811 | _flags &= ~fl; |
| 812 | } |
| 813 | |
| 814 | public: |
| 815 | const juint class_id() const { return _class_id; } |
| 816 | |
| 817 | const juint flags() const { return _flags; } |
| 818 | |
| 819 | void add_flag(juint fl) { init_flags(fl); } |
| 820 | |
| 821 | void remove_flag(juint fl) { clear_flag(fl); } |
| 822 | |
| 823 | // Return a dense integer opcode number |
| 824 | virtual int Opcode() const; |
| 825 | |
| 826 | // Virtual inherited Node size |
| 827 | virtual uint size_of() const; |
| 828 | |
| 829 | // Other interesting Node properties |
| 830 | #define DEFINE_CLASS_QUERY(type) \ |
| 831 | bool is_##type() const { \ |
| 832 | return ((_class_id & ClassMask_##type) == Class_##type); \ |
| 833 | } \ |
| 834 | type##Node *as_##type() const { \ |
| 835 | assert(is_##type(), "invalid node class: %s", Name())do { if (!(is_##type())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 835, "assert(" "is_##type()" ") failed", "invalid node class: %s" , Name()); ::breakpoint(); } } while (0); \ |
| 836 | return (type##Node*)this; \ |
| 837 | } \ |
| 838 | type##Node* isa_##type() const { \ |
| 839 | return (is_##type()) ? as_##type() : NULL__null; \ |
| 840 | } |
| 841 | |
| 842 | DEFINE_CLASS_QUERY(AbstractLock) |
| 843 | DEFINE_CLASS_QUERY(Add) |
| 844 | DEFINE_CLASS_QUERY(AddP) |
| 845 | DEFINE_CLASS_QUERY(Allocate) |
| 846 | DEFINE_CLASS_QUERY(AllocateArray) |
| 847 | DEFINE_CLASS_QUERY(ArrayCopy) |
| 848 | DEFINE_CLASS_QUERY(BaseCountedLoop) |
| 849 | DEFINE_CLASS_QUERY(BaseCountedLoopEnd) |
| 850 | DEFINE_CLASS_QUERY(Bool) |
| 851 | DEFINE_CLASS_QUERY(BoxLock) |
| 852 | DEFINE_CLASS_QUERY(Call) |
| 853 | DEFINE_CLASS_QUERY(CallNative) |
| 854 | DEFINE_CLASS_QUERY(CallDynamicJava) |
| 855 | DEFINE_CLASS_QUERY(CallJava) |
| 856 | DEFINE_CLASS_QUERY(CallLeaf) |
| 857 | DEFINE_CLASS_QUERY(CallLeafNoFP) |
| 858 | DEFINE_CLASS_QUERY(CallRuntime) |
| 859 | DEFINE_CLASS_QUERY(CallStaticJava) |
| 860 | DEFINE_CLASS_QUERY(Catch) |
| 861 | DEFINE_CLASS_QUERY(CatchProj) |
| 862 | DEFINE_CLASS_QUERY(CheckCastPP) |
| 863 | DEFINE_CLASS_QUERY(CastII) |
| 864 | DEFINE_CLASS_QUERY(CastLL) |
| 865 | DEFINE_CLASS_QUERY(ConstraintCast) |
| 866 | DEFINE_CLASS_QUERY(ClearArray) |
| 867 | DEFINE_CLASS_QUERY(CMove) |
| 868 | DEFINE_CLASS_QUERY(Cmp) |
| 869 | DEFINE_CLASS_QUERY(CountedLoop) |
| 870 | DEFINE_CLASS_QUERY(CountedLoopEnd) |
| 871 | DEFINE_CLASS_QUERY(DecodeNarrowPtr) |
| 872 | DEFINE_CLASS_QUERY(DecodeN) |
| 873 | DEFINE_CLASS_QUERY(DecodeNKlass) |
| 874 | DEFINE_CLASS_QUERY(EncodeNarrowPtr) |
| 875 | DEFINE_CLASS_QUERY(EncodeP) |
| 876 | DEFINE_CLASS_QUERY(EncodePKlass) |
| 877 | DEFINE_CLASS_QUERY(FastLock) |
| 878 | DEFINE_CLASS_QUERY(FastUnlock) |
| 879 | DEFINE_CLASS_QUERY(Halt) |
| 880 | DEFINE_CLASS_QUERY(If) |
| 881 | DEFINE_CLASS_QUERY(RangeCheck) |
| 882 | DEFINE_CLASS_QUERY(IfProj) |
| 883 | DEFINE_CLASS_QUERY(IfFalse) |
| 884 | DEFINE_CLASS_QUERY(IfTrue) |
| 885 | DEFINE_CLASS_QUERY(Initialize) |
| 886 | DEFINE_CLASS_QUERY(Jump) |
| 887 | DEFINE_CLASS_QUERY(JumpProj) |
| 888 | DEFINE_CLASS_QUERY(LongCountedLoop) |
| 889 | DEFINE_CLASS_QUERY(LongCountedLoopEnd) |
| 890 | DEFINE_CLASS_QUERY(Load) |
| 891 | DEFINE_CLASS_QUERY(LoadStore) |
| 892 | DEFINE_CLASS_QUERY(LoadStoreConditional) |
| 893 | DEFINE_CLASS_QUERY(Lock) |
| 894 | DEFINE_CLASS_QUERY(Loop) |
| 895 | DEFINE_CLASS_QUERY(LShift) |
| 896 | DEFINE_CLASS_QUERY(Mach) |
| 897 | DEFINE_CLASS_QUERY(MachBranch) |
| 898 | DEFINE_CLASS_QUERY(MachCall) |
| 899 | DEFINE_CLASS_QUERY(MachCallNative) |
| 900 | DEFINE_CLASS_QUERY(MachCallDynamicJava) |
| 901 | DEFINE_CLASS_QUERY(MachCallJava) |
| 902 | DEFINE_CLASS_QUERY(MachCallLeaf) |
| 903 | DEFINE_CLASS_QUERY(MachCallRuntime) |
| 904 | DEFINE_CLASS_QUERY(MachCallStaticJava) |
| 905 | DEFINE_CLASS_QUERY(MachConstantBase) |
| 906 | DEFINE_CLASS_QUERY(MachConstant) |
| 907 | DEFINE_CLASS_QUERY(MachGoto) |
| 908 | DEFINE_CLASS_QUERY(MachIf) |
| 909 | DEFINE_CLASS_QUERY(MachJump) |
| 910 | DEFINE_CLASS_QUERY(MachNullCheck) |
| 911 | DEFINE_CLASS_QUERY(MachProj) |
| 912 | DEFINE_CLASS_QUERY(MachReturn) |
| 913 | DEFINE_CLASS_QUERY(MachSafePoint) |
| 914 | DEFINE_CLASS_QUERY(MachSpillCopy) |
| 915 | DEFINE_CLASS_QUERY(MachTemp) |
| 916 | DEFINE_CLASS_QUERY(MachMemBar) |
| 917 | DEFINE_CLASS_QUERY(MachMerge) |
| 918 | DEFINE_CLASS_QUERY(Mem) |
| 919 | DEFINE_CLASS_QUERY(MemBar) |
| 920 | DEFINE_CLASS_QUERY(MemBarStoreStore) |
| 921 | DEFINE_CLASS_QUERY(MergeMem) |
| 922 | DEFINE_CLASS_QUERY(Move) |
| 923 | DEFINE_CLASS_QUERY(Mul) |
| 924 | DEFINE_CLASS_QUERY(Multi) |
| 925 | DEFINE_CLASS_QUERY(MultiBranch) |
| 926 | DEFINE_CLASS_QUERY(Opaque1) |
| 927 | DEFINE_CLASS_QUERY(OuterStripMinedLoop) |
| 928 | DEFINE_CLASS_QUERY(OuterStripMinedLoopEnd) |
| 929 | DEFINE_CLASS_QUERY(Parm) |
| 930 | DEFINE_CLASS_QUERY(PCTable) |
| 931 | DEFINE_CLASS_QUERY(Phi) |
| 932 | DEFINE_CLASS_QUERY(Proj) |
| 933 | DEFINE_CLASS_QUERY(Region) |
| 934 | DEFINE_CLASS_QUERY(Root) |
| 935 | DEFINE_CLASS_QUERY(SafePoint) |
| 936 | DEFINE_CLASS_QUERY(SafePointScalarObject) |
| 937 | DEFINE_CLASS_QUERY(Start) |
| 938 | DEFINE_CLASS_QUERY(Store) |
| 939 | DEFINE_CLASS_QUERY(Sub) |
| 940 | DEFINE_CLASS_QUERY(SubTypeCheck) |
| 941 | DEFINE_CLASS_QUERY(Type) |
| 942 | DEFINE_CLASS_QUERY(Vector) |
| 943 | DEFINE_CLASS_QUERY(VectorMaskCmp) |
| 944 | DEFINE_CLASS_QUERY(VectorUnbox) |
| 945 | DEFINE_CLASS_QUERY(VectorReinterpret); |
| 946 | DEFINE_CLASS_QUERY(LoadVector) |
| 947 | DEFINE_CLASS_QUERY(LoadVectorGather) |
| 948 | DEFINE_CLASS_QUERY(StoreVector) |
| 949 | DEFINE_CLASS_QUERY(StoreVectorScatter) |
| 950 | DEFINE_CLASS_QUERY(ShiftV) |
| 951 | DEFINE_CLASS_QUERY(Unlock) |
| 952 | |
| 953 | #undef DEFINE_CLASS_QUERY |
| 954 | |
| 955 | // duplicate of is_MachSpillCopy() |
| 956 | bool is_SpillCopy () const { |
| 957 | return ((_class_id & ClassMask_MachSpillCopy) == Class_MachSpillCopy); |
| 958 | } |
| 959 | |
| 960 | bool is_Con () const { return (_flags & Flag_is_Con) != 0; } |
| 961 | // The data node which is safe to leave in dead loop during IGVN optimization. |
| 962 | bool is_dead_loop_safe() const; |
| 963 | |
| 964 | // is_Copy() returns copied edge index (0 or 1) |
| 965 | uint is_Copy() const { return (_flags & Flag_is_Copy); } |
| 966 | |
| 967 | virtual bool is_CFG() const { return false; } |
| 968 | |
| 969 | // If this node is control-dependent on a test, can it be |
| 970 | // rerouted to a dominating equivalent test? This is usually |
| 971 | // true of non-CFG nodes, but can be false for operations which |
| 972 | // depend for their correct sequencing on more than one test. |
| 973 | // (In that case, hoisting to a dominating test may silently |
| 974 | // skip some other important test.) |
| 975 | virtual bool depends_only_on_test() const { assert(!is_CFG(), "")do { if (!(!is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 975, "assert(" "!is_CFG()" ") failed", ""); ::breakpoint(); } } while (0); return true; }; |
| 976 | |
| 977 | // When building basic blocks, I need to have a notion of block beginning |
| 978 | // Nodes, next block selector Nodes (block enders), and next block |
| 979 | // projections. These calls need to work on their machine equivalents. The |
| 980 | // Ideal beginning Nodes are RootNode, RegionNode and StartNode. |
| 981 | bool is_block_start() const { |
| 982 | if ( is_Region() ) |
| 983 | return this == (const Node*)in(0); |
| 984 | else |
| 985 | return is_Start(); |
| 986 | } |
| 987 | |
| 988 | // The Ideal control projection Nodes are IfTrue/IfFalse, JumpProjNode, Root, |
| 989 | // Goto and Return. This call also returns the block ending Node. |
| 990 | virtual const Node *is_block_proj() const; |
| 991 | |
| 992 | // The node is a "macro" node which needs to be expanded before matching |
| 993 | bool is_macro() const { return (_flags & Flag_is_macro) != 0; } |
| 994 | // The node is expensive: the best control is set during loop opts |
| 995 | bool is_expensive() const { return (_flags & Flag_is_expensive) != 0 && in(0) != NULL__null; } |
| 996 | |
| 997 | // An arithmetic node which accumulates a data in a loop. |
| 998 | // It must have the loop's phi as input and provide a def to the phi. |
| 999 | bool is_reduction() const { return (_flags & Flag_is_reduction) != 0; } |
| 1000 | |
| 1001 | bool is_predicated_vector() const { return (_flags & Flag_is_predicated_vector) != 0; } |
| 1002 | |
| 1003 | // The node is a CountedLoopEnd with a mask annotation so as to emit a restore context |
| 1004 | bool has_vector_mask_set() const { return (_flags & Flag_has_vector_mask_set) != 0; } |
| 1005 | |
| 1006 | // Used in lcm to mark nodes that have scheduled |
| 1007 | bool is_scheduled() const { return (_flags & Flag_is_scheduled) != 0; } |
| 1008 | |
| 1009 | bool for_post_loop_opts_igvn() const { return (_flags & Flag_for_post_loop_opts_igvn) != 0; } |
| 1010 | |
| 1011 | //----------------- Optimization |
| 1012 | |
| 1013 | // Get the worst-case Type output for this Node. |
| 1014 | virtual const class Type *bottom_type() const; |
| 1015 | |
| 1016 | // If we find a better type for a node, try to record it permanently. |
| 1017 | // Return true if this node actually changed. |
| 1018 | // Be sure to do the hash_delete game in the "rehash" variant. |
| 1019 | void raise_bottom_type(const Type* new_type); |
| 1020 | |
| 1021 | // Get the address type with which this node uses and/or defs memory, |
| 1022 | // or NULL if none. The address type is conservatively wide. |
| 1023 | // Returns non-null for calls, membars, loads, stores, etc. |
| 1024 | // Returns TypePtr::BOTTOM if the node touches memory "broadly". |
| 1025 | virtual const class TypePtr *adr_type() const { return NULL__null; } |
| 1026 | |
| 1027 | // Return an existing node which computes the same function as this node. |
| 1028 | // The optimistic combined algorithm requires this to return a Node which |
| 1029 | // is a small number of steps away (e.g., one of my inputs). |
| 1030 | virtual Node* Identity(PhaseGVN* phase); |
| 1031 | |
| 1032 | // Return the set of values this Node can take on at runtime. |
| 1033 | virtual const Type* Value(PhaseGVN* phase) const; |
| 1034 | |
| 1035 | // Return a node which is more "ideal" than the current node. |
| 1036 | // The invariants on this call are subtle. If in doubt, read the |
| 1037 | // treatise in node.cpp above the default implemention AND TEST WITH |
| 1038 | // +VerifyIterativeGVN! |
| 1039 | virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); |
| 1040 | |
| 1041 | // Some nodes have specific Ideal subgraph transformations only if they are |
| 1042 | // unique users of specific nodes. Such nodes should be put on IGVN worklist |
| 1043 | // for the transformations to happen. |
| 1044 | bool has_special_unique_user() const; |
| 1045 | |
| 1046 | // Skip Proj and CatchProj nodes chains. Check for Null and Top. |
| 1047 | Node* find_exact_control(Node* ctrl); |
| 1048 | |
| 1049 | // Check if 'this' node dominates or equal to 'sub'. |
| 1050 | bool dominates(Node* sub, Node_List &nlist); |
| 1051 | |
| 1052 | protected: |
| 1053 | bool remove_dead_region(PhaseGVN *phase, bool can_reshape); |
| 1054 | public: |
| 1055 | |
| 1056 | // See if there is valid pipeline info |
| 1057 | static const Pipeline *pipeline_class(); |
| 1058 | virtual const Pipeline *pipeline() const; |
| 1059 | |
| 1060 | // Compute the latency from the def to this instruction of the ith input node |
| 1061 | uint latency(uint i); |
| 1062 | |
| 1063 | // Hash & compare functions, for pessimistic value numbering |
| 1064 | |
| 1065 | // If the hash function returns the special sentinel value NO_HASH, |
| 1066 | // the node is guaranteed never to compare equal to any other node. |
| 1067 | // If we accidentally generate a hash with value NO_HASH the node |
| 1068 | // won't go into the table and we'll lose a little optimization. |
| 1069 | static const uint NO_HASH = 0; |
| 1070 | virtual uint hash() const; |
| 1071 | virtual bool cmp( const Node &n ) const; |
| 1072 | |
| 1073 | // Operation appears to be iteratively computed (such as an induction variable) |
| 1074 | // It is possible for this operation to return false for a loop-varying |
| 1075 | // value, if it appears (by local graph inspection) to be computed by a simple conditional. |
| 1076 | bool is_iteratively_computed(); |
| 1077 | |
| 1078 | // Determine if a node is a counted loop induction variable. |
| 1079 | // NOTE: The method is defined in "loopnode.cpp". |
| 1080 | bool is_cloop_ind_var() const; |
| 1081 | |
| 1082 | // Return a node with opcode "opc" and same inputs as "this" if one can |
| 1083 | // be found; Otherwise return NULL; |
| 1084 | Node* find_similar(int opc); |
| 1085 | |
| 1086 | // Return the unique control out if only one. Null if none or more than one. |
| 1087 | Node* unique_ctrl_out() const; |
| 1088 | |
| 1089 | // Set control or add control as precedence edge |
| 1090 | void ensure_control_or_add_prec(Node* c); |
| 1091 | |
| 1092 | //----------------- Code Generation |
| 1093 | |
| 1094 | // Ideal register class for Matching. Zero means unmatched instruction |
| 1095 | // (these are cloned instead of converted to machine nodes). |
| 1096 | virtual uint ideal_reg() const; |
| 1097 | |
| 1098 | static const uint NotAMachineReg; // must be > max. machine register |
| 1099 | |
| 1100 | // Do we Match on this edge index or not? Generally false for Control |
| 1101 | // and true for everything else. Weird for calls & returns. |
| 1102 | virtual uint match_edge(uint idx) const; |
| 1103 | |
| 1104 | // Register class output is returned in |
| 1105 | virtual const RegMask &out_RegMask() const; |
| 1106 | // Register class input is expected in |
| 1107 | virtual const RegMask &in_RegMask(uint) const; |
| 1108 | // Should we clone rather than spill this instruction? |
| 1109 | bool rematerialize() const; |
| 1110 | |
| 1111 | // Return JVM State Object if this Node carries debug info, or NULL otherwise |
| 1112 | virtual JVMState* jvms() const; |
| 1113 | |
| 1114 | // Print as assembly |
| 1115 | virtual void format( PhaseRegAlloc *, outputStream* st = tty ) const; |
| 1116 | // Emit bytes starting at parameter 'ptr' |
| 1117 | // Bump 'ptr' by the number of output bytes |
| 1118 | virtual void emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const; |
| 1119 | // Size of instruction in bytes |
| 1120 | virtual uint size(PhaseRegAlloc *ra_) const; |
| 1121 | |
| 1122 | // Convenience function to extract an integer constant from a node. |
| 1123 | // If it is not an integer constant (either Con, CastII, or Mach), |
| 1124 | // return value_if_unknown. |
| 1125 | jint find_int_con(jint value_if_unknown) const { |
| 1126 | const TypeInt* t = find_int_type(); |
| 1127 | return (t != NULL__null && t->is_con()) ? t->get_con() : value_if_unknown; |
| 1128 | } |
| 1129 | // Return the constant, knowing it is an integer constant already |
| 1130 | jint get_int() const { |
| 1131 | const TypeInt* t = find_int_type(); |
| 1132 | guarantee(t != NULL, "must be con")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1132, "guarantee(" "t != NULL" ") failed", "must be con"); :: breakpoint(); } } while (0); |
| 1133 | return t->get_con(); |
| 1134 | } |
| 1135 | // Here's where the work is done. Can produce non-constant int types too. |
| 1136 | const TypeInt* find_int_type() const; |
| 1137 | const TypeInteger* find_integer_type(BasicType bt) const; |
| 1138 | |
| 1139 | // Same thing for long (and intptr_t, via type.hpp): |
| 1140 | jlong get_long() const { |
| 1141 | const TypeLong* t = find_long_type(); |
| 1142 | guarantee(t != NULL, "must be con")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1142, "guarantee(" "t != NULL" ") failed", "must be con"); :: breakpoint(); } } while (0); |
| 1143 | return t->get_con(); |
| 1144 | } |
| 1145 | jlong find_long_con(jint value_if_unknown) const { |
| 1146 | const TypeLong* t = find_long_type(); |
| 1147 | return (t != NULL__null && t->is_con()) ? t->get_con() : value_if_unknown; |
| 1148 | } |
| 1149 | const TypeLong* find_long_type() const; |
| 1150 | |
| 1151 | jlong get_integer_as_long(BasicType bt) const { |
| 1152 | const TypeInteger* t = find_integer_type(bt); |
| 1153 | guarantee(t != NULL, "must be con")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1153, "guarantee(" "t != NULL" ") failed", "must be con"); :: breakpoint(); } } while (0); |
| 1154 | return t->get_con_as_long(bt); |
| 1155 | } |
| 1156 | const TypePtr* get_ptr_type() const; |
| 1157 | |
| 1158 | // These guys are called by code generated by ADLC: |
| 1159 | intptr_t get_ptr() const; |
| 1160 | intptr_t get_narrowcon() const; |
| 1161 | jdouble getd() const; |
| 1162 | jfloat getf() const; |
| 1163 | |
| 1164 | // Nodes which are pinned into basic blocks |
| 1165 | virtual bool pinned() const { return false; } |
| 1166 | |
| 1167 | // Nodes which use memory without consuming it, hence need antidependences |
| 1168 | // More specifically, needs_anti_dependence_check returns true iff the node |
| 1169 | // (a) does a load, and (b) does not perform a store (except perhaps to a |
| 1170 | // stack slot or some other unaliased location). |
| 1171 | bool needs_anti_dependence_check() const; |
| 1172 | |
| 1173 | // Return which operand this instruction may cisc-spill. In other words, |
| 1174 | // return operand position that can convert from reg to memory access |
| 1175 | virtual int cisc_operand() const { return AdlcVMDeps::Not_cisc_spillable; } |
| 1176 | bool is_cisc_alternate() const { return (_flags & Flag_is_cisc_alternate) != 0; } |
| 1177 | |
| 1178 | // Whether this is a memory-writing machine node. |
| 1179 | bool is_memory_writer() const { return is_Mach() && bottom_type()->has_memory(); } |
| 1180 | |
| 1181 | //----------------- Printing, etc |
| 1182 | #ifndef PRODUCT |
| 1183 | private: |
| 1184 | int _indent; |
| 1185 | |
| 1186 | public: |
| 1187 | void set_indent(int indent) { _indent = indent; } |
| 1188 | |
| 1189 | private: |
| 1190 | static bool add_to_worklist(Node* n, Node_List* worklist, Arena* old_arena, VectorSet* old_space, VectorSet* new_space); |
| 1191 | public: |
| 1192 | Node* find(int idx, bool only_ctrl = false); // Search the graph for the given idx. |
| 1193 | Node* find_ctrl(int idx); // Search control ancestors for the given idx. |
| 1194 | void dump() const { dump("\n"); } // Print this node. |
| 1195 | void dump(const char* suffix, bool mark = false, outputStream *st = tty) const; // Print this node. |
| 1196 | void dump(int depth) const; // Print this node, recursively to depth d |
| 1197 | void dump_ctrl(int depth) const; // Print control nodes, to depth d |
| 1198 | void dump_comp() const; // Print this node in compact representation. |
| 1199 | // Print this node in compact representation. |
| 1200 | void dump_comp(const char* suffix, outputStream *st = tty) const; |
| 1201 | virtual void dump_req(outputStream *st = tty) const; // Print required-edge info |
| 1202 | virtual void dump_prec(outputStream *st = tty) const; // Print precedence-edge info |
| 1203 | virtual void dump_out(outputStream *st = tty) const; // Print the output edge info |
| 1204 | virtual void dump_spec(outputStream *st) const {}; // Print per-node info |
| 1205 | // Print compact per-node info |
| 1206 | virtual void dump_compact_spec(outputStream *st) const { dump_spec(st); } |
| 1207 | void dump_related() const; // Print related nodes (depends on node at hand). |
| 1208 | // Print related nodes up to given depths for input and output nodes. |
| 1209 | void dump_related(uint d_in, uint d_out) const; |
| 1210 | void dump_related_compact() const; // Print related nodes in compact representation. |
| 1211 | // Collect related nodes. |
| 1212 | virtual void related(GrowableArray<Node*> *in_rel, GrowableArray<Node*> *out_rel, bool compact) const; |
| 1213 | // Collect nodes starting from this node, explicitly including/excluding control and data links. |
| 1214 | void collect_nodes(GrowableArray<Node*> *ns, int d, bool ctrl, bool data) const; |
| 1215 | |
| 1216 | // Node collectors, to be used in implementations of Node::rel(). |
| 1217 | // Collect the entire data input graph. Include control inputs if requested. |
| 1218 | void collect_nodes_in_all_data(GrowableArray<Node*> *ns, bool ctrl) const; |
| 1219 | // Collect the entire control input graph. Include data inputs if requested. |
| 1220 | void collect_nodes_in_all_ctrl(GrowableArray<Node*> *ns, bool data) const; |
| 1221 | // Collect the entire output graph until hitting and including control nodes. |
| 1222 | void collect_nodes_out_all_ctrl_boundary(GrowableArray<Node*> *ns) const; |
| 1223 | |
| 1224 | void verify_edges(Unique_Node_List &visited); // Verify bi-directional edges |
| 1225 | static void verify(int verify_depth, VectorSet& visited, Node_List& worklist); |
| 1226 | |
| 1227 | // This call defines a class-unique string used to identify class instances |
| 1228 | virtual const char *Name() const; |
| 1229 | |
| 1230 | void dump_format(PhaseRegAlloc *ra) const; // debug access to MachNode::format(...) |
| 1231 | // RegMask Print Functions |
| 1232 | void dump_in_regmask(int idx) { in_RegMask(idx).dump(); } |
| 1233 | void dump_out_regmask() { out_RegMask().dump(); } |
| 1234 | static bool in_dump() { return Compile::current()->_in_dump_cnt > 0; } |
| 1235 | void fast_dump() const { |
| 1236 | tty->print("%4d: %-17s", _idx, Name()); |
| 1237 | for (uint i = 0; i < len(); i++) |
| 1238 | if (in(i)) |
| 1239 | tty->print(" %4d", in(i)->_idx); |
| 1240 | else |
| 1241 | tty->print(" NULL"); |
| 1242 | tty->print("\n"); |
| 1243 | } |
| 1244 | #endif |
| 1245 | #ifdef ASSERT1 |
| 1246 | void verify_construction(); |
| 1247 | bool verify_jvms(const JVMState* jvms) const; |
| 1248 | int _debug_idx; // Unique value assigned to every node. |
| 1249 | int debug_idx() const { return _debug_idx; } |
| 1250 | void set_debug_idx( int debug_idx ) { _debug_idx = debug_idx; } |
| 1251 | |
| 1252 | Node* _debug_orig; // Original version of this, if any. |
| 1253 | Node* debug_orig() const { return _debug_orig; } |
| 1254 | void set_debug_orig(Node* orig); // _debug_orig = orig |
| 1255 | void dump_orig(outputStream *st, bool print_key = true) const; |
| 1256 | |
| 1257 | int _hash_lock; // Barrier to modifications of nodes in the hash table |
| 1258 | void enter_hash_lock() { ++_hash_lock; assert(_hash_lock < 99, "in too many hash tables?")do { if (!(_hash_lock < 99)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1258, "assert(" "_hash_lock < 99" ") failed", "in too many hash tables?" ); ::breakpoint(); } } while (0); } |
| 1259 | void exit_hash_lock() { --_hash_lock; assert(_hash_lock >= 0, "mispaired hash locks")do { if (!(_hash_lock >= 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1259, "assert(" "_hash_lock >= 0" ") failed", "mispaired hash locks" ); ::breakpoint(); } } while (0); } |
| 1260 | |
| 1261 | static void init_NodeProperty(); |
| 1262 | |
| 1263 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 1264 | const Node* _last_del; // The last deleted node. |
| 1265 | uint _del_tick; // Bumped when a deletion happens.. |
| 1266 | #endif |
| 1267 | #endif |
| 1268 | }; |
| 1269 | |
| 1270 | inline bool not_a_node(const Node* n) { |
| 1271 | if (n == NULL__null) return true; |
| 1272 | if (((intptr_t)n & 1) != 0) return true; // uninitialized, etc. |
| 1273 | if (*(address*)n == badAddress((address)::badAddressVal)) return true; // kill by Node::destruct |
| 1274 | return false; |
| 1275 | } |
| 1276 | |
| 1277 | //----------------------------------------------------------------------------- |
| 1278 | // Iterators over DU info, and associated Node functions. |
| 1279 | |
| 1280 | #if OPTO_DU_ITERATOR_ASSERT1 |
| 1281 | |
| 1282 | // Common code for assertion checking on DU iterators. |
| 1283 | class DUIterator_Common { |
| 1284 | #ifdef ASSERT1 |
| 1285 | protected: |
| 1286 | bool _vdui; // cached value of VerifyDUIterators |
| 1287 | const Node* _node; // the node containing the _out array |
| 1288 | uint _outcnt; // cached node->_outcnt |
| 1289 | uint _del_tick; // cached node->_del_tick |
| 1290 | Node* _last; // last value produced by the iterator |
| 1291 | |
| 1292 | void sample(const Node* node); // used by c'tor to set up for verifies |
| 1293 | void verify(const Node* node, bool at_end_ok = false); |
| 1294 | void verify_resync(); |
| 1295 | void reset(const DUIterator_Common& that); |
| 1296 | |
| 1297 | // The VDUI_ONLY macro protects code conditionalized on VerifyDUIterators |
| 1298 | #define I_VDUI_ONLY(i,x) { if ((i)._vdui) { x; } } |
| 1299 | #else |
| 1300 | #define I_VDUI_ONLY(i,x) { } |
| 1301 | #endif //ASSERT |
| 1302 | }; |
| 1303 | |
| 1304 | #define VDUI_ONLY(x) I_VDUI_ONLY(*this, x) |
| 1305 | |
| 1306 | // Default DU iterator. Allows appends onto the out array. |
| 1307 | // Allows deletion from the out array only at the current point. |
| 1308 | // Usage: |
| 1309 | // for (DUIterator i = x->outs(); x->has_out(i); i++) { |
| 1310 | // Node* y = x->out(i); |
| 1311 | // ... |
| 1312 | // } |
| 1313 | // Compiles in product mode to a unsigned integer index, which indexes |
| 1314 | // onto a repeatedly reloaded base pointer of x->_out. The loop predicate |
| 1315 | // also reloads x->_outcnt. If you delete, you must perform "--i" just |
| 1316 | // before continuing the loop. You must delete only the last-produced |
| 1317 | // edge. You must delete only a single copy of the last-produced edge, |
| 1318 | // or else you must delete all copies at once (the first time the edge |
| 1319 | // is produced by the iterator). |
| 1320 | class DUIterator : public DUIterator_Common { |
| 1321 | friend class Node; |
| 1322 | |
| 1323 | // This is the index which provides the product-mode behavior. |
| 1324 | // Whatever the product-mode version of the system does to the |
| 1325 | // DUI index is done to this index. All other fields in |
| 1326 | // this class are used only for assertion checking. |
| 1327 | uint _idx; |
| 1328 | |
| 1329 | #ifdef ASSERT1 |
| 1330 | uint _refresh_tick; // Records the refresh activity. |
| 1331 | |
| 1332 | void sample(const Node* node); // Initialize _refresh_tick etc. |
| 1333 | void verify(const Node* node, bool at_end_ok = false); |
| 1334 | void verify_increment(); // Verify an increment operation. |
| 1335 | void verify_resync(); // Verify that we can back up over a deletion. |
| 1336 | void verify_finish(); // Verify that the loop terminated properly. |
| 1337 | void refresh(); // Resample verification info. |
| 1338 | void reset(const DUIterator& that); // Resample after assignment. |
| 1339 | #endif |
| 1340 | |
| 1341 | DUIterator(const Node* node, int dummy_to_avoid_conversion) |
| 1342 | { _idx = 0; debug_only(sample(node))sample(node); } |
| 1343 | |
| 1344 | public: |
| 1345 | // initialize to garbage; clear _vdui to disable asserts |
| 1346 | DUIterator() |
| 1347 | { /*initialize to garbage*/ debug_only(_vdui = false)_vdui = false; } |
| 1348 | |
| 1349 | DUIterator(const DUIterator& that) |
| 1350 | { _idx = that._idx; debug_only(_vdui = false; reset(that))_vdui = false; reset(that); } |
| 1351 | |
| 1352 | void operator++(int dummy_to_specify_postfix_op) |
| 1353 | { _idx++; VDUI_ONLY(verify_increment()); } |
| 1354 | |
| 1355 | void operator--() |
| 1356 | { VDUI_ONLY(verify_resync()); --_idx; } |
| 1357 | |
| 1358 | ~DUIterator() |
| 1359 | { VDUI_ONLY(verify_finish()); } |
| 1360 | |
| 1361 | void operator=(const DUIterator& that) |
| 1362 | { _idx = that._idx; debug_only(reset(that))reset(that); } |
| 1363 | }; |
| 1364 | |
| 1365 | DUIterator Node::outs() const |
| 1366 | { return DUIterator(this, 0); } |
| 1367 | DUIterator& Node::refresh_out_pos(DUIterator& i) const |
| 1368 | { I_VDUI_ONLY(i, i.refresh()); return i; } |
| 1369 | bool Node::has_out(DUIterator& i) const |
| 1370 | { I_VDUI_ONLY(i, i.verify(this,true));return i._idx < _outcnt; } |
| 1371 | Node* Node::out(DUIterator& i) const |
| 1372 | { I_VDUI_ONLY(i, i.verify(this)); return debug_only(i._last=)i._last= _out[i._idx]; } |
| 1373 | |
| 1374 | |
| 1375 | // Faster DU iterator. Disallows insertions into the out array. |
| 1376 | // Allows deletion from the out array only at the current point. |
| 1377 | // Usage: |
| 1378 | // for (DUIterator_Fast imax, i = x->fast_outs(imax); i < imax; i++) { |
| 1379 | // Node* y = x->fast_out(i); |
| 1380 | // ... |
| 1381 | // } |
| 1382 | // Compiles in product mode to raw Node** pointer arithmetic, with |
| 1383 | // no reloading of pointers from the original node x. If you delete, |
| 1384 | // you must perform "--i; --imax" just before continuing the loop. |
| 1385 | // If you delete multiple copies of the same edge, you must decrement |
| 1386 | // imax, but not i, multiple times: "--i, imax -= num_edges". |
| 1387 | class DUIterator_Fast : public DUIterator_Common { |
| 1388 | friend class Node; |
| 1389 | friend class DUIterator_Last; |
| 1390 | |
| 1391 | // This is the pointer which provides the product-mode behavior. |
| 1392 | // Whatever the product-mode version of the system does to the |
| 1393 | // DUI pointer is done to this pointer. All other fields in |
| 1394 | // this class are used only for assertion checking. |
| 1395 | Node** _outp; |
| 1396 | |
| 1397 | #ifdef ASSERT1 |
| 1398 | void verify(const Node* node, bool at_end_ok = false); |
| 1399 | void verify_limit(); |
| 1400 | void verify_resync(); |
| 1401 | void verify_relimit(uint n); |
| 1402 | void reset(const DUIterator_Fast& that); |
| 1403 | #endif |
| 1404 | |
| 1405 | // Note: offset must be signed, since -1 is sometimes passed |
| 1406 | DUIterator_Fast(const Node* node, ptrdiff_t offset) |
| 1407 | { _outp = node->_out + offset; debug_only(sample(node))sample(node); } |
| 1408 | |
| 1409 | public: |
| 1410 | // initialize to garbage; clear _vdui to disable asserts |
| 1411 | DUIterator_Fast() |
| 1412 | { /*initialize to garbage*/ debug_only(_vdui = false)_vdui = false; } |
| 1413 | |
| 1414 | DUIterator_Fast(const DUIterator_Fast& that) |
| 1415 | { _outp = that._outp; debug_only(_vdui = false; reset(that))_vdui = false; reset(that); } |
| 1416 | |
| 1417 | void operator++(int dummy_to_specify_postfix_op) |
| 1418 | { _outp++; VDUI_ONLY(verify(_node, true)); } |
| 1419 | |
| 1420 | void operator--() |
| 1421 | { VDUI_ONLY(verify_resync()); --_outp; } |
| 1422 | |
| 1423 | void operator-=(uint n) // applied to the limit only |
| 1424 | { _outp -= n; VDUI_ONLY(verify_relimit(n)); } |
| 1425 | |
| 1426 | bool operator<(DUIterator_Fast& limit) { |
| 1427 | I_VDUI_ONLY(*this, this->verify(_node, true)); |
| 1428 | I_VDUI_ONLY(limit, limit.verify_limit()); |
| 1429 | return _outp < limit._outp; |
| 1430 | } |
| 1431 | |
| 1432 | void operator=(const DUIterator_Fast& that) |
| 1433 | { _outp = that._outp; debug_only(reset(that))reset(that); } |
| 1434 | }; |
| 1435 | |
| 1436 | DUIterator_Fast Node::fast_outs(DUIterator_Fast& imax) const { |
| 1437 | // Assign a limit pointer to the reference argument: |
| 1438 | imax = DUIterator_Fast(this, (ptrdiff_t)_outcnt); |
| 1439 | // Return the base pointer: |
| 1440 | return DUIterator_Fast(this, 0); |
| 1441 | } |
| 1442 | Node* Node::fast_out(DUIterator_Fast& i) const { |
| 1443 | I_VDUI_ONLY(i, i.verify(this)); |
| 1444 | return debug_only(i._last=)i._last= *i._outp; |
| 1445 | } |
| 1446 | |
| 1447 | |
| 1448 | // Faster DU iterator. Requires each successive edge to be removed. |
| 1449 | // Does not allow insertion of any edges. |
| 1450 | // Usage: |
| 1451 | // for (DUIterator_Last imin, i = x->last_outs(imin); i >= imin; i -= num_edges) { |
| 1452 | // Node* y = x->last_out(i); |
| 1453 | // ... |
| 1454 | // } |
| 1455 | // Compiles in product mode to raw Node** pointer arithmetic, with |
| 1456 | // no reloading of pointers from the original node x. |
| 1457 | class DUIterator_Last : private DUIterator_Fast { |
| 1458 | friend class Node; |
| 1459 | |
| 1460 | #ifdef ASSERT1 |
| 1461 | void verify(const Node* node, bool at_end_ok = false); |
| 1462 | void verify_limit(); |
| 1463 | void verify_step(uint num_edges); |
| 1464 | #endif |
| 1465 | |
| 1466 | // Note: offset must be signed, since -1 is sometimes passed |
| 1467 | DUIterator_Last(const Node* node, ptrdiff_t offset) |
| 1468 | : DUIterator_Fast(node, offset) { } |
| 1469 | |
| 1470 | void operator++(int dummy_to_specify_postfix_op) {} // do not use |
| 1471 | void operator<(int) {} // do not use |
| 1472 | |
| 1473 | public: |
| 1474 | DUIterator_Last() { } |
| 1475 | // initialize to garbage |
| 1476 | |
| 1477 | DUIterator_Last(const DUIterator_Last& that) = default; |
| 1478 | |
| 1479 | void operator--() |
| 1480 | { _outp--; VDUI_ONLY(verify_step(1)); } |
| 1481 | |
| 1482 | void operator-=(uint n) |
| 1483 | { _outp -= n; VDUI_ONLY(verify_step(n)); } |
| 1484 | |
| 1485 | bool operator>=(DUIterator_Last& limit) { |
| 1486 | I_VDUI_ONLY(*this, this->verify(_node, true)); |
| 1487 | I_VDUI_ONLY(limit, limit.verify_limit()); |
| 1488 | return _outp >= limit._outp; |
| 1489 | } |
| 1490 | |
| 1491 | DUIterator_Last& operator=(const DUIterator_Last& that) = default; |
| 1492 | }; |
| 1493 | |
| 1494 | DUIterator_Last Node::last_outs(DUIterator_Last& imin) const { |
| 1495 | // Assign a limit pointer to the reference argument: |
| 1496 | imin = DUIterator_Last(this, 0); |
| 1497 | // Return the initial pointer: |
| 1498 | return DUIterator_Last(this, (ptrdiff_t)_outcnt - 1); |
| 1499 | } |
| 1500 | Node* Node::last_out(DUIterator_Last& i) const { |
| 1501 | I_VDUI_ONLY(i, i.verify(this)); |
| 1502 | return debug_only(i._last=)i._last= *i._outp; |
| 1503 | } |
| 1504 | |
| 1505 | #endif //OPTO_DU_ITERATOR_ASSERT |
| 1506 | |
| 1507 | #undef I_VDUI_ONLY |
| 1508 | #undef VDUI_ONLY |
| 1509 | |
| 1510 | // An Iterator that truly follows the iterator pattern. Doesn't |
| 1511 | // support deletion but could be made to. |
| 1512 | // |
| 1513 | // for (SimpleDUIterator i(n); i.has_next(); i.next()) { |
| 1514 | // Node* m = i.get(); |
| 1515 | // |
| 1516 | class SimpleDUIterator : public StackObj { |
| 1517 | private: |
| 1518 | Node* node; |
| 1519 | DUIterator_Fast i; |
| 1520 | DUIterator_Fast imax; |
| 1521 | public: |
| 1522 | SimpleDUIterator(Node* n): node(n), i(n->fast_outs(imax)) {} |
| 1523 | bool has_next() { return i < imax; } |
| 1524 | void next() { i++; } |
| 1525 | Node* get() { return node->fast_out(i); } |
| 1526 | }; |
| 1527 | |
| 1528 | |
| 1529 | //----------------------------------------------------------------------------- |
| 1530 | // Map dense integer indices to Nodes. Uses classic doubling-array trick. |
| 1531 | // Abstractly provides an infinite array of Node*'s, initialized to NULL. |
| 1532 | // Note that the constructor just zeros things, and since I use Arena |
| 1533 | // allocation I do not need a destructor to reclaim storage. |
| 1534 | class Node_Array : public ResourceObj { |
| 1535 | friend class VMStructs; |
| 1536 | protected: |
| 1537 | Arena* _a; // Arena to allocate in |
| 1538 | uint _max; |
| 1539 | Node** _nodes; |
| 1540 | void grow( uint i ); // Grow array node to fit |
| 1541 | public: |
| 1542 | Node_Array(Arena* a, uint max = OptoNodeListSize) : _a(a), _max(max) { |
| 1543 | _nodes = NEW_ARENA_ARRAY(a, Node*, max)(Node**) (a)->Amalloc((max) * sizeof(Node*)); |
| 1544 | clear(); |
| 1545 | } |
| 1546 | |
| 1547 | Node_Array(Node_Array* na) : _a(na->_a), _max(na->_max), _nodes(na->_nodes) {} |
| 1548 | Node *operator[] ( uint i ) const // Lookup, or NULL for not mapped |
| 1549 | { return (i<_max) ? _nodes[i] : (Node*)NULL__null; } |
| 1550 | Node* at(uint i) const { assert(i<_max,"oob")do { if (!(i<_max)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1550, "assert(" "i<_max" ") failed", "oob"); ::breakpoint (); } } while (0); return _nodes[i]; } |
| 1551 | Node** adr() { return _nodes; } |
| 1552 | // Extend the mapping: index i maps to Node *n. |
| 1553 | void map( uint i, Node *n ) { if( i>=_max ) grow(i); _nodes[i] = n; } |
| 1554 | void insert( uint i, Node *n ); |
| 1555 | void remove( uint i ); // Remove, preserving order |
| 1556 | // Clear all entries in _nodes to NULL but keep storage |
| 1557 | void clear() { |
| 1558 | Copy::zero_to_bytes(_nodes, _max * sizeof(Node*)); |
| 1559 | } |
| 1560 | |
| 1561 | uint Size() const { return _max; } |
| 1562 | void dump() const; |
| 1563 | }; |
| 1564 | |
| 1565 | class Node_List : public Node_Array { |
| 1566 | friend class VMStructs; |
| 1567 | uint _cnt; |
| 1568 | public: |
| 1569 | Node_List(uint max = OptoNodeListSize) : Node_Array(Thread::current()->resource_area(), max), _cnt(0) {} |
| 1570 | Node_List(Arena *a, uint max = OptoNodeListSize) : Node_Array(a, max), _cnt(0) {} |
| 1571 | bool contains(const Node* n) const { |
| 1572 | for (uint e = 0; e < size(); e++) { |
| 1573 | if (at(e) == n) return true; |
| 1574 | } |
| 1575 | return false; |
| 1576 | } |
| 1577 | void insert( uint i, Node *n ) { Node_Array::insert(i,n); _cnt++; } |
| 1578 | void remove( uint i ) { Node_Array::remove(i); _cnt--; } |
| 1579 | void push( Node *b ) { map(_cnt++,b); } |
| 1580 | void yank( Node *n ); // Find and remove |
| 1581 | Node *pop() { return _nodes[--_cnt]; } |
| 1582 | void clear() { _cnt = 0; Node_Array::clear(); } // retain storage |
| 1583 | void copy(const Node_List& from) { |
| 1584 | if (from._max > _max) { |
| 1585 | grow(from._max); |
| 1586 | } |
| 1587 | _cnt = from._cnt; |
| 1588 | Copy::conjoint_words_to_higher((HeapWord*)&from._nodes[0], (HeapWord*)&_nodes[0], from._max * sizeof(Node*)); |
| 1589 | } |
| 1590 | |
| 1591 | uint size() const { return _cnt; } |
| 1592 | void dump() const; |
| 1593 | void dump_simple() const; |
| 1594 | }; |
| 1595 | |
| 1596 | //------------------------------Unique_Node_List------------------------------- |
| 1597 | class Unique_Node_List : public Node_List { |
| 1598 | friend class VMStructs; |
| 1599 | VectorSet _in_worklist; |
| 1600 | uint _clock_index; // Index in list where to pop from next |
| 1601 | public: |
| 1602 | Unique_Node_List() : Node_List(), _clock_index(0) {} |
| 1603 | Unique_Node_List(Arena *a) : Node_List(a), _in_worklist(a), _clock_index(0) {} |
| 1604 | |
| 1605 | void remove( Node *n ); |
| 1606 | bool member( Node *n ) { return _in_worklist.test(n->_idx) != 0; } |
| 1607 | VectorSet& member_set(){ return _in_worklist; } |
| 1608 | |
| 1609 | void push(Node* b) { |
| 1610 | if( !_in_worklist.test_set(b->_idx) ) |
| 1611 | Node_List::push(b); |
| 1612 | } |
| 1613 | Node *pop() { |
| 1614 | if( _clock_index >= size() ) _clock_index = 0; |
| 1615 | Node *b = at(_clock_index); |
| 1616 | map( _clock_index, Node_List::pop()); |
| 1617 | if (size() != 0) _clock_index++; // Always start from 0 |
| 1618 | _in_worklist.remove(b->_idx); |
| 1619 | return b; |
| 1620 | } |
| 1621 | Node *remove(uint i) { |
| 1622 | Node *b = Node_List::at(i); |
| 1623 | _in_worklist.remove(b->_idx); |
| 1624 | map(i,Node_List::pop()); |
| 1625 | return b; |
| 1626 | } |
| 1627 | void yank(Node *n) { |
| 1628 | _in_worklist.remove(n->_idx); |
| 1629 | Node_List::yank(n); |
| 1630 | } |
| 1631 | void clear() { |
| 1632 | _in_worklist.clear(); // Discards storage but grows automatically |
| 1633 | Node_List::clear(); |
| 1634 | _clock_index = 0; |
| 1635 | } |
| 1636 | |
| 1637 | // Used after parsing to remove useless nodes before Iterative GVN |
| 1638 | void remove_useless_nodes(VectorSet& useful); |
| 1639 | |
| 1640 | bool contains(const Node* n) const { |
| 1641 | fatal("use faster member() instead")do { (*g_assert_poison) = 'X';; report_fatal(INTERNAL_ERROR, "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1641, "use faster member() instead"); ::breakpoint(); } while (0); |
| 1642 | return false; |
| 1643 | } |
| 1644 | |
| 1645 | #ifndef PRODUCT |
| 1646 | void print_set() const { _in_worklist.print(); } |
| 1647 | #endif |
| 1648 | }; |
| 1649 | |
| 1650 | // Inline definition of Compile::record_for_igvn must be deferred to this point. |
| 1651 | inline void Compile::record_for_igvn(Node* n) { |
| 1652 | _for_igvn->push(n); |
| 1653 | } |
| 1654 | |
| 1655 | //------------------------------Node_Stack------------------------------------- |
| 1656 | class Node_Stack { |
| 1657 | friend class VMStructs; |
| 1658 | protected: |
| 1659 | struct INode { |
| 1660 | Node *node; // Processed node |
| 1661 | uint indx; // Index of next node's child |
| 1662 | }; |
| 1663 | INode *_inode_top; // tos, stack grows up |
| 1664 | INode *_inode_max; // End of _inodes == _inodes + _max |
| 1665 | INode *_inodes; // Array storage for the stack |
| 1666 | Arena *_a; // Arena to allocate in |
| 1667 | void grow(); |
| 1668 | public: |
| 1669 | Node_Stack(int size) { |
| 1670 | size_t max = (size > OptoNodeListSize) ? size : OptoNodeListSize; |
| 1671 | _a = Thread::current()->resource_area(); |
| 1672 | _inodes = NEW_ARENA_ARRAY( _a, INode, max )(INode*) (_a)->Amalloc((max) * sizeof(INode)); |
| 1673 | _inode_max = _inodes + max; |
| 1674 | _inode_top = _inodes - 1; // stack is empty |
| 1675 | } |
| 1676 | |
| 1677 | Node_Stack(Arena *a, int size) : _a(a) { |
| 1678 | size_t max = (size > OptoNodeListSize) ? size : OptoNodeListSize; |
| 1679 | _inodes = NEW_ARENA_ARRAY( _a, INode, max )(INode*) (_a)->Amalloc((max) * sizeof(INode)); |
| 1680 | _inode_max = _inodes + max; |
| 1681 | _inode_top = _inodes - 1; // stack is empty |
| 1682 | } |
| 1683 | |
| 1684 | void pop() { |
| 1685 | assert(_inode_top >= _inodes, "node stack underflow")do { if (!(_inode_top >= _inodes)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1685, "assert(" "_inode_top >= _inodes" ") failed", "node stack underflow" ); ::breakpoint(); } } while (0); |
| 1686 | --_inode_top; |
| 1687 | } |
| 1688 | void push(Node *n, uint i) { |
| 1689 | ++_inode_top; |
| 1690 | if (_inode_top >= _inode_max) grow(); |
| 1691 | INode *top = _inode_top; // optimization |
| 1692 | top->node = n; |
| 1693 | top->indx = i; |
| 1694 | } |
| 1695 | Node *node() const { |
| 1696 | return _inode_top->node; |
| 1697 | } |
| 1698 | Node* node_at(uint i) const { |
| 1699 | assert(_inodes + i <= _inode_top, "in range")do { if (!(_inodes + i <= _inode_top)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1699, "assert(" "_inodes + i <= _inode_top" ") failed", "in range" ); ::breakpoint(); } } while (0); |
| 1700 | return _inodes[i].node; |
| 1701 | } |
| 1702 | uint index() const { |
| 1703 | return _inode_top->indx; |
| 1704 | } |
| 1705 | uint index_at(uint i) const { |
| 1706 | assert(_inodes + i <= _inode_top, "in range")do { if (!(_inodes + i <= _inode_top)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1706, "assert(" "_inodes + i <= _inode_top" ") failed", "in range" ); ::breakpoint(); } } while (0); |
| 1707 | return _inodes[i].indx; |
| 1708 | } |
| 1709 | void set_node(Node *n) { |
| 1710 | _inode_top->node = n; |
| 1711 | } |
| 1712 | void set_index(uint i) { |
| 1713 | _inode_top->indx = i; |
| 1714 | } |
| 1715 | uint size_max() const { return (uint)pointer_delta(_inode_max, _inodes, sizeof(INode)); } // Max size |
| 1716 | uint size() const { return (uint)pointer_delta((_inode_top+1), _inodes, sizeof(INode)); } // Current size |
| 1717 | bool is_nonempty() const { return (_inode_top >= _inodes); } |
| 1718 | bool is_empty() const { return (_inode_top < _inodes); } |
| 1719 | void clear() { _inode_top = _inodes - 1; } // retain storage |
| 1720 | |
| 1721 | // Node_Stack is used to map nodes. |
| 1722 | Node* find(uint idx) const; |
| 1723 | }; |
| 1724 | |
| 1725 | |
| 1726 | //-----------------------------Node_Notes-------------------------------------- |
| 1727 | // Debugging or profiling annotations loosely and sparsely associated |
| 1728 | // with some nodes. See Compile::node_notes_at for the accessor. |
| 1729 | class Node_Notes { |
| 1730 | friend class VMStructs; |
| 1731 | JVMState* _jvms; |
| 1732 | |
| 1733 | public: |
| 1734 | Node_Notes(JVMState* jvms = NULL__null) { |
| 1735 | _jvms = jvms; |
| 1736 | } |
| 1737 | |
| 1738 | JVMState* jvms() { return _jvms; } |
| 1739 | void set_jvms(JVMState* x) { _jvms = x; } |
| 1740 | |
| 1741 | // True if there is nothing here. |
| 1742 | bool is_clear() { |
| 1743 | return (_jvms == NULL__null); |
| 1744 | } |
| 1745 | |
| 1746 | // Make there be nothing here. |
| 1747 | void clear() { |
| 1748 | _jvms = NULL__null; |
| 1749 | } |
| 1750 | |
| 1751 | // Make a new, clean node notes. |
| 1752 | static Node_Notes* make(Compile* C) { |
| 1753 | Node_Notes* nn = NEW_ARENA_ARRAY(C->comp_arena(), Node_Notes, 1)(Node_Notes*) (C->comp_arena())->Amalloc((1) * sizeof(Node_Notes )); |
| 1754 | nn->clear(); |
| 1755 | return nn; |
| 1756 | } |
| 1757 | |
| 1758 | Node_Notes* clone(Compile* C) { |
| 1759 | Node_Notes* nn = NEW_ARENA_ARRAY(C->comp_arena(), Node_Notes, 1)(Node_Notes*) (C->comp_arena())->Amalloc((1) * sizeof(Node_Notes )); |
| 1760 | (*nn) = (*this); |
| 1761 | return nn; |
| 1762 | } |
| 1763 | |
| 1764 | // Absorb any information from source. |
| 1765 | bool update_from(Node_Notes* source) { |
| 1766 | bool changed = false; |
| 1767 | if (source != NULL__null) { |
| 1768 | if (source->jvms() != NULL__null) { |
| 1769 | set_jvms(source->jvms()); |
| 1770 | changed = true; |
| 1771 | } |
| 1772 | } |
| 1773 | return changed; |
| 1774 | } |
| 1775 | }; |
| 1776 | |
| 1777 | // Inlined accessors for Compile::node_nodes that require the preceding class: |
| 1778 | inline Node_Notes* |
| 1779 | Compile::locate_node_notes(GrowableArray<Node_Notes*>* arr, |
| 1780 | int idx, bool can_grow) { |
| 1781 | assert(idx >= 0, "oob")do { if (!(idx >= 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1781, "assert(" "idx >= 0" ") failed", "oob"); ::breakpoint (); } } while (0); |
| 1782 | int block_idx = (idx >> _log2_node_notes_block_size); |
| 1783 | int grow_by = (block_idx - (arr == NULL__null? 0: arr->length())); |
| 1784 | if (grow_by >= 0) { |
| 1785 | if (!can_grow) return NULL__null; |
| 1786 | grow_node_notes(arr, grow_by + 1); |
| 1787 | } |
| 1788 | if (arr == NULL__null) return NULL__null; |
| 1789 | // (Every element of arr is a sub-array of length _node_notes_block_size.) |
| 1790 | return arr->at(block_idx) + (idx & (_node_notes_block_size-1)); |
| 1791 | } |
| 1792 | |
| 1793 | inline bool |
| 1794 | Compile::set_node_notes_at(int idx, Node_Notes* value) { |
| 1795 | if (value == NULL__null || value->is_clear()) |
| 1796 | return false; // nothing to write => write nothing |
| 1797 | Node_Notes* loc = locate_node_notes(_node_note_array, idx, true); |
| 1798 | assert(loc != NULL, "")do { if (!(loc != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1798, "assert(" "loc != __null" ") failed", ""); ::breakpoint (); } } while (0); |
| 1799 | return loc->update_from(value); |
| 1800 | } |
| 1801 | |
| 1802 | |
| 1803 | //------------------------------TypeNode--------------------------------------- |
| 1804 | // Node with a Type constant. |
| 1805 | class TypeNode : public Node { |
| 1806 | protected: |
| 1807 | virtual uint hash() const; // Check the type |
| 1808 | virtual bool cmp( const Node &n ) const; |
| 1809 | virtual uint size_of() const; // Size is bigger |
| 1810 | const Type* const _type; |
| 1811 | public: |
| 1812 | void set_type(const Type* t) { |
| 1813 | assert(t != NULL, "sanity")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1813, "assert(" "t != __null" ") failed", "sanity"); ::breakpoint (); } } while (0); |
| 1814 | debug_only(uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH)uint check_hash = (VerifyHashTableKeys && _hash_lock) ? hash() : NO_HASH; |
| 1815 | *(const Type**)&_type = t; // cast away const-ness |
| 1816 | // If this node is in the hash table, make sure it doesn't need a rehash. |
| 1817 | assert(check_hash == NO_HASH || check_hash == hash(), "type change must preserve hash code")do { if (!(check_hash == NO_HASH || check_hash == hash())) { ( *g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1817, "assert(" "check_hash == NO_HASH || check_hash == hash()" ") failed", "type change must preserve hash code"); ::breakpoint (); } } while (0); |
| 1818 | } |
| 1819 | const Type* type() const { assert(_type != NULL, "sanity")do { if (!(_type != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1819, "assert(" "_type != __null" ") failed", "sanity"); :: breakpoint(); } } while (0); return _type; }; |
| 1820 | TypeNode( const Type *t, uint required ) : Node(required), _type(t) { |
| 1821 | init_class_id(Class_Type); |
| 1822 | } |
| 1823 | virtual const Type* Value(PhaseGVN* phase) const; |
| 1824 | virtual const Type *bottom_type() const; |
| 1825 | virtual uint ideal_reg() const; |
| 1826 | #ifndef PRODUCT |
| 1827 | virtual void dump_spec(outputStream *st) const; |
| 1828 | virtual void dump_compact_spec(outputStream *st) const; |
| 1829 | #endif |
| 1830 | }; |
| 1831 | |
| 1832 | #include "opto/opcodes.hpp" |
| 1833 | |
| 1834 | #define Op_IL(op)inline int Op_op(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1834, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_opI ; } return Op_opL; } \ |
| 1835 | inline int Op_ ## op(BasicType bt) { \ |
| 1836 | assert(bt == T_INT || bt == T_LONG, "only for int or longs")do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1836, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); \ |
| 1837 | if (bt == T_INT) { \ |
| 1838 | return Op_## op ## I; \ |
| 1839 | } \ |
| 1840 | return Op_## op ## L; \ |
| 1841 | } |
| 1842 | |
| 1843 | Op_IL(Add)inline int Op_Add(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1843, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_AddI ; } return Op_AddL; } |
| 1844 | Op_IL(Sub)inline int Op_Sub(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1844, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_SubI ; } return Op_SubL; } |
| 1845 | Op_IL(Mul)inline int Op_Mul(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1845, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_MulI ; } return Op_MulL; } |
| 1846 | Op_IL(URShift)inline int Op_URShift(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1846, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_URShiftI ; } return Op_URShiftL; } |
| 1847 | Op_IL(LShift)inline int Op_LShift(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1847, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_LShiftI ; } return Op_LShiftL; } |
| 1848 | Op_IL(Xor)inline int Op_Xor(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1848, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_XorI ; } return Op_XorL; } |
| 1849 | Op_IL(Cmp)inline int Op_Cmp(BasicType bt) { do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1849, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); if (bt == T_INT) { return Op_CmpI ; } return Op_CmpL; } |
| 1850 | |
| 1851 | inline int Op_Cmp_unsigned(BasicType bt) { |
| 1852 | assert(bt == T_INT || bt == T_LONG, "only for int or longs")do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1852, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); |
| 1853 | if (bt == T_INT) { |
| 1854 | return Op_CmpU; |
| 1855 | } |
| 1856 | return Op_CmpUL; |
| 1857 | } |
| 1858 | |
| 1859 | inline int Op_Cast(BasicType bt) { |
| 1860 | assert(bt == T_INT || bt == T_LONG, "only for int or longs")do { if (!(bt == T_INT || bt == T_LONG)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/node.hpp" , 1860, "assert(" "bt == T_INT || bt == T_LONG" ") failed", "only for int or longs" ); ::breakpoint(); } } while (0); |
| 1861 | if (bt == T_INT) { |
| 1862 | return Op_CastII; |
| 1863 | } |
| 1864 | return Op_CastLL; |
| 1865 | } |
| 1866 | |
| 1867 | #endif // SHARE_OPTO_NODE_HPP |
| 1 | /* |
| 2 | * Copyright (c) 1998, 2021, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_OPTO_LOOPNODE_HPP |
| 26 | #define SHARE_OPTO_LOOPNODE_HPP |
| 27 | |
| 28 | #include "opto/cfgnode.hpp" |
| 29 | #include "opto/multnode.hpp" |
| 30 | #include "opto/phaseX.hpp" |
| 31 | #include "opto/subnode.hpp" |
| 32 | #include "opto/type.hpp" |
| 33 | |
| 34 | class CmpNode; |
| 35 | class BaseCountedLoopEndNode; |
| 36 | class CountedLoopNode; |
| 37 | class IdealLoopTree; |
| 38 | class LoopNode; |
| 39 | class Node; |
| 40 | class OuterStripMinedLoopEndNode; |
| 41 | class PathFrequency; |
| 42 | class PhaseIdealLoop; |
| 43 | class CountedLoopReserveKit; |
| 44 | class VectorSet; |
| 45 | class Invariance; |
| 46 | struct small_cache; |
| 47 | |
| 48 | // |
| 49 | // I D E A L I Z E D L O O P S |
| 50 | // |
| 51 | // Idealized loops are the set of loops I perform more interesting |
| 52 | // transformations on, beyond simple hoisting. |
| 53 | |
| 54 | //------------------------------LoopNode--------------------------------------- |
| 55 | // Simple loop header. Fall in path on left, loop-back path on right. |
| 56 | class LoopNode : public RegionNode { |
| 57 | // Size is bigger to hold the flags. However, the flags do not change |
| 58 | // the semantics so it does not appear in the hash & cmp functions. |
| 59 | virtual uint size_of() const { return sizeof(*this); } |
| 60 | protected: |
| 61 | uint _loop_flags; |
| 62 | // Names for flag bitfields |
| 63 | enum { Normal=0, Pre=1, Main=2, Post=3, PreMainPostFlagsMask=3, |
| 64 | MainHasNoPreLoop = 1<<2, |
| 65 | HasExactTripCount = 1<<3, |
| 66 | InnerLoop = 1<<4, |
| 67 | PartialPeelLoop = 1<<5, |
| 68 | PartialPeelFailed = 1<<6, |
| 69 | HasReductions = 1<<7, |
| 70 | WasSlpAnalyzed = 1<<8, |
| 71 | PassedSlpAnalysis = 1<<9, |
| 72 | DoUnrollOnly = 1<<10, |
| 73 | VectorizedLoop = 1<<11, |
| 74 | HasAtomicPostLoop = 1<<12, |
| 75 | HasRangeChecks = 1<<13, |
| 76 | IsMultiversioned = 1<<14, |
| 77 | StripMined = 1<<15, |
| 78 | SubwordLoop = 1<<16, |
| 79 | ProfileTripFailed = 1<<17, |
| 80 | LoopNestInnerLoop = 1 << 18, |
| 81 | LoopNestLongOuterLoop = 1 << 19}; |
| 82 | char _unswitch_count; |
| 83 | enum { _unswitch_max=3 }; |
| 84 | char _postloop_flags; |
| 85 | enum { LoopNotRCEChecked = 0, LoopRCEChecked = 1, RCEPostLoop = 2 }; |
| 86 | |
| 87 | // Expected trip count from profile data |
| 88 | float _profile_trip_cnt; |
| 89 | |
| 90 | public: |
| 91 | // Names for edge indices |
| 92 | enum { Self=0, EntryControl, LoopBackControl }; |
| 93 | |
| 94 | bool is_inner_loop() const { return _loop_flags & InnerLoop; } |
| 95 | void set_inner_loop() { _loop_flags |= InnerLoop; } |
| 96 | |
| 97 | bool range_checks_present() const { return _loop_flags & HasRangeChecks; } |
| 98 | bool is_multiversioned() const { return _loop_flags & IsMultiversioned; } |
| 99 | bool is_vectorized_loop() const { return _loop_flags & VectorizedLoop; } |
| 100 | bool is_partial_peel_loop() const { return _loop_flags & PartialPeelLoop; } |
| 101 | void set_partial_peel_loop() { _loop_flags |= PartialPeelLoop; } |
| 102 | bool partial_peel_has_failed() const { return _loop_flags & PartialPeelFailed; } |
| 103 | bool is_strip_mined() const { return _loop_flags & StripMined; } |
| 104 | bool is_profile_trip_failed() const { return _loop_flags & ProfileTripFailed; } |
| 105 | bool is_subword_loop() const { return _loop_flags & SubwordLoop; } |
| 106 | bool is_loop_nest_inner_loop() const { return _loop_flags & LoopNestInnerLoop; } |
| 107 | bool is_loop_nest_outer_loop() const { return _loop_flags & LoopNestLongOuterLoop; } |
| 108 | |
| 109 | void mark_partial_peel_failed() { _loop_flags |= PartialPeelFailed; } |
| 110 | void mark_has_reductions() { _loop_flags |= HasReductions; } |
| 111 | void mark_was_slp() { _loop_flags |= WasSlpAnalyzed; } |
| 112 | void mark_passed_slp() { _loop_flags |= PassedSlpAnalysis; } |
| 113 | void mark_do_unroll_only() { _loop_flags |= DoUnrollOnly; } |
| 114 | void mark_loop_vectorized() { _loop_flags |= VectorizedLoop; } |
| 115 | void mark_has_atomic_post_loop() { _loop_flags |= HasAtomicPostLoop; } |
| 116 | void mark_has_range_checks() { _loop_flags |= HasRangeChecks; } |
| 117 | void mark_is_multiversioned() { _loop_flags |= IsMultiversioned; } |
| 118 | void mark_strip_mined() { _loop_flags |= StripMined; } |
| 119 | void clear_strip_mined() { _loop_flags &= ~StripMined; } |
| 120 | void mark_profile_trip_failed() { _loop_flags |= ProfileTripFailed; } |
| 121 | void mark_subword_loop() { _loop_flags |= SubwordLoop; } |
| 122 | void mark_loop_nest_inner_loop() { _loop_flags |= LoopNestInnerLoop; } |
| 123 | void mark_loop_nest_outer_loop() { _loop_flags |= LoopNestLongOuterLoop; } |
| 124 | |
| 125 | int unswitch_max() { return _unswitch_max; } |
| 126 | int unswitch_count() { return _unswitch_count; } |
| 127 | |
| 128 | int has_been_range_checked() const { return _postloop_flags & LoopRCEChecked; } |
| 129 | void set_has_been_range_checked() { _postloop_flags |= LoopRCEChecked; } |
| 130 | int is_rce_post_loop() const { return _postloop_flags & RCEPostLoop; } |
| 131 | void set_is_rce_post_loop() { _postloop_flags |= RCEPostLoop; } |
| 132 | |
| 133 | void set_unswitch_count(int val) { |
| 134 | assert (val <= unswitch_max(), "too many unswitches")do { if (!(val <= unswitch_max())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 134, "assert(" "val <= unswitch_max()" ") failed", "too many unswitches" ); ::breakpoint(); } } while (0); |
| 135 | _unswitch_count = val; |
| 136 | } |
| 137 | |
| 138 | void set_profile_trip_cnt(float ptc) { _profile_trip_cnt = ptc; } |
| 139 | float profile_trip_cnt() { return _profile_trip_cnt; } |
| 140 | |
| 141 | LoopNode(Node *entry, Node *backedge) |
| 142 | : RegionNode(3), _loop_flags(0), _unswitch_count(0), |
| 143 | _postloop_flags(0), _profile_trip_cnt(COUNT_UNKNOWN(-1.0f)) { |
| 144 | init_class_id(Class_Loop); |
| 145 | init_req(EntryControl, entry); |
| 146 | init_req(LoopBackControl, backedge); |
| 147 | } |
| 148 | |
| 149 | virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); |
| 150 | virtual int Opcode() const; |
| 151 | bool can_be_counted_loop(PhaseTransform* phase) const { |
| 152 | return req() == 3 && in(0) != NULL__null && |
| 153 | in(1) != NULL__null && phase->type(in(1)) != Type::TOP && |
| 154 | in(2) != NULL__null && phase->type(in(2)) != Type::TOP; |
| 155 | } |
| 156 | bool is_valid_counted_loop(BasicType bt) const; |
| 157 | #ifndef PRODUCT |
| 158 | virtual void dump_spec(outputStream *st) const; |
| 159 | #endif |
| 160 | |
| 161 | void verify_strip_mined(int expect_skeleton) const NOT_DEBUG_RETURN; |
| 162 | virtual LoopNode* skip_strip_mined(int expect_skeleton = 1) { return this; } |
| 163 | virtual IfTrueNode* outer_loop_tail() const { ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 163); ::breakpoint(); } while (0); return NULL__null; } |
| 164 | virtual OuterStripMinedLoopEndNode* outer_loop_end() const { ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 164); ::breakpoint(); } while (0); return NULL__null; } |
| 165 | virtual IfFalseNode* outer_loop_exit() const { ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 165); ::breakpoint(); } while (0); return NULL__null; } |
| 166 | virtual SafePointNode* outer_safepoint() const { ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here( "/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 166); ::breakpoint(); } while (0); return NULL__null; } |
| 167 | }; |
| 168 | |
| 169 | //------------------------------Counted Loops---------------------------------- |
| 170 | // Counted loops are all trip-counted loops, with exactly 1 trip-counter exit |
| 171 | // path (and maybe some other exit paths). The trip-counter exit is always |
| 172 | // last in the loop. The trip-counter have to stride by a constant; |
| 173 | // the exit value is also loop invariant. |
| 174 | |
| 175 | // CountedLoopNodes and CountedLoopEndNodes come in matched pairs. The |
| 176 | // CountedLoopNode has the incoming loop control and the loop-back-control |
| 177 | // which is always the IfTrue before the matching CountedLoopEndNode. The |
| 178 | // CountedLoopEndNode has an incoming control (possibly not the |
| 179 | // CountedLoopNode if there is control flow in the loop), the post-increment |
| 180 | // trip-counter value, and the limit. The trip-counter value is always of |
| 181 | // the form (Op old-trip-counter stride). The old-trip-counter is produced |
| 182 | // by a Phi connected to the CountedLoopNode. The stride is constant. |
| 183 | // The Op is any commutable opcode, including Add, Mul, Xor. The |
| 184 | // CountedLoopEndNode also takes in the loop-invariant limit value. |
| 185 | |
| 186 | // From a CountedLoopNode I can reach the matching CountedLoopEndNode via the |
| 187 | // loop-back control. From CountedLoopEndNodes I can reach CountedLoopNodes |
| 188 | // via the old-trip-counter from the Op node. |
| 189 | |
| 190 | //------------------------------CountedLoopNode-------------------------------- |
| 191 | // CountedLoopNodes head simple counted loops. CountedLoopNodes have as |
| 192 | // inputs the incoming loop-start control and the loop-back control, so they |
| 193 | // act like RegionNodes. They also take in the initial trip counter, the |
| 194 | // loop-invariant stride and the loop-invariant limit value. CountedLoopNodes |
| 195 | // produce a loop-body control and the trip counter value. Since |
| 196 | // CountedLoopNodes behave like RegionNodes I still have a standard CFG model. |
| 197 | |
| 198 | class BaseCountedLoopNode : public LoopNode { |
| 199 | public: |
| 200 | BaseCountedLoopNode(Node *entry, Node *backedge) |
| 201 | : LoopNode(entry, backedge) { |
| 202 | } |
| 203 | |
| 204 | Node *init_control() const { return in(EntryControl); } |
| 205 | Node *back_control() const { return in(LoopBackControl); } |
| 206 | |
| 207 | Node* init_trip() const; |
| 208 | Node* stride() const; |
| 209 | bool stride_is_con() const; |
| 210 | Node* limit() const; |
| 211 | Node* incr() const; |
| 212 | Node* phi() const; |
| 213 | |
| 214 | BaseCountedLoopEndNode* loopexit_or_null() const; |
| 215 | BaseCountedLoopEndNode* loopexit() const; |
| 216 | |
| 217 | virtual BasicType bt() const = 0; |
| 218 | |
| 219 | jlong stride_con() const; |
| 220 | |
| 221 | static BaseCountedLoopNode* make(Node* entry, Node* backedge, BasicType bt); |
| 222 | }; |
| 223 | |
| 224 | |
| 225 | class CountedLoopNode : public BaseCountedLoopNode { |
| 226 | // Size is bigger to hold _main_idx. However, _main_idx does not change |
| 227 | // the semantics so it does not appear in the hash & cmp functions. |
| 228 | virtual uint size_of() const { return sizeof(*this); } |
| 229 | |
| 230 | // For Pre- and Post-loops during debugging ONLY, this holds the index of |
| 231 | // the Main CountedLoop. Used to assert that we understand the graph shape. |
| 232 | node_idx_t _main_idx; |
| 233 | |
| 234 | // Known trip count calculated by compute_exact_trip_count() |
| 235 | uint _trip_count; |
| 236 | |
| 237 | // Log2 of original loop bodies in unrolled loop |
| 238 | int _unrolled_count_log2; |
| 239 | |
| 240 | // Node count prior to last unrolling - used to decide if |
| 241 | // unroll,optimize,unroll,optimize,... is making progress |
| 242 | int _node_count_before_unroll; |
| 243 | |
| 244 | // If slp analysis is performed we record the maximum |
| 245 | // vector mapped unroll factor here |
| 246 | int _slp_maximum_unroll_factor; |
| 247 | |
| 248 | public: |
| 249 | CountedLoopNode(Node *entry, Node *backedge) |
| 250 | : BaseCountedLoopNode(entry, backedge), _main_idx(0), _trip_count(max_juint), |
| 251 | _unrolled_count_log2(0), _node_count_before_unroll(0), |
| 252 | _slp_maximum_unroll_factor(0) { |
| 253 | init_class_id(Class_CountedLoop); |
| 254 | // Initialize _trip_count to the largest possible value. |
| 255 | // Will be reset (lower) if the loop's trip count is known. |
| 256 | } |
| 257 | |
| 258 | virtual int Opcode() const; |
| 259 | virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); |
| 260 | |
| 261 | CountedLoopEndNode* loopexit_or_null() const { return (CountedLoopEndNode*) BaseCountedLoopNode::loopexit_or_null(); } |
| 262 | CountedLoopEndNode* loopexit() const { return (CountedLoopEndNode*) BaseCountedLoopNode::loopexit(); } |
| 263 | int stride_con() const; |
| 264 | |
| 265 | // Match increment with optional truncation |
| 266 | static Node* |
| 267 | match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInteger** trunc_type, |
| 268 | BasicType bt); |
| 269 | |
| 270 | // A 'main' loop has a pre-loop and a post-loop. The 'main' loop |
| 271 | // can run short a few iterations and may start a few iterations in. |
| 272 | // It will be RCE'd and unrolled and aligned. |
| 273 | |
| 274 | // A following 'post' loop will run any remaining iterations. Used |
| 275 | // during Range Check Elimination, the 'post' loop will do any final |
| 276 | // iterations with full checks. Also used by Loop Unrolling, where |
| 277 | // the 'post' loop will do any epilog iterations needed. Basically, |
| 278 | // a 'post' loop can not profitably be further unrolled or RCE'd. |
| 279 | |
| 280 | // A preceding 'pre' loop will run at least 1 iteration (to do peeling), |
| 281 | // it may do under-flow checks for RCE and may do alignment iterations |
| 282 | // so the following main loop 'knows' that it is striding down cache |
| 283 | // lines. |
| 284 | |
| 285 | // A 'main' loop that is ONLY unrolled or peeled, never RCE'd or |
| 286 | // Aligned, may be missing it's pre-loop. |
| 287 | bool is_normal_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Normal; } |
| 288 | bool is_pre_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Pre; } |
| 289 | bool is_main_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Main; } |
| 290 | bool is_post_loop () const { return (_loop_flags&PreMainPostFlagsMask) == Post; } |
| 291 | bool is_reduction_loop() const { return (_loop_flags&HasReductions) == HasReductions; } |
| 292 | bool was_slp_analyzed () const { return (_loop_flags&WasSlpAnalyzed) == WasSlpAnalyzed; } |
| 293 | bool has_passed_slp () const { return (_loop_flags&PassedSlpAnalysis) == PassedSlpAnalysis; } |
| 294 | bool is_unroll_only () const { return (_loop_flags&DoUnrollOnly) == DoUnrollOnly; } |
| 295 | bool is_main_no_pre_loop() const { return _loop_flags & MainHasNoPreLoop; } |
| 296 | bool has_atomic_post_loop () const { return (_loop_flags & HasAtomicPostLoop) == HasAtomicPostLoop; } |
| 297 | void set_main_no_pre_loop() { _loop_flags |= MainHasNoPreLoop; } |
| 298 | |
| 299 | int main_idx() const { return _main_idx; } |
| 300 | |
| 301 | |
| 302 | void set_pre_loop (CountedLoopNode *main) { assert(is_normal_loop(),"")do { if (!(is_normal_loop())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 302, "assert(" "is_normal_loop()" ") failed", ""); ::breakpoint (); } } while (0); _loop_flags |= Pre ; _main_idx = main->_idx; } |
| 303 | void set_main_loop ( ) { assert(is_normal_loop(),"")do { if (!(is_normal_loop())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 303, "assert(" "is_normal_loop()" ") failed", ""); ::breakpoint (); } } while (0); _loop_flags |= Main; } |
| 304 | void set_post_loop (CountedLoopNode *main) { assert(is_normal_loop(),"")do { if (!(is_normal_loop())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 304, "assert(" "is_normal_loop()" ") failed", ""); ::breakpoint (); } } while (0); _loop_flags |= Post; _main_idx = main->_idx; } |
| 305 | void set_normal_loop( ) { _loop_flags &= ~PreMainPostFlagsMask; } |
| 306 | |
| 307 | void set_trip_count(uint tc) { _trip_count = tc; } |
| 308 | uint trip_count() { return _trip_count; } |
| 309 | |
| 310 | bool has_exact_trip_count() const { return (_loop_flags & HasExactTripCount) != 0; } |
| 311 | void set_exact_trip_count(uint tc) { |
| 312 | _trip_count = tc; |
| 313 | _loop_flags |= HasExactTripCount; |
| 314 | } |
| 315 | void set_nonexact_trip_count() { |
| 316 | _loop_flags &= ~HasExactTripCount; |
| 317 | } |
| 318 | void set_notpassed_slp() { |
| 319 | _loop_flags &= ~PassedSlpAnalysis; |
| 320 | } |
| 321 | |
| 322 | void double_unrolled_count() { _unrolled_count_log2++; } |
| 323 | int unrolled_count() { return 1 << MIN2(_unrolled_count_log2, BitsPerInt-3); } |
| 324 | |
| 325 | void set_node_count_before_unroll(int ct) { _node_count_before_unroll = ct; } |
| 326 | int node_count_before_unroll() { return _node_count_before_unroll; } |
| 327 | void set_slp_max_unroll(int unroll_factor) { _slp_maximum_unroll_factor = unroll_factor; } |
| 328 | int slp_max_unroll() const { return _slp_maximum_unroll_factor; } |
| 329 | |
| 330 | virtual LoopNode* skip_strip_mined(int expect_skeleton = 1); |
| 331 | OuterStripMinedLoopNode* outer_loop() const; |
| 332 | virtual IfTrueNode* outer_loop_tail() const; |
| 333 | virtual OuterStripMinedLoopEndNode* outer_loop_end() const; |
| 334 | virtual IfFalseNode* outer_loop_exit() const; |
| 335 | virtual SafePointNode* outer_safepoint() const; |
| 336 | |
| 337 | // If this is a main loop in a pre/main/post loop nest, walk over |
| 338 | // the predicates that were inserted by |
| 339 | // duplicate_predicates()/add_range_check_predicate() |
| 340 | static Node* skip_predicates_from_entry(Node* ctrl); |
| 341 | Node* skip_predicates(); |
| 342 | |
| 343 | virtual BasicType bt() const { |
| 344 | return T_INT; |
| 345 | } |
| 346 | |
| 347 | Node* is_canonical_loop_entry(); |
| 348 | |
| 349 | #ifndef PRODUCT |
| 350 | virtual void dump_spec(outputStream *st) const; |
| 351 | #endif |
| 352 | }; |
| 353 | |
| 354 | class LongCountedLoopNode : public BaseCountedLoopNode { |
| 355 | public: |
| 356 | LongCountedLoopNode(Node *entry, Node *backedge) |
| 357 | : BaseCountedLoopNode(entry, backedge) { |
| 358 | init_class_id(Class_LongCountedLoop); |
| 359 | } |
| 360 | |
| 361 | virtual int Opcode() const; |
| 362 | |
| 363 | virtual BasicType bt() const { |
| 364 | return T_LONG; |
| 365 | } |
| 366 | |
| 367 | LongCountedLoopEndNode* loopexit_or_null() const { return (LongCountedLoopEndNode*) BaseCountedLoopNode::loopexit_or_null(); } |
| 368 | LongCountedLoopEndNode* loopexit() const { return (LongCountedLoopEndNode*) BaseCountedLoopNode::loopexit(); } |
| 369 | }; |
| 370 | |
| 371 | |
| 372 | //------------------------------CountedLoopEndNode----------------------------- |
| 373 | // CountedLoopEndNodes end simple trip counted loops. They act much like |
| 374 | // IfNodes. |
| 375 | |
| 376 | class BaseCountedLoopEndNode : public IfNode { |
| 377 | public: |
| 378 | enum { TestControl, TestValue }; |
| 379 | BaseCountedLoopEndNode(Node *control, Node *test, float prob, float cnt) |
| 380 | : IfNode(control, test, prob, cnt) { |
| 381 | init_class_id(Class_BaseCountedLoopEnd); |
| 382 | } |
| 383 | |
| 384 | Node *cmp_node() const { return (in(TestValue)->req() >=2) ? in(TestValue)->in(1) : NULL__null; } |
| 385 | Node* incr() const { Node* tmp = cmp_node(); return (tmp && tmp->req() == 3) ? tmp->in(1) : NULL__null; } |
| 386 | Node* limit() const { Node* tmp = cmp_node(); return (tmp && tmp->req() == 3) ? tmp->in(2) : NULL__null; } |
| 387 | Node* stride() const { Node* tmp = incr(); return (tmp && tmp->req() == 3) ? tmp->in(2) : NULL__null; } |
| 388 | Node* init_trip() const { Node* tmp = phi(); return (tmp && tmp->req() == 3) ? tmp->in(1) : NULL__null; } |
| 389 | bool stride_is_con() const { Node *tmp = stride(); return (tmp != NULL__null && tmp->is_Con()); } |
| 390 | |
| 391 | PhiNode* phi() const { |
| 392 | Node* tmp = incr(); |
| 393 | if (tmp && tmp->req() == 3) { |
| 394 | Node* phi = tmp->in(1); |
| 395 | if (phi->is_Phi()) { |
| 396 | return phi->as_Phi(); |
| 397 | } |
| 398 | } |
| 399 | return NULL__null; |
| 400 | } |
| 401 | |
| 402 | BaseCountedLoopNode* loopnode() const { |
| 403 | // The CountedLoopNode that goes with this CountedLoopEndNode may |
| 404 | // have been optimized out by the IGVN so be cautious with the |
| 405 | // pattern matching on the graph |
| 406 | PhiNode* iv_phi = phi(); |
| 407 | if (iv_phi == NULL__null) { |
| 408 | return NULL__null; |
| 409 | } |
| 410 | Node* ln = iv_phi->in(0); |
| 411 | if (!ln->is_BaseCountedLoop() || ln->as_BaseCountedLoop()->loopexit_or_null() != this) { |
| 412 | return NULL__null; |
| 413 | } |
| 414 | if (ln->as_BaseCountedLoop()->bt() != bt()) { |
| 415 | return NULL__null; |
| 416 | } |
| 417 | return ln->as_BaseCountedLoop(); |
| 418 | } |
| 419 | |
| 420 | BoolTest::mask test_trip() const { return in(TestValue)->as_Bool()->_test._test; } |
| 421 | |
| 422 | jlong stride_con() const; |
| 423 | virtual BasicType bt() const = 0; |
| 424 | |
| 425 | static BaseCountedLoopEndNode* make(Node* control, Node* test, float prob, float cnt, BasicType bt); |
| 426 | }; |
| 427 | |
| 428 | class CountedLoopEndNode : public BaseCountedLoopEndNode { |
| 429 | public: |
| 430 | |
| 431 | CountedLoopEndNode(Node *control, Node *test, float prob, float cnt) |
| 432 | : BaseCountedLoopEndNode(control, test, prob, cnt) { |
| 433 | init_class_id(Class_CountedLoopEnd); |
| 434 | } |
| 435 | virtual int Opcode() const; |
| 436 | |
| 437 | CountedLoopNode* loopnode() const { |
| 438 | return (CountedLoopNode*) BaseCountedLoopEndNode::loopnode(); |
| 439 | } |
| 440 | |
| 441 | virtual BasicType bt() const { |
| 442 | return T_INT; |
| 443 | } |
| 444 | |
| 445 | #ifndef PRODUCT |
| 446 | virtual void dump_spec(outputStream *st) const; |
| 447 | #endif |
| 448 | }; |
| 449 | |
| 450 | class LongCountedLoopEndNode : public BaseCountedLoopEndNode { |
| 451 | public: |
| 452 | LongCountedLoopEndNode(Node *control, Node *test, float prob, float cnt) |
| 453 | : BaseCountedLoopEndNode(control, test, prob, cnt) { |
| 454 | init_class_id(Class_LongCountedLoopEnd); |
| 455 | } |
| 456 | |
| 457 | LongCountedLoopNode* loopnode() const { |
| 458 | return (LongCountedLoopNode*) BaseCountedLoopEndNode::loopnode(); |
| 459 | } |
| 460 | |
| 461 | virtual int Opcode() const; |
| 462 | |
| 463 | virtual BasicType bt() const { |
| 464 | return T_LONG; |
| 465 | } |
| 466 | }; |
| 467 | |
| 468 | |
| 469 | inline BaseCountedLoopEndNode* BaseCountedLoopNode::loopexit_or_null() const { |
| 470 | Node* bctrl = back_control(); |
| 471 | if (bctrl == NULL__null) return NULL__null; |
| 472 | |
| 473 | Node* lexit = bctrl->in(0); |
| 474 | if (!lexit->is_BaseCountedLoopEnd()) { |
| 475 | return NULL__null; |
| 476 | } |
| 477 | BaseCountedLoopEndNode* result = lexit->as_BaseCountedLoopEnd(); |
| 478 | if (result->bt() != bt()) { |
| 479 | return NULL__null; |
| 480 | } |
| 481 | return result; |
| 482 | } |
| 483 | |
| 484 | inline BaseCountedLoopEndNode* BaseCountedLoopNode::loopexit() const { |
| 485 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 486 | assert(cle != NULL, "loopexit is NULL")do { if (!(cle != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 486, "assert(" "cle != __null" ") failed", "loopexit is NULL" ); ::breakpoint(); } } while (0); |
| 487 | return cle; |
| 488 | } |
| 489 | |
| 490 | inline Node* BaseCountedLoopNode::init_trip() const { |
| 491 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 492 | return cle != NULL__null ? cle->init_trip() : NULL__null; |
| 493 | } |
| 494 | inline Node* BaseCountedLoopNode::stride() const { |
| 495 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 496 | return cle != NULL__null ? cle->stride() : NULL__null; |
| 497 | } |
| 498 | |
| 499 | inline bool BaseCountedLoopNode::stride_is_con() const { |
| 500 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 501 | return cle != NULL__null && cle->stride_is_con(); |
| 502 | } |
| 503 | inline Node* BaseCountedLoopNode::limit() const { |
| 504 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 505 | return cle != NULL__null ? cle->limit() : NULL__null; |
| 506 | } |
| 507 | inline Node* BaseCountedLoopNode::incr() const { |
| 508 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 509 | return cle != NULL__null ? cle->incr() : NULL__null; |
| 510 | } |
| 511 | inline Node* BaseCountedLoopNode::phi() const { |
| 512 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 513 | return cle != NULL__null ? cle->phi() : NULL__null; |
| 514 | } |
| 515 | |
| 516 | inline jlong BaseCountedLoopNode::stride_con() const { |
| 517 | BaseCountedLoopEndNode* cle = loopexit_or_null(); |
| 518 | return cle != NULL__null ? cle->stride_con() : 0; |
| 519 | } |
| 520 | |
| 521 | |
| 522 | //------------------------------LoopLimitNode----------------------------- |
| 523 | // Counted Loop limit node which represents exact final iterator value: |
| 524 | // trip_count = (limit - init_trip + stride - 1)/stride |
| 525 | // final_value= trip_count * stride + init_trip. |
| 526 | // Use HW instructions to calculate it when it can overflow in integer. |
| 527 | // Note, final_value should fit into integer since counted loop has |
| 528 | // limit check: limit <= max_int-stride. |
| 529 | class LoopLimitNode : public Node { |
| 530 | enum { Init=1, Limit=2, Stride=3 }; |
| 531 | public: |
| 532 | LoopLimitNode( Compile* C, Node *init, Node *limit, Node *stride ) : Node(0,init,limit,stride) { |
| 533 | // Put it on the Macro nodes list to optimize during macro nodes expansion. |
| 534 | init_flags(Flag_is_macro); |
| 535 | C->add_macro_node(this); |
| 536 | } |
| 537 | virtual int Opcode() const; |
| 538 | virtual const Type *bottom_type() const { return TypeInt::INT; } |
| 539 | virtual uint ideal_reg() const { return Op_RegI; } |
| 540 | virtual const Type* Value(PhaseGVN* phase) const; |
| 541 | virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); |
| 542 | virtual Node* Identity(PhaseGVN* phase); |
| 543 | }; |
| 544 | |
| 545 | // Support for strip mining |
| 546 | class OuterStripMinedLoopNode : public LoopNode { |
| 547 | private: |
| 548 | CountedLoopNode* inner_loop() const; |
| 549 | public: |
| 550 | OuterStripMinedLoopNode(Compile* C, Node *entry, Node *backedge) |
| 551 | : LoopNode(entry, backedge) { |
| 552 | init_class_id(Class_OuterStripMinedLoop); |
| 553 | init_flags(Flag_is_macro); |
| 554 | C->add_macro_node(this); |
| 555 | } |
| 556 | |
| 557 | virtual int Opcode() const; |
| 558 | |
| 559 | virtual IfTrueNode* outer_loop_tail() const; |
| 560 | virtual OuterStripMinedLoopEndNode* outer_loop_end() const; |
| 561 | virtual IfFalseNode* outer_loop_exit() const; |
| 562 | virtual SafePointNode* outer_safepoint() const; |
| 563 | void adjust_strip_mined_loop(PhaseIterGVN* igvn); |
| 564 | }; |
| 565 | |
| 566 | class OuterStripMinedLoopEndNode : public IfNode { |
| 567 | public: |
| 568 | OuterStripMinedLoopEndNode(Node *control, Node *test, float prob, float cnt) |
| 569 | : IfNode(control, test, prob, cnt) { |
| 570 | init_class_id(Class_OuterStripMinedLoopEnd); |
| 571 | } |
| 572 | |
| 573 | virtual int Opcode() const; |
| 574 | |
| 575 | virtual const Type* Value(PhaseGVN* phase) const; |
| 576 | virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); |
| 577 | |
| 578 | bool is_expanded(PhaseGVN *phase) const; |
| 579 | }; |
| 580 | |
| 581 | // -----------------------------IdealLoopTree---------------------------------- |
| 582 | class IdealLoopTree : public ResourceObj { |
| 583 | public: |
| 584 | IdealLoopTree *_parent; // Parent in loop tree |
| 585 | IdealLoopTree *_next; // Next sibling in loop tree |
| 586 | IdealLoopTree *_child; // First child in loop tree |
| 587 | |
| 588 | // The head-tail backedge defines the loop. |
| 589 | // If a loop has multiple backedges, this is addressed during cleanup where |
| 590 | // we peel off the multiple backedges, merging all edges at the bottom and |
| 591 | // ensuring that one proper backedge flow into the loop. |
| 592 | Node *_head; // Head of loop |
| 593 | Node *_tail; // Tail of loop |
| 594 | inline Node *tail(); // Handle lazy update of _tail field |
| 595 | inline Node *head(); // Handle lazy update of _head field |
| 596 | PhaseIdealLoop* _phase; |
| 597 | int _local_loop_unroll_limit; |
| 598 | int _local_loop_unroll_factor; |
| 599 | |
| 600 | Node_List _body; // Loop body for inner loops |
| 601 | |
| 602 | uint16_t _nest; // Nesting depth |
| 603 | uint8_t _irreducible:1, // True if irreducible |
| 604 | _has_call:1, // True if has call safepoint |
| 605 | _has_sfpt:1, // True if has non-call safepoint |
| 606 | _rce_candidate:1; // True if candidate for range check elimination |
| 607 | |
| 608 | Node_List* _safepts; // List of safepoints in this loop |
| 609 | Node_List* _required_safept; // A inner loop cannot delete these safepts; |
| 610 | bool _allow_optimizations; // Allow loop optimizations |
| 611 | |
| 612 | IdealLoopTree( PhaseIdealLoop* phase, Node *head, Node *tail ) |
| 613 | : _parent(0), _next(0), _child(0), |
| 614 | _head(head), _tail(tail), |
| 615 | _phase(phase), |
| 616 | _local_loop_unroll_limit(0), _local_loop_unroll_factor(0), |
| 617 | _nest(0), _irreducible(0), _has_call(0), _has_sfpt(0), _rce_candidate(0), |
| 618 | _safepts(NULL__null), |
| 619 | _required_safept(NULL__null), |
| 620 | _allow_optimizations(true) |
| 621 | { |
| 622 | precond(_head != NULL)do { if (!(_head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 622, "assert(" "_head != __null" ") failed", "precond"); :: breakpoint(); } } while (0); |
| 623 | precond(_tail != NULL)do { if (!(_tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 623, "assert(" "_tail != __null" ") failed", "precond"); :: breakpoint(); } } while (0); |
| 624 | } |
| 625 | |
| 626 | // Is 'l' a member of 'this'? |
| 627 | bool is_member(const IdealLoopTree *l) const; // Test for nested membership |
| 628 | |
| 629 | // Set loop nesting depth. Accumulate has_call bits. |
| 630 | int set_nest( uint depth ); |
| 631 | |
| 632 | // Split out multiple fall-in edges from the loop header. Move them to a |
| 633 | // private RegionNode before the loop. This becomes the loop landing pad. |
| 634 | void split_fall_in( PhaseIdealLoop *phase, int fall_in_cnt ); |
| 635 | |
| 636 | // Split out the outermost loop from this shared header. |
| 637 | void split_outer_loop( PhaseIdealLoop *phase ); |
| 638 | |
| 639 | // Merge all the backedges from the shared header into a private Region. |
| 640 | // Feed that region as the one backedge to this loop. |
| 641 | void merge_many_backedges( PhaseIdealLoop *phase ); |
| 642 | |
| 643 | // Split shared headers and insert loop landing pads. |
| 644 | // Insert a LoopNode to replace the RegionNode. |
| 645 | // Returns TRUE if loop tree is structurally changed. |
| 646 | bool beautify_loops( PhaseIdealLoop *phase ); |
| 647 | |
| 648 | // Perform optimization to use the loop predicates for null checks and range checks. |
| 649 | // Applies to any loop level (not just the innermost one) |
| 650 | bool loop_predication( PhaseIdealLoop *phase); |
| 651 | |
| 652 | // Perform iteration-splitting on inner loops. Split iterations to |
| 653 | // avoid range checks or one-shot null checks. Returns false if the |
| 654 | // current round of loop opts should stop. |
| 655 | bool iteration_split( PhaseIdealLoop *phase, Node_List &old_new ); |
| 656 | |
| 657 | // Driver for various flavors of iteration splitting. Returns false |
| 658 | // if the current round of loop opts should stop. |
| 659 | bool iteration_split_impl( PhaseIdealLoop *phase, Node_List &old_new ); |
| 660 | |
| 661 | // Given dominators, try to find loops with calls that must always be |
| 662 | // executed (call dominates loop tail). These loops do not need non-call |
| 663 | // safepoints (ncsfpt). |
| 664 | void check_safepts(VectorSet &visited, Node_List &stack); |
| 665 | |
| 666 | // Allpaths backwards scan from loop tail, terminating each path at first safepoint |
| 667 | // encountered. |
| 668 | void allpaths_check_safepts(VectorSet &visited, Node_List &stack); |
| 669 | |
| 670 | // Remove safepoints from loop. Optionally keeping one. |
| 671 | void remove_safepoints(PhaseIdealLoop* phase, bool keep_one); |
| 672 | |
| 673 | // Convert to counted loops where possible |
| 674 | void counted_loop( PhaseIdealLoop *phase ); |
| 675 | |
| 676 | // Check for Node being a loop-breaking test |
| 677 | Node *is_loop_exit(Node *iff) const; |
| 678 | |
| 679 | // Remove simplistic dead code from loop body |
| 680 | void DCE_loop_body(); |
| 681 | |
| 682 | // Look for loop-exit tests with my 50/50 guesses from the Parsing stage. |
| 683 | // Replace with a 1-in-10 exit guess. |
| 684 | void adjust_loop_exit_prob( PhaseIdealLoop *phase ); |
| 685 | |
| 686 | // Return TRUE or FALSE if the loop should never be RCE'd or aligned. |
| 687 | // Useful for unrolling loops with NO array accesses. |
| 688 | bool policy_peel_only( PhaseIdealLoop *phase ) const; |
| 689 | |
| 690 | // Return TRUE or FALSE if the loop should be unswitched -- clone |
| 691 | // loop with an invariant test |
| 692 | bool policy_unswitching( PhaseIdealLoop *phase ) const; |
| 693 | |
| 694 | // Micro-benchmark spamming. Remove empty loops. |
| 695 | bool do_remove_empty_loop( PhaseIdealLoop *phase ); |
| 696 | |
| 697 | // Convert one iteration loop into normal code. |
| 698 | bool do_one_iteration_loop( PhaseIdealLoop *phase ); |
| 699 | |
| 700 | // Return TRUE or FALSE if the loop should be peeled or not. Peel if we can |
| 701 | // move some loop-invariant test (usually a null-check) before the loop. |
| 702 | bool policy_peeling(PhaseIdealLoop *phase); |
| 703 | |
| 704 | uint estimate_peeling(PhaseIdealLoop *phase); |
| 705 | |
| 706 | // Return TRUE or FALSE if the loop should be maximally unrolled. Stash any |
| 707 | // known trip count in the counted loop node. |
| 708 | bool policy_maximally_unroll(PhaseIdealLoop *phase) const; |
| 709 | |
| 710 | // Return TRUE or FALSE if the loop should be unrolled or not. Apply unroll |
| 711 | // if the loop is a counted loop and the loop body is small enough. |
| 712 | bool policy_unroll(PhaseIdealLoop *phase); |
| 713 | |
| 714 | // Loop analyses to map to a maximal superword unrolling for vectorization. |
| 715 | void policy_unroll_slp_analysis(CountedLoopNode *cl, PhaseIdealLoop *phase, int future_unroll_ct); |
| 716 | |
| 717 | // Return TRUE or FALSE if the loop should be range-check-eliminated. |
| 718 | // Gather a list of IF tests that are dominated by iteration splitting; |
| 719 | // also gather the end of the first split and the start of the 2nd split. |
| 720 | bool policy_range_check(PhaseIdealLoop* phase, bool provisional, BasicType bt) const; |
| 721 | |
| 722 | // Return TRUE if "iff" is a range check. |
| 723 | bool is_range_check_if(IfNode *iff, PhaseIdealLoop *phase, Invariance& invar DEBUG_ONLY(COMMA ProjNode *predicate_proj), ProjNode *predicate_proj) const; |
| 724 | bool is_range_check_if(IfNode* iff, PhaseIdealLoop* phase, BasicType bt, Node* iv, Node*& range, Node*& offset, |
| 725 | jlong& scale) const; |
| 726 | |
| 727 | // Estimate the number of nodes required when cloning a loop (body). |
| 728 | uint est_loop_clone_sz(uint factor) const; |
| 729 | // Estimate the number of nodes required when unrolling a loop (body). |
| 730 | uint est_loop_unroll_sz(uint factor) const; |
| 731 | |
| 732 | // Compute loop trip count if possible |
| 733 | void compute_trip_count(PhaseIdealLoop* phase); |
| 734 | |
| 735 | // Compute loop trip count from profile data |
| 736 | float compute_profile_trip_cnt_helper(Node* n); |
| 737 | void compute_profile_trip_cnt( PhaseIdealLoop *phase ); |
| 738 | |
| 739 | // Reassociate invariant expressions. |
| 740 | void reassociate_invariants(PhaseIdealLoop *phase); |
| 741 | // Reassociate invariant binary expressions. |
| 742 | Node* reassociate(Node* n1, PhaseIdealLoop *phase); |
| 743 | // Reassociate invariant add and subtract expressions. |
| 744 | Node* reassociate_add_sub(Node* n1, int inv1_idx, int inv2_idx, PhaseIdealLoop *phase); |
| 745 | // Return nonzero index of invariant operand if invariant and variant |
| 746 | // are combined with an associative binary. Helper for reassociate_invariants. |
| 747 | int find_invariant(Node* n, PhaseIdealLoop *phase); |
| 748 | // Return TRUE if "n" is associative. |
| 749 | bool is_associative(Node* n, Node* base=NULL__null); |
| 750 | |
| 751 | // Return true if n is invariant |
| 752 | bool is_invariant(Node* n) const; |
| 753 | |
| 754 | // Put loop body on igvn work list |
| 755 | void record_for_igvn(); |
| 756 | |
| 757 | bool is_root() { return _parent == NULL__null; } |
| 758 | // A proper/reducible loop w/o any (occasional) dead back-edge. |
| 759 | bool is_loop() { return !_irreducible && !tail()->is_top(); } |
| 760 | bool is_counted() { return is_loop() && _head->is_CountedLoop(); } |
| 761 | bool is_innermost() { return is_loop() && _child == NULL__null; } |
| 762 | |
| 763 | void remove_main_post_loops(CountedLoopNode *cl, PhaseIdealLoop *phase); |
| 764 | |
| 765 | #ifndef PRODUCT |
| 766 | void dump_head() const; // Dump loop head only |
| 767 | void dump() const; // Dump this loop recursively |
| 768 | void verify_tree(IdealLoopTree *loop, const IdealLoopTree *parent) const; |
| 769 | #endif |
| 770 | |
| 771 | private: |
| 772 | enum { EMPTY_LOOP_SIZE = 7 }; // Number of nodes in an empty loop. |
| 773 | |
| 774 | // Estimate the number of nodes resulting from control and data flow merge. |
| 775 | uint est_loop_flow_merge_sz() const; |
| 776 | }; |
| 777 | |
| 778 | // -----------------------------PhaseIdealLoop--------------------------------- |
| 779 | // Computes the mapping from Nodes to IdealLoopTrees. Organizes IdealLoopTrees |
| 780 | // into a loop tree. Drives the loop-based transformations on the ideal graph. |
| 781 | class PhaseIdealLoop : public PhaseTransform { |
| 782 | friend class IdealLoopTree; |
| 783 | friend class SuperWord; |
| 784 | friend class CountedLoopReserveKit; |
| 785 | friend class ShenandoahBarrierC2Support; |
| 786 | friend class AutoNodeBudget; |
| 787 | |
| 788 | // Pre-computed def-use info |
| 789 | PhaseIterGVN &_igvn; |
| 790 | |
| 791 | // Head of loop tree |
| 792 | IdealLoopTree* _ltree_root; |
| 793 | |
| 794 | // Array of pre-order numbers, plus post-visited bit. |
| 795 | // ZERO for not pre-visited. EVEN for pre-visited but not post-visited. |
| 796 | // ODD for post-visited. Other bits are the pre-order number. |
| 797 | uint *_preorders; |
| 798 | uint _max_preorder; |
| 799 | |
| 800 | const PhaseIdealLoop* _verify_me; |
| 801 | bool _verify_only; |
| 802 | |
| 803 | // Allocate _preorders[] array |
| 804 | void allocate_preorders() { |
| 805 | _max_preorder = C->unique()+8; |
| 806 | _preorders = NEW_RESOURCE_ARRAY(uint, _max_preorder)(uint*) resource_allocate_bytes((_max_preorder) * sizeof(uint )); |
| 807 | memset(_preorders, 0, sizeof(uint) * _max_preorder); |
| 808 | } |
| 809 | |
| 810 | // Allocate _preorders[] array |
| 811 | void reallocate_preorders() { |
| 812 | if ( _max_preorder < C->unique() ) { |
| 813 | _preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, C->unique())(uint*) resource_reallocate_bytes((char*)(_preorders), (_max_preorder ) * sizeof(uint), (C->unique()) * sizeof(uint)); |
| 814 | _max_preorder = C->unique(); |
| 815 | } |
| 816 | memset(_preorders, 0, sizeof(uint) * _max_preorder); |
| 817 | } |
| 818 | |
| 819 | // Check to grow _preorders[] array for the case when build_loop_tree_impl() |
| 820 | // adds new nodes. |
| 821 | void check_grow_preorders( ) { |
| 822 | if ( _max_preorder < C->unique() ) { |
| 823 | uint newsize = _max_preorder<<1; // double size of array |
| 824 | _preorders = REALLOC_RESOURCE_ARRAY(uint, _preorders, _max_preorder, newsize)(uint*) resource_reallocate_bytes((char*)(_preorders), (_max_preorder ) * sizeof(uint), (newsize) * sizeof(uint)); |
| 825 | memset(&_preorders[_max_preorder],0,sizeof(uint)*(newsize-_max_preorder)); |
| 826 | _max_preorder = newsize; |
| 827 | } |
| 828 | } |
| 829 | // Check for pre-visited. Zero for NOT visited; non-zero for visited. |
| 830 | int is_visited( Node *n ) const { return _preorders[n->_idx]; } |
| 831 | // Pre-order numbers are written to the Nodes array as low-bit-set values. |
| 832 | void set_preorder_visited( Node *n, int pre_order ) { |
| 833 | assert( !is_visited( n ), "already set" )do { if (!(!is_visited( n ))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 833, "assert(" "!is_visited( n )" ") failed", "already set" ); ::breakpoint(); } } while (0); |
| 834 | _preorders[n->_idx] = (pre_order<<1); |
| 835 | }; |
| 836 | // Return pre-order number. |
| 837 | int get_preorder( Node *n ) const { assert( is_visited(n), "" )do { if (!(is_visited(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 837, "assert(" "is_visited(n)" ") failed", ""); ::breakpoint (); } } while (0); return _preorders[n->_idx]>>1; } |
| 838 | |
| 839 | // Check for being post-visited. |
| 840 | // Should be previsited already (checked with assert(is_visited(n))). |
| 841 | int is_postvisited( Node *n ) const { assert( is_visited(n), "" )do { if (!(is_visited(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 841, "assert(" "is_visited(n)" ") failed", ""); ::breakpoint (); } } while (0); return _preorders[n->_idx]&1; } |
| 842 | |
| 843 | // Mark as post visited |
| 844 | void set_postvisited( Node *n ) { assert( !is_postvisited( n ), "" )do { if (!(!is_postvisited( n ))) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 844, "assert(" "!is_postvisited( n )" ") failed", ""); ::breakpoint (); } } while (0); _preorders[n->_idx] |= 1; } |
| 845 | |
| 846 | public: |
| 847 | // Set/get control node out. Set lower bit to distinguish from IdealLoopTree |
| 848 | // Returns true if "n" is a data node, false if it's a control node. |
| 849 | bool has_ctrl( Node *n ) const { return ((intptr_t)_nodes[n->_idx]) & 1; } |
| 850 | |
| 851 | private: |
| 852 | // clear out dead code after build_loop_late |
| 853 | Node_List _deadlist; |
| 854 | |
| 855 | // Support for faster execution of get_late_ctrl()/dom_lca() |
| 856 | // when a node has many uses and dominator depth is deep. |
| 857 | GrowableArray<jlong> _dom_lca_tags; |
| 858 | uint _dom_lca_tags_round; |
| 859 | void init_dom_lca_tags(); |
| 860 | |
| 861 | // Helper for debugging bad dominance relationships |
| 862 | bool verify_dominance(Node* n, Node* use, Node* LCA, Node* early); |
| 863 | |
| 864 | Node* compute_lca_of_uses(Node* n, Node* early, bool verify = false); |
| 865 | |
| 866 | // Inline wrapper for frequent cases: |
| 867 | // 1) only one use |
| 868 | // 2) a use is the same as the current LCA passed as 'n1' |
| 869 | Node *dom_lca_for_get_late_ctrl( Node *lca, Node *n, Node *tag ) { |
| 870 | assert( n->is_CFG(), "" )do { if (!(n->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 870, "assert(" "n->is_CFG()" ") failed", ""); ::breakpoint (); } } while (0); |
| 871 | // Fast-path NULL lca |
| 872 | if( lca != NULL__null && lca != n ) { |
| 873 | assert( lca->is_CFG(), "" )do { if (!(lca->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 873, "assert(" "lca->is_CFG()" ") failed", ""); ::breakpoint (); } } while (0); |
| 874 | // find LCA of all uses |
| 875 | n = dom_lca_for_get_late_ctrl_internal( lca, n, tag ); |
| 876 | } |
| 877 | return find_non_split_ctrl(n); |
| 878 | } |
| 879 | Node *dom_lca_for_get_late_ctrl_internal( Node *lca, Node *n, Node *tag ); |
| 880 | |
| 881 | // Helper function for directing control inputs away from CFG split points. |
| 882 | Node *find_non_split_ctrl( Node *ctrl ) const { |
| 883 | if (ctrl != NULL__null) { |
| 884 | if (ctrl->is_MultiBranch()) { |
| 885 | ctrl = ctrl->in(0); |
| 886 | } |
| 887 | assert(ctrl->is_CFG(), "CFG")do { if (!(ctrl->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 887, "assert(" "ctrl->is_CFG()" ") failed", "CFG"); ::breakpoint (); } } while (0); |
| 888 | } |
| 889 | return ctrl; |
| 890 | } |
| 891 | |
| 892 | Node* cast_incr_before_loop(Node* incr, Node* ctrl, Node* loop); |
| 893 | |
| 894 | #ifdef ASSERT1 |
| 895 | void ensure_zero_trip_guard_proj(Node* node, bool is_main_loop); |
| 896 | #endif |
| 897 | void copy_skeleton_predicates_to_main_loop_helper(Node* predicate, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head, |
| 898 | uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre, |
| 899 | Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new); |
| 900 | void copy_skeleton_predicates_to_main_loop(CountedLoopNode* pre_head, Node* init, Node* stride, IdealLoopTree* outer_loop, LoopNode* outer_main_head, |
| 901 | uint dd_main_head, const uint idx_before_pre_post, const uint idx_after_post_before_pre, |
| 902 | Node* zero_trip_guard_proj_main, Node* zero_trip_guard_proj_post, const Node_List &old_new); |
| 903 | Node* clone_skeleton_predicate_for_main_or_post_loop(Node* iff, Node* new_init, Node* new_stride, Node* predicate, Node* uncommon_proj, Node* control, |
| 904 | IdealLoopTree* outer_loop, Node* input_proj); |
| 905 | Node* clone_skeleton_predicate_bool(Node* iff, Node* new_init, Node* new_stride, Node* control); |
| 906 | static bool skeleton_predicate_has_opaque(IfNode* iff); |
| 907 | static void get_skeleton_predicates(Node* predicate, Unique_Node_List& list, bool get_opaque = false); |
| 908 | void update_main_loop_skeleton_predicates(Node* ctrl, CountedLoopNode* loop_head, Node* init, int stride_con); |
| 909 | void copy_skeleton_predicates_to_post_loop(LoopNode* main_loop_head, CountedLoopNode* post_loop_head, Node* init, Node* stride); |
| 910 | void insert_loop_limit_check(ProjNode* limit_check_proj, Node* cmp_limit, Node* bol); |
| 911 | #ifdef ASSERT1 |
| 912 | bool only_has_infinite_loops(); |
| 913 | #endif |
| 914 | |
| 915 | void log_loop_tree(); |
| 916 | |
| 917 | public: |
| 918 | |
| 919 | PhaseIterGVN &igvn() const { return _igvn; } |
| 920 | |
| 921 | bool has_node( Node* n ) const { |
| 922 | guarantee(n != NULL, "No Node.")do { if (!(n != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 922, "guarantee(" "n != NULL" ") failed", "No Node."); ::breakpoint (); } } while (0); |
| 923 | return _nodes[n->_idx] != NULL__null; |
| 924 | } |
| 925 | // check if transform created new nodes that need _ctrl recorded |
| 926 | Node *get_late_ctrl( Node *n, Node *early ); |
| 927 | Node *get_early_ctrl( Node *n ); |
| 928 | Node *get_early_ctrl_for_expensive(Node *n, Node* earliest); |
| 929 | void set_early_ctrl(Node* n, bool update_body); |
| 930 | void set_subtree_ctrl(Node* n, bool update_body); |
| 931 | void set_ctrl( Node *n, Node *ctrl ) { |
| 932 | assert( !has_node(n) || has_ctrl(n), "" )do { if (!(!has_node(n) || has_ctrl(n))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 932, "assert(" "!has_node(n) || has_ctrl(n)" ") failed", "" ); ::breakpoint(); } } while (0); |
| 933 | assert( ctrl->in(0), "cannot set dead control node" )do { if (!(ctrl->in(0))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 933, "assert(" "ctrl->in(0)" ") failed", "cannot set dead control node" ); ::breakpoint(); } } while (0); |
| 934 | assert( ctrl == find_non_split_ctrl(ctrl), "must set legal crtl" )do { if (!(ctrl == find_non_split_ctrl(ctrl))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 934, "assert(" "ctrl == find_non_split_ctrl(ctrl)" ") failed" , "must set legal crtl"); ::breakpoint(); } } while (0); |
| 935 | _nodes.map( n->_idx, (Node*)((intptr_t)ctrl + 1) ); |
| 936 | } |
| 937 | // Set control and update loop membership |
| 938 | void set_ctrl_and_loop(Node* n, Node* ctrl) { |
| 939 | IdealLoopTree* old_loop = get_loop(get_ctrl(n)); |
| 940 | IdealLoopTree* new_loop = get_loop(ctrl); |
| 941 | if (old_loop != new_loop) { |
| 942 | if (old_loop->_child == NULL__null) old_loop->_body.yank(n); |
| 943 | if (new_loop->_child == NULL__null) new_loop->_body.push(n); |
| 944 | } |
| 945 | set_ctrl(n, ctrl); |
| 946 | } |
| 947 | // Control nodes can be replaced or subsumed. During this pass they |
| 948 | // get their replacement Node in slot 1. Instead of updating the block |
| 949 | // location of all Nodes in the subsumed block, we lazily do it. As we |
| 950 | // pull such a subsumed block out of the array, we write back the final |
| 951 | // correct block. |
| 952 | Node *get_ctrl( Node *i ) { |
| 953 | |
| 954 | assert(has_node(i), "")do { if (!(has_node(i))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 954, "assert(" "has_node(i)" ") failed", ""); ::breakpoint( ); } } while (0); |
| 955 | Node *n = get_ctrl_no_update(i); |
| 956 | _nodes.map( i->_idx, (Node*)((intptr_t)n + 1) ); |
| 957 | assert(has_node(i) && has_ctrl(i), "")do { if (!(has_node(i) && has_ctrl(i))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 957, "assert(" "has_node(i) && has_ctrl(i)" ") failed" , ""); ::breakpoint(); } } while (0); |
| 958 | assert(n == find_non_split_ctrl(n), "must return legal ctrl" )do { if (!(n == find_non_split_ctrl(n))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 958, "assert(" "n == find_non_split_ctrl(n)" ") failed", "must return legal ctrl" ); ::breakpoint(); } } while (0); |
| 959 | return n; |
| 960 | } |
| 961 | // true if CFG node d dominates CFG node n |
| 962 | bool is_dominator(Node *d, Node *n); |
| 963 | // return get_ctrl for a data node and self(n) for a CFG node |
| 964 | Node* ctrl_or_self(Node* n) { |
| 965 | if (has_ctrl(n)) |
| 966 | return get_ctrl(n); |
| 967 | else { |
| 968 | assert (n->is_CFG(), "must be a CFG node")do { if (!(n->is_CFG())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 968, "assert(" "n->is_CFG()" ") failed", "must be a CFG node" ); ::breakpoint(); } } while (0); |
| 969 | return n; |
| 970 | } |
| 971 | } |
| 972 | |
| 973 | Node *get_ctrl_no_update_helper(Node *i) const { |
| 974 | assert(has_ctrl(i), "should be control, not loop")do { if (!(has_ctrl(i))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 974, "assert(" "has_ctrl(i)" ") failed", "should be control, not loop" ); ::breakpoint(); } } while (0); |
| 975 | return (Node*)(((intptr_t)_nodes[i->_idx]) & ~1); |
| 976 | } |
| 977 | |
| 978 | Node *get_ctrl_no_update(Node *i) const { |
| 979 | assert( has_ctrl(i), "" )do { if (!(has_ctrl(i))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 979, "assert(" "has_ctrl(i)" ") failed", ""); ::breakpoint( ); } } while (0); |
| 980 | Node *n = get_ctrl_no_update_helper(i); |
| 981 | if (!n->in(0)) { |
| 982 | // Skip dead CFG nodes |
| 983 | do { |
| 984 | n = get_ctrl_no_update_helper(n); |
| 985 | } while (!n->in(0)); |
| 986 | n = find_non_split_ctrl(n); |
| 987 | } |
| 988 | return n; |
| 989 | } |
| 990 | |
| 991 | // Check for loop being set |
| 992 | // "n" must be a control node. Returns true if "n" is known to be in a loop. |
| 993 | bool has_loop( Node *n ) const { |
| 994 | assert(!has_node(n) || !has_ctrl(n), "")do { if (!(!has_node(n) || !has_ctrl(n))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 994, "assert(" "!has_node(n) || !has_ctrl(n)" ") failed", "" ); ::breakpoint(); } } while (0); |
| 995 | return has_node(n); |
| 996 | } |
| 997 | // Set loop |
| 998 | void set_loop( Node *n, IdealLoopTree *loop ) { |
| 999 | _nodes.map(n->_idx, (Node*)loop); |
| 1000 | } |
| 1001 | // Lazy-dazy update of 'get_ctrl' and 'idom_at' mechanisms. Replace |
| 1002 | // the 'old_node' with 'new_node'. Kill old-node. Add a reference |
| 1003 | // from old_node to new_node to support the lazy update. Reference |
| 1004 | // replaces loop reference, since that is not needed for dead node. |
| 1005 | void lazy_update(Node *old_node, Node *new_node) { |
| 1006 | assert(old_node != new_node, "no cycles please")do { if (!(old_node != new_node)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1006, "assert(" "old_node != new_node" ") failed", "no cycles please" ); ::breakpoint(); } } while (0); |
| 1007 | // Re-use the side array slot for this node to provide the |
| 1008 | // forwarding pointer. |
| 1009 | _nodes.map(old_node->_idx, (Node*)((intptr_t)new_node + 1)); |
| 1010 | } |
| 1011 | void lazy_replace(Node *old_node, Node *new_node) { |
| 1012 | _igvn.replace_node(old_node, new_node); |
| 1013 | lazy_update(old_node, new_node); |
| 1014 | } |
| 1015 | |
| 1016 | private: |
| 1017 | |
| 1018 | // Place 'n' in some loop nest, where 'n' is a CFG node |
| 1019 | void build_loop_tree(); |
| 1020 | int build_loop_tree_impl( Node *n, int pre_order ); |
| 1021 | // Insert loop into the existing loop tree. 'innermost' is a leaf of the |
| 1022 | // loop tree, not the root. |
| 1023 | IdealLoopTree *sort( IdealLoopTree *loop, IdealLoopTree *innermost ); |
| 1024 | |
| 1025 | // Place Data nodes in some loop nest |
| 1026 | void build_loop_early( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ); |
| 1027 | void build_loop_late ( VectorSet &visited, Node_List &worklist, Node_Stack &nstack ); |
| 1028 | void build_loop_late_post_work(Node* n, bool pinned); |
| 1029 | void build_loop_late_post(Node* n); |
| 1030 | void verify_strip_mined_scheduling(Node *n, Node* least); |
| 1031 | |
| 1032 | // Array of immediate dominance info for each CFG node indexed by node idx |
| 1033 | private: |
| 1034 | uint _idom_size; |
| 1035 | Node **_idom; // Array of immediate dominators |
| 1036 | uint *_dom_depth; // Used for fast LCA test |
| 1037 | GrowableArray<uint>* _dom_stk; // For recomputation of dom depth |
| 1038 | |
| 1039 | // build the loop tree and perform any requested optimizations |
| 1040 | void build_and_optimize(LoopOptsMode mode); |
| 1041 | |
| 1042 | // Dominators for the sea of nodes |
| 1043 | void Dominators(); |
| 1044 | |
| 1045 | // Compute the Ideal Node to Loop mapping |
| 1046 | PhaseIdealLoop(PhaseIterGVN& igvn, LoopOptsMode mode) : |
| 1047 | PhaseTransform(Ideal_Loop), |
| 1048 | _igvn(igvn), |
| 1049 | _verify_me(nullptr), |
| 1050 | _verify_only(false), |
| 1051 | _nodes_required(UINT_MAX(2147483647 *2U +1U)) { |
| 1052 | assert(mode != LoopOptsVerify, "wrong constructor to verify IdealLoop")do { if (!(mode != LoopOptsVerify)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1052, "assert(" "mode != LoopOptsVerify" ") failed", "wrong constructor to verify IdealLoop" ); ::breakpoint(); } } while (0); |
| 1053 | build_and_optimize(mode); |
| 1054 | } |
| 1055 | |
| 1056 | #ifndef PRODUCT |
| 1057 | // Verify that verify_me made the same decisions as a fresh run |
| 1058 | // or only verify that the graph is valid if verify_me is null. |
| 1059 | PhaseIdealLoop(PhaseIterGVN& igvn, const PhaseIdealLoop* verify_me = nullptr) : |
| 1060 | PhaseTransform(Ideal_Loop), |
| 1061 | _igvn(igvn), |
| 1062 | _verify_me(verify_me), |
| 1063 | _verify_only(verify_me == nullptr), |
| 1064 | _nodes_required(UINT_MAX(2147483647 *2U +1U)) { |
| 1065 | build_and_optimize(LoopOptsVerify); |
| 1066 | } |
| 1067 | #endif |
| 1068 | |
| 1069 | public: |
| 1070 | Node* idom_no_update(Node* d) const { |
| 1071 | return idom_no_update(d->_idx); |
| 1072 | } |
| 1073 | |
| 1074 | Node* idom_no_update(uint didx) const { |
| 1075 | assert(didx < _idom_size, "oob")do { if (!(didx < _idom_size)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1075, "assert(" "didx < _idom_size" ") failed", "oob"); :: breakpoint(); } } while (0); |
| 1076 | Node* n = _idom[didx]; |
| 1077 | assert(n != NULL,"Bad immediate dominator info.")do { if (!(n != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1077, "assert(" "n != __null" ") failed", "Bad immediate dominator info." ); ::breakpoint(); } } while (0); |
| 1078 | while (n->in(0) == NULL__null) { // Skip dead CFG nodes |
| 1079 | n = (Node*)(((intptr_t)_nodes[n->_idx]) & ~1); |
| 1080 | assert(n != NULL,"Bad immediate dominator info.")do { if (!(n != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1080, "assert(" "n != __null" ") failed", "Bad immediate dominator info." ); ::breakpoint(); } } while (0); |
| 1081 | } |
| 1082 | return n; |
| 1083 | } |
| 1084 | |
| 1085 | Node *idom(Node* d) const { |
| 1086 | return idom(d->_idx); |
| 1087 | } |
| 1088 | |
| 1089 | Node *idom(uint didx) const { |
| 1090 | Node *n = idom_no_update(didx); |
| 1091 | _idom[didx] = n; // Lazily remove dead CFG nodes from table. |
| 1092 | return n; |
| 1093 | } |
| 1094 | |
| 1095 | uint dom_depth(Node* d) const { |
| 1096 | guarantee(d != NULL, "Null dominator info.")do { if (!(d != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1096, "guarantee(" "d != NULL" ") failed", "Null dominator info." ); ::breakpoint(); } } while (0); |
| 1097 | guarantee(d->_idx < _idom_size, "")do { if (!(d->_idx < _idom_size)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1097, "guarantee(" "d->_idx < _idom_size" ") failed", ""); ::breakpoint(); } } while (0); |
| 1098 | return _dom_depth[d->_idx]; |
| 1099 | } |
| 1100 | void set_idom(Node* d, Node* n, uint dom_depth); |
| 1101 | // Locally compute IDOM using dom_lca call |
| 1102 | Node *compute_idom( Node *region ) const; |
| 1103 | // Recompute dom_depth |
| 1104 | void recompute_dom_depth(); |
| 1105 | |
| 1106 | // Is safept not required by an outer loop? |
| 1107 | bool is_deleteable_safept(Node* sfpt); |
| 1108 | |
| 1109 | // Replace parallel induction variable (parallel to trip counter) |
| 1110 | void replace_parallel_iv(IdealLoopTree *loop); |
| 1111 | |
| 1112 | Node *dom_lca( Node *n1, Node *n2 ) const { |
| 1113 | return find_non_split_ctrl(dom_lca_internal(n1, n2)); |
| 1114 | } |
| 1115 | Node *dom_lca_internal( Node *n1, Node *n2 ) const; |
| 1116 | |
| 1117 | // Build and verify the loop tree without modifying the graph. This |
| 1118 | // is useful to verify that all inputs properly dominate their uses. |
| 1119 | static void verify(PhaseIterGVN& igvn) { |
| 1120 | #ifdef ASSERT1 |
| 1121 | ResourceMark rm; |
| 1122 | Compile::TracePhase tp("idealLoopVerify", &timers[_t_idealLoopVerify]); |
| 1123 | PhaseIdealLoop v(igvn); |
| 1124 | #endif |
| 1125 | } |
| 1126 | |
| 1127 | // Recommended way to use PhaseIdealLoop. |
| 1128 | // Run PhaseIdealLoop in some mode and allocates a local scope for memory allocations. |
| 1129 | static void optimize(PhaseIterGVN &igvn, LoopOptsMode mode) { |
| 1130 | ResourceMark rm; |
| 1131 | PhaseIdealLoop v(igvn, mode); |
| 1132 | |
| 1133 | Compile* C = Compile::current(); |
| 1134 | if (!C->failing()) { |
| 1135 | // Cleanup any modified bits |
| 1136 | igvn.optimize(); |
| 1137 | |
| 1138 | v.log_loop_tree(); |
| 1139 | } |
| 1140 | } |
| 1141 | |
| 1142 | // True if the method has at least 1 irreducible loop |
| 1143 | bool _has_irreducible_loops; |
| 1144 | |
| 1145 | // Per-Node transform |
| 1146 | virtual Node* transform(Node* n) { return NULL__null; } |
| 1147 | |
| 1148 | Node* loop_exit_control(Node* x, IdealLoopTree* loop); |
| 1149 | Node* loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob); |
| 1150 | Node* loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr); |
| 1151 | Node* loop_iv_stride(Node* incr, IdealLoopTree* loop, Node*& xphi); |
| 1152 | PhiNode* loop_iv_phi(Node* xphi, Node* phi_incr, Node* x, IdealLoopTree* loop); |
| 1153 | |
| 1154 | bool is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt); |
| 1155 | |
| 1156 | Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt); |
| 1157 | bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new); |
| 1158 | #ifdef ASSERT1 |
| 1159 | bool convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop); |
| 1160 | #endif |
| 1161 | void add_empty_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt); |
| 1162 | SafePointNode* find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop); |
| 1163 | IdealLoopTree* insert_outer_loop(IdealLoopTree* loop, LoopNode* outer_l, Node* outer_ift); |
| 1164 | IdealLoopTree* create_outer_strip_mined_loop(BoolNode *test, Node *cmp, Node *init_control, |
| 1165 | IdealLoopTree* loop, float cl_prob, float le_fcnt, |
| 1166 | Node*& entry_control, Node*& iffalse); |
| 1167 | |
| 1168 | Node* exact_limit( IdealLoopTree *loop ); |
| 1169 | |
| 1170 | // Return a post-walked LoopNode |
| 1171 | IdealLoopTree *get_loop( Node *n ) const { |
| 1172 | // Dead nodes have no loop, so return the top level loop instead |
| 1173 | if (!has_node(n)) return _ltree_root; |
| 1174 | assert(!has_ctrl(n), "")do { if (!(!has_ctrl(n))) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1174, "assert(" "!has_ctrl(n)" ") failed", ""); ::breakpoint (); } } while (0); |
| 1175 | return (IdealLoopTree*)_nodes[n->_idx]; |
| 1176 | } |
| 1177 | |
| 1178 | IdealLoopTree* ltree_root() const { return _ltree_root; } |
| 1179 | |
| 1180 | // Is 'n' a (nested) member of 'loop'? |
| 1181 | int is_member( const IdealLoopTree *loop, Node *n ) const { |
| 1182 | return loop->is_member(get_loop(n)); } |
| 1183 | |
| 1184 | // This is the basic building block of the loop optimizations. It clones an |
| 1185 | // entire loop body. It makes an old_new loop body mapping; with this |
| 1186 | // mapping you can find the new-loop equivalent to an old-loop node. All |
| 1187 | // new-loop nodes are exactly equal to their old-loop counterparts, all |
| 1188 | // edges are the same. All exits from the old-loop now have a RegionNode |
| 1189 | // that merges the equivalent new-loop path. This is true even for the |
| 1190 | // normal "loop-exit" condition. All uses of loop-invariant old-loop values |
| 1191 | // now come from (one or more) Phis that merge their new-loop equivalents. |
| 1192 | // Parameter side_by_side_idom: |
| 1193 | // When side_by_size_idom is NULL, the dominator tree is constructed for |
| 1194 | // the clone loop to dominate the original. Used in construction of |
| 1195 | // pre-main-post loop sequence. |
| 1196 | // When nonnull, the clone and original are side-by-side, both are |
| 1197 | // dominated by the passed in side_by_side_idom node. Used in |
| 1198 | // construction of unswitched loops. |
| 1199 | enum CloneLoopMode { |
| 1200 | IgnoreStripMined = 0, // Only clone inner strip mined loop |
| 1201 | CloneIncludesStripMined = 1, // clone both inner and outer strip mined loops |
| 1202 | ControlAroundStripMined = 2 // Only clone inner strip mined loop, |
| 1203 | // result control flow branches |
| 1204 | // either to inner clone or outer |
| 1205 | // strip mined loop. |
| 1206 | }; |
| 1207 | void clone_loop( IdealLoopTree *loop, Node_List &old_new, int dom_depth, |
| 1208 | CloneLoopMode mode, Node* side_by_side_idom = NULL__null); |
| 1209 | void clone_loop_handle_data_uses(Node* old, Node_List &old_new, |
| 1210 | IdealLoopTree* loop, IdealLoopTree* companion_loop, |
| 1211 | Node_List*& split_if_set, Node_List*& split_bool_set, |
| 1212 | Node_List*& split_cex_set, Node_List& worklist, |
| 1213 | uint new_counter, CloneLoopMode mode); |
| 1214 | void clone_outer_loop(LoopNode* head, CloneLoopMode mode, IdealLoopTree *loop, |
| 1215 | IdealLoopTree* outer_loop, int dd, Node_List &old_new, |
| 1216 | Node_List& extra_data_nodes); |
| 1217 | |
| 1218 | // If we got the effect of peeling, either by actually peeling or by |
| 1219 | // making a pre-loop which must execute at least once, we can remove |
| 1220 | // all loop-invariant dominated tests in the main body. |
| 1221 | void peeled_dom_test_elim( IdealLoopTree *loop, Node_List &old_new ); |
| 1222 | |
| 1223 | // Generate code to do a loop peel for the given loop (and body). |
| 1224 | // old_new is a temp array. |
| 1225 | void do_peeling( IdealLoopTree *loop, Node_List &old_new ); |
| 1226 | |
| 1227 | // Add pre and post loops around the given loop. These loops are used |
| 1228 | // during RCE, unrolling and aligning loops. |
| 1229 | void insert_pre_post_loops( IdealLoopTree *loop, Node_List &old_new, bool peel_only ); |
| 1230 | |
| 1231 | // Add post loop after the given loop. |
| 1232 | Node *insert_post_loop(IdealLoopTree* loop, Node_List& old_new, |
| 1233 | CountedLoopNode* main_head, CountedLoopEndNode* main_end, |
| 1234 | Node*& incr, Node* limit, CountedLoopNode*& post_head); |
| 1235 | |
| 1236 | // Add an RCE'd post loop which we will multi-version adapt for run time test path usage |
| 1237 | void insert_scalar_rced_post_loop( IdealLoopTree *loop, Node_List &old_new ); |
| 1238 | |
| 1239 | // Add a vector post loop between a vector main loop and the current post loop |
| 1240 | void insert_vector_post_loop(IdealLoopTree *loop, Node_List &old_new); |
| 1241 | // If Node n lives in the back_ctrl block, we clone a private version of n |
| 1242 | // in preheader_ctrl block and return that, otherwise return n. |
| 1243 | Node *clone_up_backedge_goo( Node *back_ctrl, Node *preheader_ctrl, Node *n, VectorSet &visited, Node_Stack &clones ); |
| 1244 | |
| 1245 | // Take steps to maximally unroll the loop. Peel any odd iterations, then |
| 1246 | // unroll to do double iterations. The next round of major loop transforms |
| 1247 | // will repeat till the doubled loop body does all remaining iterations in 1 |
| 1248 | // pass. |
| 1249 | void do_maximally_unroll( IdealLoopTree *loop, Node_List &old_new ); |
| 1250 | |
| 1251 | // Unroll the loop body one step - make each trip do 2 iterations. |
| 1252 | void do_unroll( IdealLoopTree *loop, Node_List &old_new, bool adjust_min_trip ); |
| 1253 | |
| 1254 | // Mark vector reduction candidates before loop unrolling |
| 1255 | void mark_reductions( IdealLoopTree *loop ); |
| 1256 | |
| 1257 | // Return true if exp is a constant times an induction var |
| 1258 | bool is_scaled_iv(Node* exp, Node* iv, jlong* p_scale, BasicType bt, bool* converted); |
| 1259 | |
| 1260 | bool is_iv(Node* exp, Node* iv, BasicType bt); |
| 1261 | |
| 1262 | // Return true if exp is a scaled induction var plus (or minus) constant |
| 1263 | bool is_scaled_iv_plus_offset(Node* exp, Node* iv, jlong* p_scale, Node** p_offset, BasicType bt, bool* converted = NULL__null, int depth = 0); |
| 1264 | bool is_scaled_iv_plus_offset(Node* exp, Node* iv, int* p_scale, Node** p_offset) { |
| 1265 | jlong long_scale; |
| 1266 | if (is_scaled_iv_plus_offset(exp, iv, &long_scale, p_offset, T_INT)) { |
| 1267 | int int_scale = checked_cast<int>(long_scale); |
| 1268 | if (p_scale != NULL__null) { |
| 1269 | *p_scale = int_scale; |
| 1270 | } |
| 1271 | return true; |
| 1272 | } |
| 1273 | return false; |
| 1274 | } |
| 1275 | |
| 1276 | // Enum to determine the action to be performed in create_new_if_for_predicate() when processing phis of UCT regions. |
| 1277 | enum class UnswitchingAction { |
| 1278 | None, // No special action. |
| 1279 | FastLoopCloning, // Need to clone nodes for the fast loop. |
| 1280 | SlowLoopRewiring // Need to rewire nodes for the slow loop. |
| 1281 | }; |
| 1282 | |
| 1283 | // Create a new if above the uncommon_trap_if_pattern for the predicate to be promoted |
| 1284 | ProjNode* create_new_if_for_predicate(ProjNode* cont_proj, Node* new_entry, Deoptimization::DeoptReason reason, |
| 1285 | int opcode, bool if_cont_is_true_proj = true, Node_List* old_new = NULL__null, |
| 1286 | UnswitchingAction unswitching_action = UnswitchingAction::None); |
| 1287 | |
| 1288 | // Clone data nodes for the fast loop while creating a new If with create_new_if_for_predicate. |
| 1289 | Node* clone_data_nodes_for_fast_loop(Node* phi_input, ProjNode* uncommon_proj, Node* if_uct, Node_List* old_new); |
| 1290 | |
| 1291 | void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true); |
| 1292 | |
| 1293 | static Node* skip_all_loop_predicates(Node* entry); |
| 1294 | static Node* skip_loop_predicates(Node* entry); |
| 1295 | |
| 1296 | // Find a good location to insert a predicate |
| 1297 | static ProjNode* find_predicate_insertion_point(Node* start_c, Deoptimization::DeoptReason reason); |
| 1298 | // Find a predicate |
| 1299 | static Node* find_predicate(Node* entry); |
| 1300 | // Construct a range check for a predicate if |
| 1301 | BoolNode* rc_predicate(IdealLoopTree *loop, Node* ctrl, |
| 1302 | int scale, Node* offset, |
| 1303 | Node* init, Node* limit, jint stride, |
| 1304 | Node* range, bool upper, bool &overflow, |
| 1305 | bool negate); |
| 1306 | |
| 1307 | // Implementation of the loop predication to promote checks outside the loop |
| 1308 | bool loop_predication_impl(IdealLoopTree *loop); |
| 1309 | bool loop_predication_impl_helper(IdealLoopTree *loop, ProjNode* proj, ProjNode *predicate_proj, |
| 1310 | CountedLoopNode *cl, ConNode* zero, Invariance& invar, |
| 1311 | Deoptimization::DeoptReason reason); |
| 1312 | bool loop_predication_should_follow_branches(IdealLoopTree *loop, ProjNode *predicate_proj, float& loop_trip_cnt); |
| 1313 | void loop_predication_follow_branches(Node *c, IdealLoopTree *loop, float loop_trip_cnt, |
| 1314 | PathFrequency& pf, Node_Stack& stack, VectorSet& seen, |
| 1315 | Node_List& if_proj_list); |
| 1316 | ProjNode* insert_initial_skeleton_predicate(IfNode* iff, IdealLoopTree *loop, |
| 1317 | ProjNode* proj, ProjNode *predicate_proj, |
| 1318 | ProjNode* upper_bound_proj, |
| 1319 | int scale, Node* offset, |
| 1320 | Node* init, Node* limit, jint stride, |
| 1321 | Node* rng, bool& overflow, |
| 1322 | Deoptimization::DeoptReason reason); |
| 1323 | Node* add_range_check_predicate(IdealLoopTree* loop, CountedLoopNode* cl, |
| 1324 | Node* predicate_proj, int scale_con, Node* offset, |
| 1325 | Node* limit, jint stride_con, Node* value); |
| 1326 | |
| 1327 | // Helper function to collect predicate for eliminating the useless ones |
| 1328 | void collect_potentially_useful_predicates(IdealLoopTree *loop, Unique_Node_List &predicate_opaque1); |
| 1329 | void eliminate_useless_predicates(); |
| 1330 | |
| 1331 | // Change the control input of expensive nodes to allow commoning by |
| 1332 | // IGVN when it is guaranteed to not result in a more frequent |
| 1333 | // execution of the expensive node. Return true if progress. |
| 1334 | bool process_expensive_nodes(); |
| 1335 | |
| 1336 | // Check whether node has become unreachable |
| 1337 | bool is_node_unreachable(Node *n) const { |
| 1338 | return !has_node(n) || n->is_unreachable(_igvn); |
| 1339 | } |
| 1340 | |
| 1341 | // Eliminate range-checks and other trip-counter vs loop-invariant tests. |
| 1342 | int do_range_check( IdealLoopTree *loop, Node_List &old_new ); |
| 1343 | |
| 1344 | // Check to see if do_range_check(...) cleaned the main loop of range-checks |
| 1345 | void has_range_checks(IdealLoopTree *loop); |
| 1346 | |
| 1347 | // Process post loops which have range checks and try to build a multi-version |
| 1348 | // guard to safely determine if we can execute the post loop which was RCE'd. |
| 1349 | bool multi_version_post_loops(IdealLoopTree *rce_loop, IdealLoopTree *legacy_loop); |
| 1350 | |
| 1351 | // Cause the rce'd post loop to optimized away, this happens if we cannot complete multiverioning |
| 1352 | void poison_rce_post_loop(IdealLoopTree *rce_loop); |
| 1353 | |
| 1354 | // Create a slow version of the loop by cloning the loop |
| 1355 | // and inserting an if to select fast-slow versions. |
| 1356 | // Return the inserted if. |
| 1357 | IfNode* create_slow_version_of_loop(IdealLoopTree *loop, |
| 1358 | Node_List &old_new, |
| 1359 | IfNode* unswitch_iff, |
| 1360 | CloneLoopMode mode); |
| 1361 | |
| 1362 | // Clone a loop and return the clone head (clone_loop_head). |
| 1363 | // Added nodes include int(1), int(0) - disconnected, If, IfTrue, IfFalse, |
| 1364 | // This routine was created for usage in CountedLoopReserveKit. |
| 1365 | // |
| 1366 | // int(1) -> If -> IfTrue -> original_loop_head |
| 1367 | // | |
| 1368 | // V |
| 1369 | // IfFalse -> clone_loop_head (returned by function pointer) |
| 1370 | // |
| 1371 | LoopNode* create_reserve_version_of_loop(IdealLoopTree *loop, CountedLoopReserveKit* lk); |
| 1372 | // Clone loop with an invariant test (that does not exit) and |
| 1373 | // insert a clone of the test that selects which version to |
| 1374 | // execute. |
| 1375 | void do_unswitching (IdealLoopTree *loop, Node_List &old_new); |
| 1376 | |
| 1377 | // Find candidate "if" for unswitching |
| 1378 | IfNode* find_unswitching_candidate(const IdealLoopTree *loop) const; |
| 1379 | |
| 1380 | // Range Check Elimination uses this function! |
| 1381 | // Constrain the main loop iterations so the affine function: |
| 1382 | // low_limit <= scale_con * I + offset < upper_limit |
| 1383 | // always holds true. That is, either increase the number of iterations in |
| 1384 | // the pre-loop or the post-loop until the condition holds true in the main |
| 1385 | // loop. Scale_con, offset and limit are all loop invariant. |
| 1386 | void add_constraint(jlong stride_con, jlong scale_con, Node* offset, Node* low_limit, Node* upper_limit, Node* pre_ctrl, Node** pre_limit, Node** main_limit); |
| 1387 | // Helper function for add_constraint(). |
| 1388 | Node* adjust_limit(bool reduce, Node* scale, Node* offset, Node* rc_limit, Node* old_limit, Node* pre_ctrl, bool round); |
| 1389 | |
| 1390 | // Partially peel loop up through last_peel node. |
| 1391 | bool partial_peel( IdealLoopTree *loop, Node_List &old_new ); |
| 1392 | |
| 1393 | // Create a scheduled list of nodes control dependent on ctrl set. |
| 1394 | void scheduled_nodelist( IdealLoopTree *loop, VectorSet& ctrl, Node_List &sched ); |
| 1395 | // Has a use in the vector set |
| 1396 | bool has_use_in_set( Node* n, VectorSet& vset ); |
| 1397 | // Has use internal to the vector set (ie. not in a phi at the loop head) |
| 1398 | bool has_use_internal_to_set( Node* n, VectorSet& vset, IdealLoopTree *loop ); |
| 1399 | // clone "n" for uses that are outside of loop |
| 1400 | int clone_for_use_outside_loop( IdealLoopTree *loop, Node* n, Node_List& worklist ); |
| 1401 | // clone "n" for special uses that are in the not_peeled region |
| 1402 | void clone_for_special_use_inside_loop( IdealLoopTree *loop, Node* n, |
| 1403 | VectorSet& not_peel, Node_List& sink_list, Node_List& worklist ); |
| 1404 | // Insert phi(lp_entry_val, back_edge_val) at use->in(idx) for loop lp if phi does not already exist |
| 1405 | void insert_phi_for_loop( Node* use, uint idx, Node* lp_entry_val, Node* back_edge_val, LoopNode* lp ); |
| 1406 | #ifdef ASSERT1 |
| 1407 | // Validate the loop partition sets: peel and not_peel |
| 1408 | bool is_valid_loop_partition( IdealLoopTree *loop, VectorSet& peel, Node_List& peel_list, VectorSet& not_peel ); |
| 1409 | // Ensure that uses outside of loop are of the right form |
| 1410 | bool is_valid_clone_loop_form( IdealLoopTree *loop, Node_List& peel_list, |
| 1411 | uint orig_exit_idx, uint clone_exit_idx); |
| 1412 | bool is_valid_clone_loop_exit_use( IdealLoopTree *loop, Node* use, uint exit_idx); |
| 1413 | #endif |
| 1414 | |
| 1415 | // Returns nonzero constant stride if-node is a possible iv test (otherwise returns zero.) |
| 1416 | int stride_of_possible_iv( Node* iff ); |
| 1417 | bool is_possible_iv_test( Node* iff ) { return stride_of_possible_iv(iff) != 0; } |
| 1418 | // Return the (unique) control output node that's in the loop (if it exists.) |
| 1419 | Node* stay_in_loop( Node* n, IdealLoopTree *loop); |
| 1420 | // Insert a signed compare loop exit cloned from an unsigned compare. |
| 1421 | IfNode* insert_cmpi_loop_exit(IfNode* if_cmpu, IdealLoopTree *loop); |
| 1422 | void remove_cmpi_loop_exit(IfNode* if_cmp, IdealLoopTree *loop); |
| 1423 | // Utility to register node "n" with PhaseIdealLoop |
| 1424 | void register_node(Node* n, IdealLoopTree *loop, Node* pred, int ddepth); |
| 1425 | // Utility to create an if-projection |
| 1426 | ProjNode* proj_clone(ProjNode* p, IfNode* iff); |
| 1427 | // Force the iff control output to be the live_proj |
| 1428 | Node* short_circuit_if(IfNode* iff, ProjNode* live_proj); |
| 1429 | // Insert a region before an if projection |
| 1430 | RegionNode* insert_region_before_proj(ProjNode* proj); |
| 1431 | // Insert a new if before an if projection |
| 1432 | ProjNode* insert_if_before_proj(Node* left, bool Signed, BoolTest::mask relop, Node* right, ProjNode* proj); |
| 1433 | |
| 1434 | // Passed in a Phi merging (recursively) some nearly equivalent Bool/Cmps. |
| 1435 | // "Nearly" because all Nodes have been cloned from the original in the loop, |
| 1436 | // but the fall-in edges to the Cmp are different. Clone bool/Cmp pairs |
| 1437 | // through the Phi recursively, and return a Bool. |
| 1438 | Node *clone_iff( PhiNode *phi, IdealLoopTree *loop ); |
| 1439 | CmpNode *clone_bool( PhiNode *phi, IdealLoopTree *loop ); |
| 1440 | |
| 1441 | |
| 1442 | // Rework addressing expressions to get the most loop-invariant stuff |
| 1443 | // moved out. We'd like to do all associative operators, but it's especially |
| 1444 | // important (common) to do address expressions. |
| 1445 | Node *remix_address_expressions( Node *n ); |
| 1446 | |
| 1447 | // Convert add to muladd to generate MuladdS2I under certain criteria |
| 1448 | Node * convert_add_to_muladd(Node * n); |
| 1449 | |
| 1450 | // Attempt to use a conditional move instead of a phi/branch |
| 1451 | Node *conditional_move( Node *n ); |
| 1452 | |
| 1453 | // Reorganize offset computations to lower register pressure. |
| 1454 | // Mostly prevent loop-fallout uses of the pre-incremented trip counter |
| 1455 | // (which are then alive with the post-incremented trip counter |
| 1456 | // forcing an extra register move) |
| 1457 | void reorg_offsets( IdealLoopTree *loop ); |
| 1458 | |
| 1459 | // Check for aggressive application of 'split-if' optimization, |
| 1460 | // using basic block level info. |
| 1461 | void split_if_with_blocks ( VectorSet &visited, Node_Stack &nstack); |
| 1462 | Node *split_if_with_blocks_pre ( Node *n ); |
| 1463 | void split_if_with_blocks_post( Node *n ); |
| 1464 | Node *has_local_phi_input( Node *n ); |
| 1465 | // Mark an IfNode as being dominated by a prior test, |
| 1466 | // without actually altering the CFG (and hence IDOM info). |
| 1467 | void dominated_by( Node *prevdom, Node *iff, bool flip = false, bool exclude_loop_predicate = false ); |
| 1468 | |
| 1469 | // Split Node 'n' through merge point |
| 1470 | Node *split_thru_region( Node *n, Node *region ); |
| 1471 | // Split Node 'n' through merge point if there is enough win. |
| 1472 | Node *split_thru_phi( Node *n, Node *region, int policy ); |
| 1473 | // Found an If getting its condition-code input from a Phi in the |
| 1474 | // same block. Split thru the Region. |
| 1475 | void do_split_if( Node *iff ); |
| 1476 | |
| 1477 | // Conversion of fill/copy patterns into intrinsic versions |
| 1478 | bool do_intrinsify_fill(); |
| 1479 | bool intrinsify_fill(IdealLoopTree* lpt); |
| 1480 | bool match_fill_loop(IdealLoopTree* lpt, Node*& store, Node*& store_value, |
| 1481 | Node*& shift, Node*& offset); |
| 1482 | |
| 1483 | private: |
| 1484 | // Return a type based on condition control flow |
| 1485 | const TypeInt* filtered_type( Node *n, Node* n_ctrl); |
| 1486 | const TypeInt* filtered_type( Node *n ) { return filtered_type(n, NULL__null); } |
| 1487 | // Helpers for filtered type |
| 1488 | const TypeInt* filtered_type_from_dominators( Node* val, Node *val_ctrl); |
| 1489 | |
| 1490 | // Helper functions |
| 1491 | Node *spinup( Node *iff, Node *new_false, Node *new_true, Node *region, Node *phi, small_cache *cache ); |
| 1492 | Node *find_use_block( Node *use, Node *def, Node *old_false, Node *new_false, Node *old_true, Node *new_true ); |
| 1493 | void handle_use( Node *use, Node *def, small_cache *cache, Node *region_dom, Node *new_false, Node *new_true, Node *old_false, Node *old_true ); |
| 1494 | bool split_up( Node *n, Node *blk1, Node *blk2 ); |
| 1495 | void sink_use( Node *use, Node *post_loop ); |
| 1496 | Node* place_outside_loop(Node* useblock, IdealLoopTree* loop) const; |
| 1497 | Node* try_move_store_before_loop(Node* n, Node *n_ctrl); |
| 1498 | void try_move_store_after_loop(Node* n); |
| 1499 | bool identical_backtoback_ifs(Node *n); |
| 1500 | bool can_split_if(Node *n_ctrl); |
| 1501 | |
| 1502 | // Determine if a method is too big for a/another round of split-if, based on |
| 1503 | // a magic (approximate) ratio derived from the equally magic constant 35000, |
| 1504 | // previously used for this purpose (but without relating to the node limit). |
| 1505 | bool must_throttle_split_if() { |
| 1506 | uint threshold = C->max_node_limit() * 2 / 5; |
| 1507 | return C->live_nodes() > threshold; |
| 1508 | } |
| 1509 | |
| 1510 | // A simplistic node request tracking mechanism, where |
| 1511 | // = UINT_MAX Request not valid or made final. |
| 1512 | // < UINT_MAX Nodes currently requested (estimate). |
| 1513 | uint _nodes_required; |
| 1514 | |
| 1515 | enum { REQUIRE_MIN = 70 }; |
| 1516 | |
| 1517 | uint nodes_required() const { return _nodes_required; } |
| 1518 | |
| 1519 | // Given the _currently_ available number of nodes, check whether there is |
| 1520 | // "room" for an additional request or not, considering the already required |
| 1521 | // number of nodes. Return TRUE if the new request is exceeding the node |
| 1522 | // budget limit, otherwise return FALSE. Note that this interpretation will |
| 1523 | // act pessimistic on additional requests when new nodes have already been |
| 1524 | // generated since the 'begin'. This behaviour fits with the intention that |
| 1525 | // node estimates/requests should be made upfront. |
| 1526 | bool exceeding_node_budget(uint required = 0) { |
| 1527 | assert(C->live_nodes() < C->max_node_limit(), "sanity")do { if (!(C->live_nodes() < C->max_node_limit())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1527, "assert(" "C->live_nodes() < C->max_node_limit()" ") failed", "sanity"); ::breakpoint(); } } while (0); |
| 1528 | uint available = C->max_node_limit() - C->live_nodes(); |
| 1529 | return available < required + _nodes_required + REQUIRE_MIN; |
| 1530 | } |
| 1531 | |
| 1532 | uint require_nodes(uint require, uint minreq = REQUIRE_MIN) { |
| 1533 | precond(require > 0)do { if (!(require > 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1533, "assert(" "require > 0" ") failed", "precond"); :: breakpoint(); } } while (0); |
| 1534 | _nodes_required += MAX2(require, minreq); |
| 1535 | return _nodes_required; |
| 1536 | } |
| 1537 | |
| 1538 | bool may_require_nodes(uint require, uint minreq = REQUIRE_MIN) { |
| 1539 | return !exceeding_node_budget(require) && require_nodes(require, minreq) > 0; |
| 1540 | } |
| 1541 | |
| 1542 | uint require_nodes_begin() { |
| 1543 | assert(_nodes_required == UINT_MAX, "Bad state (begin).")do { if (!(_nodes_required == (2147483647 *2U +1U))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1543, "assert(" "_nodes_required == (2147483647 *2U +1U)" ") failed" , "Bad state (begin)."); ::breakpoint(); } } while (0); |
| 1544 | _nodes_required = 0; |
| 1545 | return C->live_nodes(); |
| 1546 | } |
| 1547 | |
| 1548 | // When a node request is final, optionally check that the requested number |
| 1549 | // of nodes was reasonably correct with respect to the number of new nodes |
| 1550 | // introduced since the last 'begin'. Always check that we have not exceeded |
| 1551 | // the maximum node limit. |
| 1552 | void require_nodes_final(uint live_at_begin, bool check_estimate) { |
| 1553 | assert(_nodes_required < UINT_MAX, "Bad state (final).")do { if (!(_nodes_required < (2147483647 *2U +1U))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1553, "assert(" "_nodes_required < (2147483647 *2U +1U)" ") failed", "Bad state (final)."); ::breakpoint(); } } while (0); |
| 1554 | |
| 1555 | #ifdef ASSERT1 |
| 1556 | if (check_estimate) { |
| 1557 | // Check that the node budget request was not off by too much (x2). |
| 1558 | // Should this be the case we _surely_ need to improve the estimates |
| 1559 | // used in our budget calculations. |
| 1560 | if (C->live_nodes() - live_at_begin > 2 * _nodes_required) { |
| 1561 | log_info(compilation)(!(LogImpl<(LogTag::_compilation), (LogTag::__NO_TAG), (LogTag ::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag:: __NO_TAG)>::is_level(LogLevel::Info))) ? (void)0 : LogImpl <(LogTag::_compilation), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::write<LogLevel::Info>("Bad node estimate: actual = %d >> request = %d", |
| 1562 | C->live_nodes() - live_at_begin, _nodes_required); |
| 1563 | } |
| 1564 | } |
| 1565 | #endif |
| 1566 | // Assert that we have stayed within the node budget limit. |
| 1567 | assert(C->live_nodes() < C->max_node_limit(),do { if (!(C->live_nodes() < C->max_node_limit())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1570, "assert(" "C->live_nodes() < C->max_node_limit()" ") failed", "Exceeding node budget limit: %d + %d > %d (request = %d)" , C->live_nodes() - live_at_begin, live_at_begin, C->max_node_limit (), _nodes_required); ::breakpoint(); } } while (0) |
| 1568 | "Exceeding node budget limit: %d + %d > %d (request = %d)",do { if (!(C->live_nodes() < C->max_node_limit())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1570, "assert(" "C->live_nodes() < C->max_node_limit()" ") failed", "Exceeding node budget limit: %d + %d > %d (request = %d)" , C->live_nodes() - live_at_begin, live_at_begin, C->max_node_limit (), _nodes_required); ::breakpoint(); } } while (0) |
| 1569 | C->live_nodes() - live_at_begin, live_at_begin,do { if (!(C->live_nodes() < C->max_node_limit())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1570, "assert(" "C->live_nodes() < C->max_node_limit()" ") failed", "Exceeding node budget limit: %d + %d > %d (request = %d)" , C->live_nodes() - live_at_begin, live_at_begin, C->max_node_limit (), _nodes_required); ::breakpoint(); } } while (0) |
| 1570 | C->max_node_limit(), _nodes_required)do { if (!(C->live_nodes() < C->max_node_limit())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1570, "assert(" "C->live_nodes() < C->max_node_limit()" ") failed", "Exceeding node budget limit: %d + %d > %d (request = %d)" , C->live_nodes() - live_at_begin, live_at_begin, C->max_node_limit (), _nodes_required); ::breakpoint(); } } while (0); |
| 1571 | |
| 1572 | _nodes_required = UINT_MAX(2147483647 *2U +1U); |
| 1573 | } |
| 1574 | |
| 1575 | // Clone loop predicates to slow and fast loop when unswitching a loop |
| 1576 | void clone_predicates_to_unswitched_loop(IdealLoopTree* loop, Node_List& old_new, ProjNode*& iffast_pred, ProjNode*& ifslow_pred); |
| 1577 | ProjNode* clone_predicate_to_unswitched_loop(ProjNode* predicate_proj, Node* new_entry, Deoptimization::DeoptReason reason, |
| 1578 | Node_List* old_new = NULL__null); |
| 1579 | void clone_skeleton_predicates_to_unswitched_loop(IdealLoopTree* loop, const Node_List& old_new, Deoptimization::DeoptReason reason, |
| 1580 | ProjNode* old_predicate_proj, ProjNode* iffast_pred, ProjNode* ifslow_pred); |
| 1581 | ProjNode* clone_skeleton_predicate_for_unswitched_loops(Node* iff, ProjNode* predicate, |
| 1582 | Deoptimization::DeoptReason reason, |
| 1583 | ProjNode* output_proj); |
| 1584 | static void check_created_predicate_for_unswitching(const Node* new_entry) PRODUCT_RETURN; |
| 1585 | |
| 1586 | bool _created_loop_node; |
| 1587 | #ifdef ASSERT1 |
| 1588 | void dump_real_LCA(Node* early, Node* wrong_lca); |
| 1589 | bool check_idom_chains_intersection(const Node* n, uint& idom_idx_new, uint& idom_idx_other, const Node_List* nodes_seen) const; |
| 1590 | #endif |
| 1591 | |
| 1592 | public: |
| 1593 | void set_created_loop_node() { _created_loop_node = true; } |
| 1594 | bool created_loop_node() { return _created_loop_node; } |
| 1595 | void register_new_node(Node* n, Node* blk); |
| 1596 | |
| 1597 | #ifdef ASSERT1 |
| 1598 | void dump_bad_graph(const char* msg, Node* n, Node* early, Node* LCA); |
| 1599 | #endif |
| 1600 | |
| 1601 | #ifndef PRODUCT |
| 1602 | void dump() const; |
| 1603 | void dump_idom(Node* n) const; |
| 1604 | void dump(IdealLoopTree* loop, uint rpo_idx, Node_List &rpo_list) const; |
| 1605 | void verify() const; // Major slow :-) |
| 1606 | void verify_compare(Node* n, const PhaseIdealLoop* loop_verify, VectorSet &visited) const; |
| 1607 | IdealLoopTree* get_loop_idx(Node* n) const { |
| 1608 | // Dead nodes have no loop, so return the top level loop instead |
| 1609 | return _nodes[n->_idx] ? (IdealLoopTree*)_nodes[n->_idx] : _ltree_root; |
| 1610 | } |
| 1611 | // Print some stats |
| 1612 | static void print_statistics(); |
| 1613 | static int _loop_invokes; // Count of PhaseIdealLoop invokes |
| 1614 | static int _loop_work; // Sum of PhaseIdealLoop x _unique |
| 1615 | static volatile int _long_loop_candidates; |
| 1616 | static volatile int _long_loop_nests; |
| 1617 | static volatile int _long_loop_counted_loops; |
| 1618 | #endif |
| 1619 | |
| 1620 | void rpo(Node* start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list) const; |
| 1621 | |
| 1622 | void check_counted_loop_shape(IdealLoopTree* loop, Node* x, BasicType bt) NOT_DEBUG_RETURN; |
| 1623 | |
| 1624 | LoopNode* create_inner_head(IdealLoopTree* loop, BaseCountedLoopNode* head, IfNode* exit_test); |
| 1625 | |
| 1626 | |
| 1627 | int extract_long_range_checks(const IdealLoopTree* loop, jlong stride_con, int iters_limit, PhiNode* phi, |
| 1628 | Node_List &range_checks); |
| 1629 | |
| 1630 | void transform_long_range_checks(int stride_con, const Node_List &range_checks, Node* outer_phi, |
| 1631 | Node* inner_iters_actual_int, Node* inner_phi, |
| 1632 | Node* iv_add, LoopNode* inner_head); |
| 1633 | |
| 1634 | Node* get_late_ctrl_with_anti_dep(LoadNode* n, Node* early, Node* LCA); |
| 1635 | |
| 1636 | bool ctrl_of_use_out_of_loop(const Node* n, Node* n_ctrl, IdealLoopTree* n_loop, Node* ctrl); |
| 1637 | |
| 1638 | bool ctrl_of_all_uses_out_of_loop(const Node* n, Node* n_ctrl, IdealLoopTree* n_loop); |
| 1639 | |
| 1640 | Node* compute_early_ctrl(Node* n, Node* n_ctrl); |
| 1641 | |
| 1642 | void try_sink_out_of_loop(Node* n); |
| 1643 | |
| 1644 | Node* clamp(Node* R, Node* L, Node* H); |
| 1645 | |
| 1646 | bool safe_for_if_replacement(const Node* dom) const; |
| 1647 | |
| 1648 | void strip_mined_nest_back_to_counted_loop(IdealLoopTree* loop, const BaseCountedLoopNode* head, Node* back_control, |
| 1649 | IfNode*&exit_test, SafePointNode*&safepoint); |
| 1650 | }; |
| 1651 | |
| 1652 | |
| 1653 | class AutoNodeBudget : public StackObj |
| 1654 | { |
| 1655 | public: |
| 1656 | enum budget_check_t { BUDGET_CHECK, NO_BUDGET_CHECK }; |
| 1657 | |
| 1658 | AutoNodeBudget(PhaseIdealLoop* phase, budget_check_t chk = BUDGET_CHECK) |
| 1659 | : _phase(phase), |
| 1660 | _check_at_final(chk == BUDGET_CHECK), |
| 1661 | _nodes_at_begin(0) |
| 1662 | { |
| 1663 | precond(_phase != NULL)do { if (!(_phase != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/opto/loopnode.hpp" , 1663, "assert(" "_phase != __null" ") failed", "precond"); :: breakpoint(); } } while (0); |
| 1664 | |
| 1665 | _nodes_at_begin = _phase->require_nodes_begin(); |
| 1666 | } |
| 1667 | |
| 1668 | ~AutoNodeBudget() { |
| 1669 | #ifndef PRODUCT |
| 1670 | if (TraceLoopOpts) { |
| 1671 | uint request = _phase->nodes_required(); |
| 1672 | uint delta = _phase->C->live_nodes() - _nodes_at_begin; |
| 1673 | |
| 1674 | if (request < delta) { |
| 1675 | tty->print_cr("Exceeding node budget: %d < %d", request, delta); |
| 1676 | } else { |
| 1677 | uint const REQUIRE_MIN = PhaseIdealLoop::REQUIRE_MIN; |
| 1678 | // Identify the worst estimates as "poor" ones. |
| 1679 | if (request > REQUIRE_MIN && delta > 0) { |
| 1680 | if ((delta > REQUIRE_MIN && request > 3 * delta) || |
| 1681 | (delta <= REQUIRE_MIN && request > 10 * delta)) { |
| 1682 | tty->print_cr("Poor node estimate: %d >> %d", request, delta); |
| 1683 | } |
| 1684 | } |
| 1685 | } |
| 1686 | } |
| 1687 | #endif // PRODUCT |
| 1688 | _phase->require_nodes_final(_nodes_at_begin, _check_at_final); |
| 1689 | } |
| 1690 | |
| 1691 | private: |
| 1692 | PhaseIdealLoop* _phase; |
| 1693 | bool _check_at_final; |
| 1694 | uint _nodes_at_begin; |
| 1695 | }; |
| 1696 | |
| 1697 | |
| 1698 | // This kit may be used for making of a reserved copy of a loop before this loop |
| 1699 | // goes under non-reversible changes. |
| 1700 | // |
| 1701 | // Function create_reserve() creates a reserved copy (clone) of the loop. |
| 1702 | // The reserved copy is created by calling |
| 1703 | // PhaseIdealLoop::create_reserve_version_of_loop - see there how |
| 1704 | // the original and reserved loops are connected in the outer graph. |
| 1705 | // If create_reserve succeeded, it returns 'true' and _has_reserved is set to 'true'. |
| 1706 | // |
| 1707 | // By default the reserved copy (clone) of the loop is created as dead code - it is |
| 1708 | // dominated in the outer loop by this node chain: |
| 1709 | // intcon(1)->If->IfFalse->reserved_copy. |
| 1710 | // The original loop is dominated by the the same node chain but IfTrue projection: |
| 1711 | // intcon(0)->If->IfTrue->original_loop. |
| 1712 | // |
| 1713 | // In this implementation of CountedLoopReserveKit the ctor includes create_reserve() |
| 1714 | // and the dtor, checks _use_new value. |
| 1715 | // If _use_new == false, it "switches" control to reserved copy of the loop |
| 1716 | // by simple replacing of node intcon(1) with node intcon(0). |
| 1717 | // |
| 1718 | // Here is a proposed example of usage (see also SuperWord::output in superword.cpp). |
| 1719 | // |
| 1720 | // void CountedLoopReserveKit_example() |
| 1721 | // { |
| 1722 | // CountedLoopReserveKit lrk((phase, lpt, DoReserveCopy = true); // create local object |
| 1723 | // if (DoReserveCopy && !lrk.has_reserved()) { |
| 1724 | // return; //failed to create reserved loop copy |
| 1725 | // } |
| 1726 | // ... |
| 1727 | // //something is wrong, switch to original loop |
| 1728 | /// if(something_is_wrong) return; // ~CountedLoopReserveKit makes the switch |
| 1729 | // ... |
| 1730 | // //everything worked ok, return with the newly modified loop |
| 1731 | // lrk.use_new(); |
| 1732 | // return; // ~CountedLoopReserveKit does nothing once use_new() was called |
| 1733 | // } |
| 1734 | // |
| 1735 | // Keep in mind, that by default if create_reserve() is not followed by use_new() |
| 1736 | // the dtor will "switch to the original" loop. |
| 1737 | // NOTE. You you modify outside of the original loop this class is no help. |
| 1738 | // |
| 1739 | class CountedLoopReserveKit { |
| 1740 | private: |
| 1741 | PhaseIdealLoop* _phase; |
| 1742 | IdealLoopTree* _lpt; |
| 1743 | LoopNode* _lp; |
| 1744 | IfNode* _iff; |
| 1745 | LoopNode* _lp_reserved; |
| 1746 | bool _has_reserved; |
| 1747 | bool _use_new; |
| 1748 | const bool _active; //may be set to false in ctor, then the object is dummy |
| 1749 | |
| 1750 | public: |
| 1751 | CountedLoopReserveKit(PhaseIdealLoop* phase, IdealLoopTree *loop, bool active); |
| 1752 | ~CountedLoopReserveKit(); |
| 1753 | void use_new() {_use_new = true;} |
| 1754 | void set_iff(IfNode* x) {_iff = x;} |
| 1755 | bool has_reserved() const { return _active && _has_reserved;} |
| 1756 | private: |
| 1757 | bool create_reserve(); |
| 1758 | };// class CountedLoopReserveKit |
| 1759 | |
| 1760 | inline Node* IdealLoopTree::tail() { |
| 1761 | // Handle lazy update of _tail field. |
| 1762 | if (_tail->in(0) == NULL__null) { |
| 1763 | _tail = _phase->get_ctrl(_tail); |
| 1764 | } |
| 1765 | return _tail; |
| 1766 | } |
| 1767 | |
| 1768 | inline Node* IdealLoopTree::head() { |
| 1769 | // Handle lazy update of _head field. |
| 1770 | if (_head->in(0) == NULL__null) { |
| 1771 | _head = _phase->get_ctrl(_head); |
| 1772 | } |
| 1773 | return _head; |
| 1774 | } |
| 1775 | |
| 1776 | // Iterate over the loop tree using a preorder, left-to-right traversal. |
| 1777 | // |
| 1778 | // Example that visits all counted loops from within PhaseIdealLoop |
| 1779 | // |
| 1780 | // for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { |
| 1781 | // IdealLoopTree* lpt = iter.current(); |
| 1782 | // if (!lpt->is_counted()) continue; |
| 1783 | // ... |
| 1784 | class LoopTreeIterator : public StackObj { |
| 1785 | private: |
| 1786 | IdealLoopTree* _root; |
| 1787 | IdealLoopTree* _curnt; |
| 1788 | |
| 1789 | public: |
| 1790 | LoopTreeIterator(IdealLoopTree* root) : _root(root), _curnt(root) {} |
| 1791 | |
| 1792 | bool done() { return _curnt == NULL__null; } // Finished iterating? |
| 1793 | |
| 1794 | void next(); // Advance to next loop tree |
| 1795 | |
| 1796 | IdealLoopTree* current() { return _curnt; } // Return current value of iterator. |
| 1797 | }; |
| 1798 | |
| 1799 | #endif // SHARE_OPTO_LOOPNODE_HPP |