| File: | jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp |
| Warning: | line 93, column 9 1st function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | |||
| 2 | * Copyright (c) 2016, 2021, Oracle and/or its affiliates. All rights reserved. | |||
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |||
| 4 | * | |||
| 5 | * This code is free software; you can redistribute it and/or modify it | |||
| 6 | * under the terms of the GNU General Public License version 2 only, as | |||
| 7 | * published by the Free Software Foundation. | |||
| 8 | * | |||
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT | |||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |||
| 12 | * version 2 for more details (a copy is included in the LICENSE file that | |||
| 13 | * accompanied this code). | |||
| 14 | * | |||
| 15 | * You should have received a copy of the GNU General Public License version | |||
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, | |||
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |||
| 18 | * | |||
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |||
| 20 | * or visit www.oracle.com if you need additional information or have any | |||
| 21 | * questions. | |||
| 22 | * | |||
| 23 | */ | |||
| 24 | ||||
| 25 | #include "precompiled.hpp" | |||
| 26 | #include "classfile/javaClasses.inline.hpp" | |||
| 27 | #include "jfr/jni/jfrJavaSupport.hpp" | |||
| 28 | #include "jfr/leakprofiler/checkpoint/objectSampleCheckpoint.hpp" | |||
| 29 | #include "jfr/leakprofiler/leakProfiler.hpp" | |||
| 30 | #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" | |||
| 31 | #include "jfr/recorder/checkpoint/jfrCheckpointWriter.hpp" | |||
| 32 | #include "jfr/recorder/checkpoint/types/jfrTypeManager.hpp" | |||
| 33 | #include "jfr/recorder/checkpoint/types/jfrTypeSet.hpp" | |||
| 34 | #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" | |||
| 35 | #include "jfr/recorder/jfrRecorder.hpp" | |||
| 36 | #include "jfr/recorder/repository/jfrChunkWriter.hpp" | |||
| 37 | #include "jfr/recorder/service/jfrOptionSet.hpp" | |||
| 38 | #include "jfr/recorder/storage/jfrEpochStorage.inline.hpp" | |||
| 39 | #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" | |||
| 40 | #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" | |||
| 41 | #include "jfr/support/jfrKlassUnloading.hpp" | |||
| 42 | #include "jfr/utilities/jfrBigEndian.hpp" | |||
| 43 | #include "jfr/utilities/jfrIterator.hpp" | |||
| 44 | #include "jfr/utilities/jfrLinkedList.inline.hpp" | |||
| 45 | #include "jfr/utilities/jfrSignal.hpp" | |||
| 46 | #include "jfr/utilities/jfrThreadIterator.hpp" | |||
| 47 | #include "jfr/utilities/jfrTypes.hpp" | |||
| 48 | #include "jfr/writers/jfrJavaEventWriter.hpp" | |||
| 49 | #include "logging/log.hpp" | |||
| 50 | #include "memory/iterator.hpp" | |||
| 51 | #include "memory/resourceArea.hpp" | |||
| 52 | #include "runtime/atomic.hpp" | |||
| 53 | #include "runtime/handles.inline.hpp" | |||
| 54 | #include "runtime/interfaceSupport.inline.hpp" | |||
| 55 | #include "runtime/mutex.hpp" | |||
| 56 | #include "runtime/safepoint.hpp" | |||
| 57 | ||||
| 58 | typedef JfrCheckpointManager::BufferPtr BufferPtr; | |||
| 59 | ||||
| 60 | static JfrSignal _new_checkpoint; | |||
| 61 | static JfrCheckpointManager* _instance = NULL__null; | |||
| 62 | ||||
| 63 | JfrCheckpointManager& JfrCheckpointManager::instance() { | |||
| 64 | return *_instance; | |||
| 65 | } | |||
| 66 | ||||
| 67 | JfrCheckpointManager* JfrCheckpointManager::create(JfrChunkWriter& cw) { | |||
| 68 | assert(_instance == NULL, "invariant")do { if (!(_instance == __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 68, "assert(" "_instance == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 69 | _instance = new JfrCheckpointManager(cw); | |||
| 70 | return _instance; | |||
| 71 | } | |||
| 72 | ||||
| 73 | void JfrCheckpointManager::destroy() { | |||
| 74 | assert(_instance != NULL, "invariant")do { if (!(_instance != __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 74, "assert(" "_instance != __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 75 | delete _instance; | |||
| 76 | _instance = NULL__null; | |||
| 77 | } | |||
| 78 | ||||
| 79 | JfrCheckpointManager::JfrCheckpointManager(JfrChunkWriter& cw) : | |||
| 80 | _global_mspace(NULL__null), | |||
| 81 | _thread_local_mspace(NULL__null), | |||
| 82 | _chunkwriter(cw) {} | |||
| 83 | ||||
| 84 | JfrCheckpointManager::~JfrCheckpointManager() { | |||
| 85 | JfrTraceIdLoadBarrier::destroy(); | |||
| 86 | JfrTypeManager::destroy(); | |||
| 87 | delete _global_mspace; | |||
| 88 | delete _thread_local_mspace; | |||
| ||||
| 89 | } | |||
| 90 | ||||
| 91 | static const size_t global_buffer_prealloc_count = 2; | |||
| 92 | static const size_t global_buffer_size = 512 * K; | |||
| 93 | ||||
| 94 | static const size_t thread_local_buffer_prealloc_count = 16; | |||
| 95 | static const size_t thread_local_buffer_size = 128; | |||
| 96 | ||||
| 97 | bool JfrCheckpointManager::initialize() { | |||
| 98 | assert(_global_mspace == NULL, "invariant")do { if (!(_global_mspace == __null)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 98, "assert(" "_global_mspace == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 99 | _global_mspace = create_mspace<JfrCheckpointMspace, JfrCheckpointManager>(global_buffer_size, 0, 0, false, this); // post-pone preallocation | |||
| 100 | if (_global_mspace == NULL__null) { | |||
| 101 | return false; | |||
| 102 | } | |||
| 103 | // preallocate buffer count to each of the epoch live lists | |||
| 104 | for (size_t i = 0; i < global_buffer_prealloc_count * 2; ++i) { | |||
| 105 | Buffer* const buffer = mspace_allocate(global_buffer_size, _global_mspace); | |||
| 106 | _global_mspace->add_to_live_list(buffer, i % 2 == 0); | |||
| 107 | } | |||
| 108 | assert(_global_mspace->free_list_is_empty(), "invariant")do { if (!(_global_mspace->free_list_is_empty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 108, "assert(" "_global_mspace->free_list_is_empty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 109 | ||||
| 110 | assert(_thread_local_mspace == NULL, "invariant")do { if (!(_thread_local_mspace == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 110, "assert(" "_thread_local_mspace == __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 111 | _thread_local_mspace = new JfrThreadLocalCheckpointMspace(); | |||
| 112 | if (_thread_local_mspace == NULL__null || !_thread_local_mspace->initialize(thread_local_buffer_size, | |||
| 113 | JFR_MSPACE_UNLIMITED_CACHE_SIZE, | |||
| 114 | thread_local_buffer_prealloc_count)) { | |||
| 115 | return false; | |||
| 116 | } | |||
| 117 | return JfrTypeManager::initialize() && JfrTraceIdLoadBarrier::initialize(); | |||
| 118 | } | |||
| 119 | ||||
| 120 | #ifdef ASSERT1 | |||
| 121 | static void assert_lease(const BufferPtr buffer) { | |||
| 122 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 122, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 123 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 123, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 124 | assert(buffer->lease(), "invariant")do { if (!(buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 124, "assert(" "buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 125 | } | |||
| 126 | ||||
| 127 | static void assert_release(const BufferPtr buffer) { | |||
| 128 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 128, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 129 | assert(buffer->lease(), "invariant")do { if (!(buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 129, "assert(" "buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 130 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 130, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 131 | } | |||
| 132 | ||||
| 133 | static void assert_retired(const BufferPtr buffer, Thread* thread) { | |||
| 134 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 134, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 135 | assert(buffer->acquired_by(thread), "invariant")do { if (!(buffer->acquired_by(thread))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 135, "assert(" "buffer->acquired_by(thread)" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 136 | assert(buffer->retired(), "invariant")do { if (!(buffer->retired())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 136, "assert(" "buffer->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 137 | } | |||
| 138 | #endif // ASSERT | |||
| 139 | ||||
| 140 | void JfrCheckpointManager::register_full(BufferPtr buffer, Thread* thread) { | |||
| 141 | DEBUG_ONLY(assert_retired(buffer, thread);)assert_retired(buffer, thread); | |||
| 142 | // nothing here at the moment | |||
| 143 | } | |||
| 144 | ||||
| 145 | BufferPtr JfrCheckpointManager::lease(Thread* thread, bool previous_epoch /* false */, size_t size /* 0 */) { | |||
| 146 | JfrCheckpointMspace* const mspace = instance()._global_mspace; | |||
| 147 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 147, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 148 | static const size_t max_elem_size = mspace->min_element_size(); // min is max | |||
| 149 | BufferPtr buffer; | |||
| 150 | if (size <= max_elem_size) { | |||
| 151 | buffer = mspace_acquire_live(size, mspace, thread, previous_epoch); | |||
| 152 | if (buffer != NULL__null) { | |||
| 153 | buffer->set_lease(); | |||
| 154 | DEBUG_ONLY(assert_lease(buffer);)assert_lease(buffer); | |||
| 155 | return buffer; | |||
| 156 | } | |||
| 157 | } | |||
| 158 | buffer = mspace_allocate_transient_lease_to_live_list(size, mspace, thread, previous_epoch); | |||
| 159 | DEBUG_ONLY(assert_lease(buffer);)assert_lease(buffer); | |||
| 160 | return buffer; | |||
| 161 | } | |||
| 162 | ||||
| 163 | const u1 thread_local_context = 1; | |||
| 164 | ||||
| 165 | static bool is_thread_local(JfrBuffer* buffer) { | |||
| 166 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 166, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 167 | return buffer->context() == thread_local_context; | |||
| 168 | } | |||
| 169 | ||||
| 170 | static void retire(JfrBuffer* buffer) { | |||
| 171 | DEBUG_ONLY(assert_release(buffer);)assert_release(buffer); | |||
| 172 | buffer->clear_lease(); | |||
| 173 | buffer->set_retired(); | |||
| 174 | } | |||
| 175 | ||||
| 176 | /* | |||
| 177 | * The buffer is effectively invalidated for the thread post-return, | |||
| 178 | * and the caller should take means to ensure that it is not referenced. | |||
| 179 | */ | |||
| 180 | static void release(JfrBuffer* buffer) { | |||
| 181 | DEBUG_ONLY(assert_release(buffer);)assert_release(buffer); | |||
| 182 | if (is_thread_local(buffer)) { | |||
| 183 | retire(buffer); | |||
| 184 | } else { | |||
| 185 | buffer->clear_lease(); | |||
| 186 | buffer->release(); | |||
| 187 | } | |||
| 188 | } | |||
| 189 | BufferPtr JfrCheckpointManager::acquire_thread_local(size_t size, Thread* thread) { | |||
| 190 | assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 190, "assert(" "thread != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 191 | JfrBuffer* const buffer = instance()._thread_local_mspace->acquire(size, thread); | |||
| 192 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 192, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 193 | assert(buffer->free_size() >= size, "invariant")do { if (!(buffer->free_size() >= size)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 193, "assert(" "buffer->free_size() >= size" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 194 | buffer->set_context(thread_local_context); | |||
| 195 | assert(is_thread_local(buffer), "invariant")do { if (!(is_thread_local(buffer))) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 195, "assert(" "is_thread_local(buffer)" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 196 | buffer->set_lease(); | |||
| 197 | return buffer; | |||
| 198 | } | |||
| 199 | ||||
| 200 | BufferPtr JfrCheckpointManager::lease_thread_local(Thread* thread, size_t size /* 0 */) { | |||
| 201 | JfrBuffer* const buffer = acquire_thread_local(size, thread); | |||
| 202 | DEBUG_ONLY(assert_lease(buffer);)assert_lease(buffer); | |||
| 203 | return buffer; | |||
| 204 | } | |||
| 205 | ||||
| 206 | BufferPtr JfrCheckpointManager::lease(BufferPtr old, Thread* thread, size_t size) { | |||
| 207 | assert(old != NULL, "invariant")do { if (!(old != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 207, "assert(" "old != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 208 | return is_thread_local(old) ? acquire_thread_local(size, thread) : | |||
| 209 | lease(thread, instance()._global_mspace->in_previous_epoch_list(old), size); | |||
| 210 | } | |||
| 211 | ||||
| 212 | BufferPtr JfrCheckpointManager::flush(BufferPtr old, size_t used, size_t requested, Thread* thread) { | |||
| 213 | assert(old != NULL, "invariant")do { if (!(old != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 213, "assert(" "old != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 214 | assert(old->lease(), "invariant")do { if (!(old->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 214, "assert(" "old->lease()" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 215 | if (0 == requested) { | |||
| 216 | // indicates a lease is being returned | |||
| 217 | release(old); | |||
| 218 | // signal completion of a new checkpoint | |||
| 219 | _new_checkpoint.signal(); | |||
| 220 | return NULL__null; | |||
| 221 | } | |||
| 222 | BufferPtr new_buffer = lease(old, thread, used + requested); | |||
| 223 | assert(new_buffer != NULL, "invariant")do { if (!(new_buffer != __null)) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 223, "assert(" "new_buffer != __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 224 | migrate_outstanding_writes(old, new_buffer, used, requested); | |||
| 225 | retire(old); | |||
| 226 | return new_buffer; | |||
| 227 | } | |||
| 228 | ||||
| 229 | // offsets into the JfrCheckpointEntry | |||
| 230 | static const juint starttime_offset = sizeof(jlong); | |||
| 231 | static const juint duration_offset = starttime_offset + sizeof(jlong); | |||
| 232 | static const juint checkpoint_type_offset = duration_offset + sizeof(jlong); | |||
| 233 | static const juint types_offset = checkpoint_type_offset + sizeof(juint); | |||
| 234 | static const juint payload_offset = types_offset + sizeof(juint); | |||
| 235 | ||||
| 236 | template <typename Return> | |||
| 237 | static Return read_data(const u1* data) { | |||
| 238 | return JfrBigEndian::read<Return>(data); | |||
| 239 | } | |||
| 240 | ||||
| 241 | static jlong total_size(const u1* data) { | |||
| 242 | return read_data<jlong>(data); | |||
| 243 | } | |||
| 244 | ||||
| 245 | static jlong starttime(const u1* data) { | |||
| 246 | return read_data<jlong>(data + starttime_offset); | |||
| 247 | } | |||
| 248 | ||||
| 249 | static jlong duration(const u1* data) { | |||
| 250 | return read_data<jlong>(data + duration_offset); | |||
| 251 | } | |||
| 252 | ||||
| 253 | static u1 checkpoint_type(const u1* data) { | |||
| 254 | return read_data<u1>(data + checkpoint_type_offset); | |||
| 255 | } | |||
| 256 | ||||
| 257 | static juint number_of_types(const u1* data) { | |||
| 258 | return read_data<juint>(data + types_offset); | |||
| 259 | } | |||
| 260 | ||||
| 261 | static void write_checkpoint_header(JfrChunkWriter& cw, int64_t delta_to_last_checkpoint, const u1* data) { | |||
| 262 | cw.reserve(sizeof(u4)); | |||
| 263 | cw.write<u8>(EVENT_CHECKPOINT); | |||
| 264 | cw.write(starttime(data)); | |||
| 265 | cw.write(duration(data)); | |||
| 266 | cw.write(delta_to_last_checkpoint); | |||
| 267 | cw.write(checkpoint_type(data)); | |||
| 268 | cw.write(number_of_types(data)); | |||
| 269 | } | |||
| 270 | ||||
| 271 | static void write_checkpoint_content(JfrChunkWriter& cw, const u1* data, size_t size) { | |||
| 272 | assert(data != NULL, "invariant")do { if (!(data != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 272, "assert(" "data != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 273 | cw.write_unbuffered(data + payload_offset, size - sizeof(JfrCheckpointEntry)); | |||
| 274 | } | |||
| 275 | ||||
| 276 | static size_t write_checkpoint_event(JfrChunkWriter& cw, const u1* data) { | |||
| 277 | assert(data != NULL, "invariant")do { if (!(data != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 277, "assert(" "data != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 278 | const int64_t event_begin = cw.current_offset(); | |||
| 279 | const int64_t last_checkpoint_event = cw.last_checkpoint_offset(); | |||
| 280 | const int64_t delta_to_last_checkpoint = last_checkpoint_event == 0 ? 0 : last_checkpoint_event - event_begin; | |||
| 281 | const int64_t checkpoint_size = total_size(data); | |||
| 282 | write_checkpoint_header(cw, delta_to_last_checkpoint, data); | |||
| 283 | write_checkpoint_content(cw, data, checkpoint_size); | |||
| 284 | const int64_t event_size = cw.current_offset() - event_begin; | |||
| 285 | cw.write_padded_at_offset<u4>(event_size, event_begin); | |||
| 286 | cw.set_last_checkpoint_offset(event_begin); | |||
| 287 | return (size_t)checkpoint_size; | |||
| 288 | } | |||
| 289 | ||||
| 290 | static size_t write_checkpoints(JfrChunkWriter& cw, const u1* data, size_t size) { | |||
| 291 | assert(cw.is_valid(), "invariant")do { if (!(cw.is_valid())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 291, "assert(" "cw.is_valid()" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 292 | assert(data != NULL, "invariant")do { if (!(data != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 292, "assert(" "data != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 293 | assert(size > 0, "invariant")do { if (!(size > 0)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 293, "assert(" "size > 0" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 294 | const u1* const limit = data + size; | |||
| 295 | const u1* next = data; | |||
| 296 | size_t processed = 0; | |||
| 297 | while (next < limit) { | |||
| 298 | const size_t checkpoint_size = write_checkpoint_event(cw, next); | |||
| 299 | processed += checkpoint_size; | |||
| 300 | next += checkpoint_size; | |||
| 301 | } | |||
| 302 | assert(next == limit, "invariant")do { if (!(next == limit)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 302, "assert(" "next == limit" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 303 | return processed; | |||
| 304 | } | |||
| 305 | ||||
| 306 | template <typename T> | |||
| 307 | class CheckpointWriteOp { | |||
| 308 | private: | |||
| 309 | JfrChunkWriter& _writer; | |||
| 310 | size_t _processed; | |||
| 311 | public: | |||
| 312 | typedef T Type; | |||
| 313 | CheckpointWriteOp(JfrChunkWriter& writer) : _writer(writer), _processed(0) {} | |||
| 314 | bool write(Type* t, const u1* data, size_t size) { | |||
| 315 | _processed += write_checkpoints(_writer, data, size); | |||
| 316 | return true; | |||
| 317 | } | |||
| 318 | size_t processed() const { return _processed; } | |||
| 319 | }; | |||
| 320 | ||||
| 321 | typedef CheckpointWriteOp<JfrCheckpointManager::Buffer> WriteOperation; | |||
| 322 | typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation; | |||
| 323 | typedef ReleaseWithExcisionOp<JfrCheckpointMspace, JfrCheckpointMspace::LiveList> ReleaseOperation; | |||
| 324 | typedef CompositeOperation<MutexedWriteOperation, ReleaseOperation> WriteReleaseOperation; | |||
| 325 | ||||
| 326 | void JfrCheckpointManager::begin_epoch_shift() { | |||
| 327 | assert(SafepointSynchronize::is_at_safepoint(), "invariant")do { if (!(SafepointSynchronize::is_at_safepoint())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 327, "assert(" "SafepointSynchronize::is_at_safepoint()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 328 | JfrTraceIdEpoch::begin_epoch_shift(); | |||
| 329 | } | |||
| 330 | ||||
| 331 | void JfrCheckpointManager::end_epoch_shift() { | |||
| 332 | assert(SafepointSynchronize::is_at_safepoint(), "invariant")do { if (!(SafepointSynchronize::is_at_safepoint())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 332, "assert(" "SafepointSynchronize::is_at_safepoint()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 333 | debug_only(const u1 current_epoch = JfrTraceIdEpoch::current();)const u1 current_epoch = JfrTraceIdEpoch::current(); | |||
| 334 | JfrTraceIdEpoch::end_epoch_shift(); | |||
| 335 | assert(current_epoch != JfrTraceIdEpoch::current(), "invariant")do { if (!(current_epoch != JfrTraceIdEpoch::current())) { (* g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 335, "assert(" "current_epoch != JfrTraceIdEpoch::current()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 336 | } | |||
| 337 | ||||
| 338 | size_t JfrCheckpointManager::write() { | |||
| 339 | DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(JavaThread::current()))JfrJavaSupport::check_java_thread_in_native(JavaThread::current ()); | |||
| 340 | WriteOperation wo(_chunkwriter); | |||
| 341 | MutexedWriteOperation mwo(wo); | |||
| 342 | _thread_local_mspace->iterate(mwo, true); // previous epoch list | |||
| 343 | assert(_global_mspace->free_list_is_empty(), "invariant")do { if (!(_global_mspace->free_list_is_empty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 343, "assert(" "_global_mspace->free_list_is_empty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 344 | ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true)); | |||
| 345 | WriteReleaseOperation wro(&mwo, &ro); | |||
| 346 | process_live_list(wro, _global_mspace, true); // previous epoch list | |||
| 347 | return wo.processed(); | |||
| 348 | } | |||
| 349 | ||||
| 350 | typedef DiscardOp<DefaultDiscarder<JfrCheckpointManager::Buffer> > DiscardOperation; | |||
| 351 | typedef CompositeOperation<DiscardOperation, ReleaseOperation> DiscardReleaseOperation; | |||
| 352 | ||||
| 353 | size_t JfrCheckpointManager::clear() { | |||
| 354 | JfrTraceIdLoadBarrier::clear(); | |||
| 355 | clear_type_set(); | |||
| 356 | DiscardOperation discard_operation(mutexed); // mutexed discard mode | |||
| 357 | _thread_local_mspace->iterate(discard_operation, true); // previous epoch list | |||
| 358 | ReleaseOperation ro(_global_mspace, _global_mspace->live_list(true)); | |||
| 359 | DiscardReleaseOperation discard_op(&discard_operation, &ro); | |||
| 360 | assert(_global_mspace->free_list_is_empty(), "invariant")do { if (!(_global_mspace->free_list_is_empty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 360, "assert(" "_global_mspace->free_list_is_empty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 361 | process_live_list(discard_op, _global_mspace, true); // previous epoch list | |||
| 362 | return discard_operation.elements(); | |||
| 363 | } | |||
| 364 | ||||
| 365 | size_t JfrCheckpointManager::write_static_type_set(Thread* thread) { | |||
| 366 | assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 366, "assert(" "thread != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 367 | JfrCheckpointWriter writer(true, thread, STATICS); | |||
| 368 | JfrTypeManager::write_static_types(writer); | |||
| 369 | return writer.used_size(); | |||
| 370 | } | |||
| 371 | ||||
| 372 | size_t JfrCheckpointManager::write_threads(JavaThread* thread) { | |||
| 373 | assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 373, "assert(" "thread != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 374 | // can safepoint here | |||
| 375 | ThreadInVMfromNative transition(thread); | |||
| 376 | ResourceMark rm(thread); | |||
| 377 | HandleMark hm(thread); | |||
| 378 | JfrCheckpointWriter writer(true, thread, THREADS); | |||
| 379 | JfrTypeManager::write_threads(writer); | |||
| 380 | return writer.used_size(); | |||
| 381 | } | |||
| 382 | ||||
| 383 | size_t JfrCheckpointManager::write_static_type_set_and_threads() { | |||
| 384 | JavaThread* const thread = JavaThread::current(); | |||
| 385 | DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread))JfrJavaSupport::check_java_thread_in_native(thread); | |||
| 386 | write_static_type_set(thread); | |||
| 387 | write_threads(thread); | |||
| 388 | return write(); | |||
| 389 | } | |||
| 390 | ||||
| 391 | void JfrCheckpointManager::on_rotation() { | |||
| 392 | assert(SafepointSynchronize::is_at_safepoint(), "invariant")do { if (!(SafepointSynchronize::is_at_safepoint())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 392, "assert(" "SafepointSynchronize::is_at_safepoint()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 393 | JfrTypeManager::on_rotation(); | |||
| 394 | notify_threads(); | |||
| 395 | } | |||
| 396 | ||||
| 397 | void JfrCheckpointManager::clear_type_set() { | |||
| 398 | assert(!JfrRecorder::is_recording(), "invariant")do { if (!(!JfrRecorder::is_recording())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 398, "assert(" "!JfrRecorder::is_recording()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 399 | JavaThread* t = JavaThread::current(); | |||
| 400 | DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(t))JfrJavaSupport::check_java_thread_in_native(t); | |||
| 401 | // can safepoint here | |||
| 402 | ThreadInVMfromNative transition(t); | |||
| 403 | MutexLocker cld_lock(ClassLoaderDataGraph_lock); | |||
| 404 | MutexLocker module_lock(Module_lock); | |||
| 405 | JfrTypeSet::clear(); | |||
| 406 | } | |||
| 407 | ||||
| 408 | void JfrCheckpointManager::write_type_set() { | |||
| 409 | { | |||
| 410 | JavaThread* const thread = JavaThread::current(); | |||
| 411 | DEBUG_ONLY(JfrJavaSupport::check_java_thread_in_native(thread))JfrJavaSupport::check_java_thread_in_native(thread); | |||
| 412 | // can safepoint here | |||
| 413 | ThreadInVMfromNative transition(thread); | |||
| 414 | MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock); | |||
| 415 | MutexLocker module_lock(thread, Module_lock); | |||
| 416 | if (LeakProfiler::is_running()) { | |||
| 417 | JfrCheckpointWriter leakp_writer(true, thread); | |||
| 418 | JfrCheckpointWriter writer(true, thread); | |||
| 419 | JfrTypeSet::serialize(&writer, &leakp_writer, false, false); | |||
| 420 | ObjectSampleCheckpoint::on_type_set(leakp_writer); | |||
| 421 | } else { | |||
| 422 | JfrCheckpointWriter writer(true, thread); | |||
| 423 | JfrTypeSet::serialize(&writer, NULL__null, false, false); | |||
| 424 | } | |||
| 425 | } | |||
| 426 | write(); | |||
| 427 | } | |||
| 428 | ||||
| 429 | void JfrCheckpointManager::on_unloading_classes() { | |||
| 430 | assert_locked_or_safepoint(ClassLoaderDataGraph_lock); | |||
| 431 | JfrCheckpointWriter writer(Thread::current()); | |||
| 432 | JfrTypeSet::on_unloading_classes(&writer); | |||
| 433 | if (LeakProfiler::is_running()) { | |||
| 434 | ObjectSampleCheckpoint::on_type_set_unload(writer); | |||
| 435 | } | |||
| 436 | } | |||
| 437 | ||||
| 438 | static size_t flush_type_set(Thread* thread) { | |||
| 439 | assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 439, "assert(" "thread != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 440 | JfrCheckpointWriter writer(thread); | |||
| 441 | MutexLocker cld_lock(thread, ClassLoaderDataGraph_lock); | |||
| 442 | MutexLocker module_lock(thread, Module_lock); | |||
| 443 | return JfrTypeSet::serialize(&writer, NULL__null, false, true); | |||
| 444 | } | |||
| 445 | ||||
| 446 | size_t JfrCheckpointManager::flush_type_set() { | |||
| 447 | size_t elements = 0; | |||
| 448 | if (JfrTraceIdEpoch::has_changed_tag_state()) { | |||
| 449 | Thread* const thread = Thread::current(); | |||
| 450 | if (thread->is_Java_thread()) { | |||
| 451 | // can safepoint here | |||
| 452 | ThreadInVMfromNative transition(JavaThread::cast(thread)); | |||
| 453 | elements = ::flush_type_set(thread); | |||
| 454 | } else { | |||
| 455 | elements = ::flush_type_set(thread); | |||
| 456 | } | |||
| 457 | } | |||
| 458 | if (_new_checkpoint.is_signaled_with_reset()) { | |||
| 459 | WriteOperation wo(_chunkwriter); | |||
| 460 | MutexedWriteOperation mwo(wo); | |||
| 461 | _thread_local_mspace->iterate(mwo); // current epoch list | |||
| 462 | assert(_global_mspace->live_list_is_nonempty(), "invariant")do { if (!(_global_mspace->live_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 462, "assert(" "_global_mspace->live_list_is_nonempty()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 463 | process_live_list(mwo, _global_mspace); // current epoch list | |||
| 464 | } | |||
| 465 | return elements; | |||
| 466 | } | |||
| 467 | ||||
| 468 | void JfrCheckpointManager::create_thread_blob(Thread* thread) { | |||
| 469 | JfrTypeManager::create_thread_blob(thread); | |||
| 470 | } | |||
| 471 | ||||
| 472 | void JfrCheckpointManager::write_thread_checkpoint(Thread* thread) { | |||
| 473 | JfrTypeManager::write_thread_checkpoint(thread); | |||
| 474 | } | |||
| 475 | ||||
| 476 | class JfrNotifyClosure : public ThreadClosure { | |||
| 477 | public: | |||
| 478 | void do_thread(Thread* thread) { | |||
| 479 | assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 479, "assert(" "thread != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 480 | assert_locked_or_safepoint(Threads_lock); | |||
| 481 | JfrJavaEventWriter::notify(JavaThread::cast(thread)); | |||
| 482 | } | |||
| 483 | }; | |||
| 484 | ||||
| 485 | void JfrCheckpointManager::notify_threads() { | |||
| 486 | assert(SafepointSynchronize::is_at_safepoint(), "invariant")do { if (!(SafepointSynchronize::is_at_safepoint())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/jfrCheckpointManager.cpp" , 486, "assert(" "SafepointSynchronize::is_at_safepoint()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 487 | JfrNotifyClosure tc; | |||
| 488 | JfrJavaThreadIterator iter; | |||
| 489 | while (iter.has_next()) { | |||
| 490 | tc.do_thread(iter.next()); | |||
| 491 | } | |||
| 492 | } |
| 1 | /* |
| 2 | * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_JFR_RECORDER_STORAGE_JFREPOCHSTORAGE_INLINE_HPP |
| 26 | #define SHARE_JFR_RECORDER_STORAGE_JFREPOCHSTORAGE_INLINE_HPP |
| 27 | |
| 28 | #include "jfr/recorder/storage/jfrEpochStorage.hpp" |
| 29 | |
| 30 | #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" |
| 31 | #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" |
| 32 | #include "jfr/utilities/jfrConcurrentQueue.inline.hpp" |
| 33 | #include "jfr/utilities/jfrLinkedList.inline.hpp" |
| 34 | #include "logging/log.hpp" |
| 35 | |
| 36 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 37 | JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::JfrEpochStorageHost() : _mspace(NULL__null) {} |
| 38 | |
| 39 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 40 | JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::~JfrEpochStorageHost() { |
| 41 | delete _mspace; |
| 42 | } |
| 43 | |
| 44 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 45 | bool JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::initialize(size_t min_elem_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count) { |
| 46 | assert(_mspace == NULL, "invariant")do { if (!(_mspace == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp" , 46, "assert(" "_mspace == __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 47 | _mspace = new EpochMspace(min_elem_size, free_list_cache_count_limit, this); |
| 48 | return _mspace != NULL__null && _mspace->initialize(cache_prealloc_count); |
| 49 | } |
| 50 | |
| 51 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 52 | inline NodeType* JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::acquire(size_t size, Thread* thread) { |
| 53 | BufferPtr buffer = mspace_acquire_to_live_list(size, _mspace, thread); |
| 54 | if (buffer == NULL__null) { |
| 55 | log_warning(jfr)(!(LogImpl<(LogTag::_jfr), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Warning))) ? (void)0 : LogImpl<(LogTag ::_jfr), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>::write<LogLevel ::Warning>("Unable to allocate " SIZE_FORMAT"%" "l" "u" " bytes of %s.", _mspace->min_element_size(), "epoch storage"); |
| 56 | return NULL__null; |
| 57 | } |
| 58 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp" , 58, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 59 | return buffer; |
| 60 | } |
| 61 | |
| 62 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 63 | void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::release(NodeType* buffer) { |
| 64 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp" , 64, "assert(" "buffer != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 65 | buffer->set_retired(); |
| 66 | } |
| 67 | |
| 68 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 69 | void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::register_full(NodeType* buffer, Thread* thread) { |
| 70 | // nothing here at the moment |
| 71 | } |
| 72 | |
| 73 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 74 | template <typename Functor> |
| 75 | void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::iterate(Functor& functor, bool previous_epoch) { |
| 76 | typedef ReinitializeAllReleaseRetiredOp<EpochMspace, typename EpochMspace::LiveList> PreviousEpochReleaseOperation; |
| 77 | typedef CompositeOperation<Functor, PreviousEpochReleaseOperation> PreviousEpochOperation; |
| 78 | typedef ReleaseRetiredOp<EpochMspace, typename EpochMspace::LiveList> CurrentEpochReleaseOperation; |
| 79 | typedef CompositeOperation<Functor, CurrentEpochReleaseOperation> CurrentEpochOperation; |
| 80 | if (previous_epoch) { |
| 81 | PreviousEpochReleaseOperation pero(_mspace, _mspace->live_list(true)); |
| 82 | PreviousEpochOperation peo(&functor, &pero); |
| 83 | process_live_list(peo, _mspace, true); // previous epoch list |
| 84 | return; |
| 85 | } |
| 86 | if (EagerReclaim) { |
| 87 | CurrentEpochReleaseOperation cero(_mspace, _mspace->live_list()); |
| 88 | CurrentEpochOperation ceo(&functor, &cero); |
| 89 | process_live_list(ceo, _mspace, false); // current epoch list |
| 90 | return; |
| 91 | } |
| 92 | process_live_list(functor, _mspace, false); // current epoch list |
| 93 | } |
| 94 | |
| 95 | #ifdef ASSERT1 |
| 96 | |
| 97 | template <typename Mspace> |
| 98 | class EmptyVerifier { |
| 99 | private: |
| 100 | Mspace* _mspace; |
| 101 | public: |
| 102 | typedef typename Mspace::Node Node; |
| 103 | typedef typename Mspace::NodePtr NodePtr; |
| 104 | EmptyVerifier(Mspace* mspace) : _mspace(mspace) {} |
| 105 | bool process(NodePtr node) { |
| 106 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp" , 106, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 107 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp" , 107, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 108 | return true; |
| 109 | } |
| 110 | }; |
| 111 | |
| 112 | template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim> |
| 113 | void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::verify_previous_empty() const { |
| 114 | typedef EmptyVerifier<JfrEpochStorage::Mspace> VerifyEmptyMspace; |
| 115 | VerifyEmptyMspace vem(_mspace); |
| 116 | process_live_list(vem, _mspace, true); |
| 117 | } |
| 118 | |
| 119 | #endif // ASSERT |
| 120 | |
| 121 | #endif // SHARE_JFR_RECORDER_STORAGE_JFREPOCHSTORAGE_INLINE_HPP |
| 1 | /* |
| 2 | * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP |
| 26 | #define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP |
| 27 | |
| 28 | #include "jfr/recorder/storage/jfrMemorySpace.hpp" |
| 29 | |
| 30 | #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" |
| 31 | #include "runtime/atomic.hpp" |
| 32 | #include "runtime/os.hpp" |
| 33 | |
| 34 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 35 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>:: |
| 36 | JfrMemorySpace(size_t min_element_size, size_t free_list_cache_count_limit, Client* client) : |
| 37 | _free_list(), |
| 38 | _live_list_epoch_0(), |
| 39 | _live_list_epoch_1(), |
| 40 | _client(client), |
| 41 | _min_element_size(min_element_size), |
| 42 | _free_list_cache_count_limit(free_list_cache_count_limit), |
| 43 | _free_list_cache_count(0) {} |
| 44 | |
| 45 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 46 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::~JfrMemorySpace() { |
| 47 | while (_live_list_epoch_0.is_nonempty()) { |
| 48 | deallocate(_live_list_epoch_0.remove()); |
| 49 | } |
| 50 | while (_live_list_epoch_1.is_nonempty()) { |
| 51 | deallocate(_live_list_epoch_1.remove()); |
| 52 | } |
| 53 | while (_free_list.is_nonempty()) { |
| 54 | deallocate(_free_list.remove()); |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 59 | bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::initialize(size_t cache_prealloc_count, bool prealloc_to_free_list) { |
| 60 | if (!(_free_list.initialize() && _live_list_epoch_0.initialize() && _live_list_epoch_1.initialize())) { |
| 61 | return false; |
| 62 | } |
| 63 | // pre-allocate elements to be cached in the requested list |
| 64 | for (size_t i = 0; i < cache_prealloc_count; ++i) { |
| 65 | NodePtr const node = allocate(_min_element_size); |
| 66 | if (node == NULL__null) { |
| 67 | return false; |
| 68 | } |
| 69 | if (prealloc_to_free_list) { |
| 70 | add_to_free_list(node); |
| 71 | } else { |
| 72 | add_to_live_list(node); |
| 73 | } |
| 74 | } |
| 75 | return true; |
| 76 | } |
| 77 | |
| 78 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 79 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::should_populate_free_list_cache() const { |
| 80 | return !is_free_list_cache_limited() || _free_list_cache_count < _free_list_cache_count_limit; |
| 81 | } |
| 82 | |
| 83 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 84 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::is_free_list_cache_limited() const { |
| 85 | return _free_list_cache_count_limit != JFR_MSPACE_UNLIMITED_CACHE_SIZE; |
| 86 | } |
| 87 | |
| 88 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 89 | inline size_t JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::min_element_size() const { |
| 90 | return _min_element_size; |
| 91 | } |
| 92 | |
| 93 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 94 | inline FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() { |
| 95 | return _free_list; |
| 96 | } |
| 97 | |
| 98 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 99 | inline const FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() const { |
| 100 | return _free_list; |
| 101 | } |
| 102 | |
| 103 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 104 | inline FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) { |
| 105 | if (epoch_aware) { |
| 106 | return previous_epoch ? previous_epoch_list() : current_epoch_list(); |
| 107 | } |
| 108 | return _live_list_epoch_0; |
| 109 | } |
| 110 | |
| 111 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 112 | inline const FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) const { |
| 113 | if (epoch_aware) { |
| 114 | return previous_epoch ? previous_epoch_list() : current_epoch_list(); |
| 115 | } |
| 116 | return _live_list_epoch_0; |
| 117 | } |
| 118 | |
| 119 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 120 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_empty() const { |
| 121 | return _free_list.is_empty(); |
| 122 | } |
| 123 | |
| 124 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 125 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_nonempty() const { |
| 126 | return !free_list_is_empty(); |
| 127 | } |
| 128 | |
| 129 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 130 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_empty(bool previous_epoch) const { |
| 131 | return live_list().is_empty(); |
| 132 | } |
| 133 | |
| 134 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 135 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_nonempty(bool previous_epoch) const { |
| 136 | return live_list().is_nonempty(); |
| 137 | } |
| 138 | |
| 139 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 140 | bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_free_list(const typename FreeListType::Node* node) const { |
| 141 | return _free_list.in_list(node); |
| 142 | } |
| 143 | |
| 144 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 145 | inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 146 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) const { |
| 147 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 147, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 148 | return epoch == 0 ? _live_list_epoch_0 : _live_list_epoch_1; |
| 149 | } |
| 150 | |
| 151 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 152 | inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 153 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) { |
| 154 | return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>( |
| 155 | const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->epoch_list_selector(epoch)); |
| 156 | } |
| 157 | |
| 158 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 159 | inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 160 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() const { |
| 161 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 161, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 162 | return epoch_list_selector(JfrTraceIdEpoch::current()); |
| 163 | } |
| 164 | |
| 165 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 166 | inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 167 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() { |
| 168 | return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>( |
| 169 | const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->current_epoch_list()); |
| 170 | } |
| 171 | |
| 172 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 173 | inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 174 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() const { |
| 175 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 175, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 176 | return epoch_list_selector(JfrTraceIdEpoch::previous()); |
| 177 | } |
| 178 | |
| 179 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 180 | inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 181 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() { |
| 182 | return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>( |
| 183 | const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->previous_epoch_list()); |
| 184 | } |
| 185 | |
| 186 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 187 | bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_live_list(const typename FreeListType::Node* node, bool previous_epoch) const { |
| 188 | return live_list(previous_epoch).in_list(node); |
| 189 | } |
| 190 | |
| 191 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 192 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_current_epoch_list(const typename FreeListType::Node* node) const { |
| 193 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 193, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 194 | return current_epoch_list().in_list(node); |
| 195 | } |
| 196 | |
| 197 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 198 | inline bool JfrMemorySpace< Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_previous_epoch_list(const typename FreeListType::Node* node) const { |
| 199 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 199, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 200 | return previous_epoch_list().in_list(node); |
| 201 | } |
| 202 | |
| 203 | // allocations are even multiples of the mspace min size |
| 204 | static inline size_t align_allocation_size(size_t requested_size, size_t min_element_size) { |
| 205 | u8 alloc_size_bytes = min_element_size; |
| 206 | while (requested_size > alloc_size_bytes) { |
| 207 | alloc_size_bytes <<= 1; |
| 208 | } |
| 209 | return (size_t)alloc_size_bytes; |
| 210 | } |
| 211 | |
| 212 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 213 | inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::allocate(size_t size) { |
| 214 | const size_t aligned_size_bytes = align_allocation_size(size, _min_element_size); |
| 215 | void* const allocation = JfrCHeapObj::new_array<u1>(aligned_size_bytes + sizeof(Node)); |
| 216 | if (allocation == NULL__null) { |
| 217 | return NULL__null; |
| 218 | } |
| 219 | NodePtr node = new (allocation) Node(); |
| 220 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 220, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 221 | if (!node->initialize(sizeof(Node), aligned_size_bytes)) { |
| 222 | JfrCHeapObj::free(node, aligned_size_bytes + sizeof(Node)); |
| 223 | return NULL__null; |
| 224 | } |
| 225 | return node; |
| 226 | } |
| 227 | |
| 228 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 229 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::deallocate(typename FreeListType::NodePtr node) { |
| 230 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 230, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 231 | assert(!in_free_list(node), "invariant")do { if (!(!in_free_list(node))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 231, "assert(" "!in_free_list(node)" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 232 | assert(!_live_list_epoch_0.in_list(node), "invariant")do { if (!(!_live_list_epoch_0.in_list(node))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 232, "assert(" "!_live_list_epoch_0.in_list(node)" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 233 | assert(!_live_list_epoch_1.in_list(node), "invariant")do { if (!(!_live_list_epoch_1.in_list(node))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 233, "assert(" "!_live_list_epoch_1.in_list(node)" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 234 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 234, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 235 | JfrCHeapObj::free(node, node->total_size()); |
| 236 | } |
| 237 | |
| 238 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 239 | inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::acquire(size_t size, bool free_list, Thread* thread, bool previous_epoch) { |
| 240 | return RetrievalPolicy<JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware> >::acquire(this, free_list, thread, size, previous_epoch); |
| 241 | } |
| 242 | |
| 243 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 244 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::release(typename FreeListType::NodePtr node) { |
| 245 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 245, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 246 | if (node->transient()) { |
| 247 | deallocate(node); |
| 248 | return; |
| 249 | } |
| 250 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 250, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 251 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 251, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 252 | assert(node->identity() == NULL, "invariant")do { if (!(node->identity() == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 252, "assert(" "node->identity() == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 253 | if (should_populate_free_list_cache()) { |
| 254 | add_to_free_list(node); |
| 255 | } else { |
| 256 | deallocate(node); |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 261 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_free_list(typename FreeListType::NodePtr node) { |
| 262 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 262, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 263 | _free_list.add(node); |
| 264 | if (is_free_list_cache_limited()) { |
| 265 | Atomic::inc(&_free_list_cache_count); |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 270 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_live_list(typename FreeListType::NodePtr node, bool previous_epoch) { |
| 271 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 271, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 272 | live_list(previous_epoch).add(node); |
| 273 | } |
| 274 | |
| 275 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 276 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::decrement_free_list_count() { |
| 277 | if (is_free_list_cache_limited()) { |
| 278 | Atomic::dec(&_free_list_cache_count); |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 283 | template <typename Callback> |
| 284 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_free_list(Callback& callback) { |
| 285 | return _free_list.iterate(callback); |
| 286 | } |
| 287 | |
| 288 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 289 | template <typename Callback> |
| 290 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_live_list(Callback& callback, bool previous_epoch) { |
| 291 | if (epoch_aware) { |
| 292 | live_list(previous_epoch).iterate(callback); |
| 293 | return; |
| 294 | } |
| 295 | _live_list_epoch_0.iterate(callback); |
| 296 | } |
| 297 | |
| 298 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 299 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::register_full(typename FreeListType::NodePtr node, Thread* thread) { |
| 300 | _client->register_full(node, thread); |
| 301 | } |
| 302 | |
| 303 | template <typename Mspace, typename Client> |
| 304 | static inline Mspace* create_mspace(size_t min_element_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count, bool prealloc_to_free_list, Client* cb) { |
| 305 | Mspace* const mspace = new Mspace(min_element_size, free_list_cache_count_limit, cb); |
| 306 | if (mspace != NULL__null) { |
| 307 | mspace->initialize(cache_prealloc_count, prealloc_to_free_list); |
| 308 | } |
| 309 | return mspace; |
| 310 | } |
| 311 | |
| 312 | template <typename Mspace> |
| 313 | inline typename Mspace::NodePtr mspace_allocate(size_t size, Mspace* mspace) { |
| 314 | return mspace->allocate(size); |
| 315 | } |
| 316 | |
| 317 | template <typename Mspace> |
| 318 | inline typename Mspace::NodePtr mspace_allocate_acquired(size_t size, Mspace* mspace, Thread* thread) { |
| 319 | typename Mspace::NodePtr node = mspace_allocate(size, mspace); |
| 320 | if (node == NULL__null) return NULL__null; |
| 321 | node->set_identity(thread); |
| 322 | return node; |
| 323 | } |
| 324 | |
| 325 | template <typename Mspace> |
| 326 | inline typename Mspace::NodePtr mspace_allocate_transient(size_t size, Mspace* mspace, Thread* thread) { |
| 327 | typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread); |
| 328 | if (node == NULL__null) return NULL__null; |
| 329 | assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 329, "assert(" "node->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 330 | node->set_transient(); |
| 331 | return node; |
| 332 | } |
| 333 | |
| 334 | template <typename Mspace> |
| 335 | inline typename Mspace::NodePtr mspace_allocate_transient_lease(size_t size, Mspace* mspace, Thread* thread) { |
| 336 | typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread); |
| 337 | if (node == NULL__null) return NULL__null; |
| 338 | assert(node->transient(), "invariant")do { if (!(node->transient())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 338, "assert(" "node->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 339 | node->set_lease(); |
| 340 | return node; |
| 341 | } |
| 342 | |
| 343 | template <typename Mspace> |
| 344 | inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_free(size_t size, Mspace* mspace, Thread* thread) { |
| 345 | typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread); |
| 346 | if (node == NULL__null) return NULL__null; |
| 347 | assert(node->lease(), "invariant")do { if (!(node->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 347, "assert(" "node->lease()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 348 | mspace->add_to_free_list(node); |
| 349 | return node; |
| 350 | } |
| 351 | |
| 352 | template <typename Mspace> |
| 353 | inline typename Mspace::NodePtr mspace_acquire_free(size_t size, Mspace* mspace, Thread* thread) { |
| 354 | return mspace->acquire(size, true, thread); |
| 355 | } |
| 356 | |
| 357 | template <typename Mspace> |
| 358 | inline typename Mspace::NodePtr mspace_acquire_free_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) { |
| 359 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 359, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 360 | for (size_t i = 0; i < retry_count; ++i) { |
| 361 | typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread); |
| 362 | if (node != NULL__null) { |
| 363 | return node; |
| 364 | } |
| 365 | } |
| 366 | return NULL__null; |
| 367 | } |
| 368 | |
| 369 | template <typename Mspace> |
| 370 | inline typename Mspace::NodePtr mspace_allocate_to_live_list(size_t size, Mspace* mspace, Thread* thread) { |
| 371 | typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread); |
| 372 | if (node == NULL__null) return NULL__null; |
| 373 | assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 373, "assert(" "node->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 374 | mspace->add_to_live_list(node); |
| 375 | return node; |
| 376 | } |
| 377 | |
| 378 | template <typename Mspace> |
| 379 | inline typename Mspace::NodePtr mspace_allocate_transient_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 380 | typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread); |
| 381 | if (node == NULL__null) return NULL__null; |
| 382 | assert(node->transient(), "invariant")do { if (!(node->transient())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 382, "assert(" "node->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 383 | mspace->add_to_live_list(node, previous_epoch); |
| 384 | return node; |
| 385 | } |
| 386 | |
| 387 | template <typename Mspace> |
| 388 | inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 389 | typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread); |
| 390 | if (node == NULL__null) return NULL__null; |
| 391 | assert(node->lease(), "invariant")do { if (!(node->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 391, "assert(" "node->lease()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 392 | mspace->add_to_live_list(node, previous_epoch); |
| 393 | return node; |
| 394 | } |
| 395 | |
| 396 | template <typename Mspace> |
| 397 | inline typename Mspace::NodePtr mspace_acquire_free_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 398 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 398, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 399 | typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread); |
| 400 | if (node == NULL__null) { |
| 401 | return NULL__null; |
| 402 | } |
| 403 | assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 403, "assert(" "node->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 404 | mspace->add_to_live_list(node, previous_epoch); |
| 405 | return node; |
| 406 | } |
| 407 | |
| 408 | template <typename Mspace> |
| 409 | inline typename Mspace::NodePtr mspace_acquire_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 410 | if (size <= mspace->min_element_size()) { |
| 411 | typename Mspace::NodePtr node = mspace_acquire_free_to_live_list(size, mspace, thread, previous_epoch); |
| 412 | if (node != NULL__null) { |
| 413 | return node; |
| 414 | } |
| 415 | } |
| 416 | return mspace_allocate_to_live_list(size, mspace, thread); |
| 417 | } |
| 418 | |
| 419 | template <typename Mspace> |
| 420 | inline typename Mspace::NodePtr mspace_acquire_live(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 421 | return mspace->acquire(size, false, thread, previous_epoch); |
| 422 | } |
| 423 | |
| 424 | template <typename Mspace> |
| 425 | inline typename Mspace::NodePtr mspace_acquire_live_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) { |
| 426 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 426, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 427 | for (size_t i = 0; i < retry_count; ++i) { |
| 428 | typename Mspace::NodePtr const node = mspace_acquire_live(size, mspace, thread, previous_epoch); |
| 429 | if (node != NULL__null) { |
| 430 | return node; |
| 431 | } |
| 432 | } |
| 433 | return NULL__null; |
| 434 | } |
| 435 | |
| 436 | template <typename Mspace> |
| 437 | inline typename Mspace::NodePtr mspace_acquire_lease_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) { |
| 438 | typename Mspace::NodePtr node = mspace_acquire_live_with_retry(size, mspace, retry_count, thread, previous_epoch); |
| 439 | if (node != NULL__null) { |
| 440 | node->set_lease(); |
| 441 | } |
| 442 | return node; |
| 443 | } |
| 444 | |
| 445 | template <typename Mspace> |
| 446 | inline void mspace_release(typename Mspace::NodePtr node, Mspace* mspace) { |
| 447 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 447, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 448 | assert(node->unflushed_size() == 0, "invariant")do { if (!(node->unflushed_size() == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 448, "assert(" "node->unflushed_size() == 0" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 449 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 449, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 450 | mspace->release(node); |
| 451 | } |
| 452 | |
| 453 | template <typename Callback, typename Mspace> |
| 454 | inline void process_live_list(Callback& callback, Mspace* mspace, bool previous_epoch = false) { |
| 455 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 455, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 456 | mspace->iterate_live_list(callback, previous_epoch); |
| 457 | } |
| 458 | |
| 459 | template <typename Callback, typename Mspace> |
| 460 | inline void process_free_list(Callback& callback, Mspace* mspace) { |
| 461 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 461, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 462 | assert(mspace->free_list_is_nonempty(), "invariant")do { if (!(mspace->free_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 462, "assert(" "mspace->free_list_is_nonempty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 463 | mspace->iterate_free_list(callback); |
| 464 | } |
| 465 | |
| 466 | template <typename Mspace> |
| 467 | class ReleaseOp : public StackObj { |
| 468 | private: |
| 469 | Mspace* _mspace; |
| 470 | bool _previous_epoch; |
| 471 | public: |
| 472 | typedef typename Mspace::Node Node; |
| 473 | ReleaseOp(Mspace* mspace) : _mspace(mspace) {} |
| 474 | bool process(typename Mspace::NodePtr node); |
| 475 | size_t processed() const { return 0; } |
| 476 | }; |
| 477 | |
| 478 | template <typename Mspace> |
| 479 | inline bool ReleaseOp<Mspace>::process(typename Mspace::NodePtr node) { |
| 480 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 480, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 481 | // assumes some means of exclusive access to the node |
| 482 | if (node->transient()) { |
| 483 | // make sure the transient node is already detached |
| 484 | _mspace->release(node); |
| 485 | return true; |
| 486 | } |
| 487 | node->reinitialize(); |
| 488 | if (node->identity() != NULL__null) { |
| 489 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 489, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 490 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 490, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 491 | node->release(); // publish |
| 492 | } |
| 493 | return true; |
| 494 | } |
| 495 | |
| 496 | template <typename Mspace, typename List> |
| 497 | class ReleaseWithExcisionOp : public ReleaseOp<Mspace> { |
| 498 | private: |
| 499 | List& _list; |
| 500 | typename List::NodePtr _prev; |
| 501 | size_t _count; |
| 502 | size_t _amount; |
| 503 | public: |
| 504 | ReleaseWithExcisionOp(Mspace* mspace, List& list) : |
| 505 | ReleaseOp<Mspace>(mspace), _list(list), _prev(NULL__null), _count(0), _amount(0) {} |
| 506 | bool process(typename List::NodePtr node); |
| 507 | size_t processed() const { return _count; } |
| 508 | size_t amount() const { return _amount; } |
| 509 | }; |
| 510 | |
| 511 | template <typename Mspace, typename List> |
| 512 | inline bool ReleaseWithExcisionOp<Mspace, List>::process(typename List::NodePtr node) { |
| 513 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 513, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 514 | if (node->transient()) { |
| 515 | _prev = _list.excise(_prev, node); |
| 516 | } else { |
| 517 | _prev = node; |
| 518 | } |
| 519 | return ReleaseOp<Mspace>::process(node); |
| 520 | } |
| 521 | |
| 522 | template <typename Mspace, typename List> |
| 523 | class ScavengingReleaseOp : public StackObj { |
| 524 | protected: |
| 525 | Mspace* _mspace; |
| 526 | List& _list; |
| 527 | typename List::NodePtr _prev; |
| 528 | size_t _count; |
| 529 | size_t _amount; |
| 530 | bool excise_with_release(typename List::NodePtr node); |
| 531 | public: |
| 532 | typedef typename List::Node Node; |
| 533 | ScavengingReleaseOp(Mspace* mspace, List& list) : |
| 534 | _mspace(mspace), _list(list), _prev(NULL__null), _count(0), _amount(0) {} |
| 535 | bool process(typename List::NodePtr node); |
| 536 | size_t processed() const { return _count; } |
| 537 | size_t amount() const { return _amount; } |
| 538 | }; |
| 539 | |
| 540 | template <typename Mspace, typename List> |
| 541 | inline bool ScavengingReleaseOp<Mspace, List>::process(typename List::NodePtr node) { |
| 542 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 542, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 543 | assert(!node->transient(), "invariant")do { if (!(!node->transient())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 543, "assert(" "!node->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 544 | if (node->retired()) { |
| 545 | return excise_with_release(node); |
| 546 | } |
| 547 | _prev = node; |
| 548 | return true; |
| 549 | } |
| 550 | |
| 551 | template <typename Mspace, typename List> |
| 552 | inline bool ScavengingReleaseOp<Mspace, List>::excise_with_release(typename List::NodePtr node) { |
| 553 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 553, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 554 | assert(node->retired(), "invariant")do { if (!(node->retired())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 554, "assert(" "node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 555 | _prev = _list.excise(_prev, node); |
| 556 | if (node->transient()) { |
| 557 | _mspace->deallocate(node); |
| 558 | return true; |
| 559 | } |
| 560 | assert(node->identity() != NULL, "invariant")do { if (!(node->identity() != __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 560, "assert(" "node->identity() != __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 561 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 561, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 562 | assert(!node->lease(), "invariant")do { if (!(!node->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 562, "assert(" "!node->lease()" ") failed", "invariant") ; ::breakpoint(); } } while (0); |
| 563 | assert(!node->excluded(), "invariant")do { if (!(!node->excluded())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 563, "assert(" "!node->excluded()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 564 | ++_count; |
| 565 | _amount += node->total_size(); |
| 566 | node->clear_retired(); |
| 567 | node->release(); |
| 568 | mspace_release(node, _mspace); |
| 569 | return true; |
| 570 | } |
| 571 | |
| 572 | template <typename Mspace, typename FromList> |
| 573 | class ReleaseRetiredOp : public StackObj { |
| 574 | private: |
| 575 | Mspace* _mspace; |
| 576 | FromList& _list; |
| 577 | typename Mspace::NodePtr _prev; |
| 578 | public: |
| 579 | typedef typename Mspace::Node Node; |
| 580 | ReleaseRetiredOp(Mspace* mspace, FromList& list) : |
| 581 | _mspace(mspace), _list(list), _prev(NULL__null) {} |
| 582 | bool process(Node* node); |
| 583 | }; |
| 584 | |
| 585 | template <typename Mspace, typename FromList> |
| 586 | inline bool ReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) { |
| 587 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 587, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 588 | if (node->retired()) { |
| 589 | _prev = _list.excise(_prev, node); |
| 590 | node->reinitialize(); |
| 591 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 591, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 592 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 592, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 593 | node->release(); |
| 594 | mspace_release(node, _mspace); |
| 595 | } else { |
| 596 | _prev = node; |
| 597 | } |
| 598 | return true; |
| 599 | } |
| 600 | |
| 601 | template <typename Mspace, typename FromList> |
| 602 | class ReinitializeAllReleaseRetiredOp : public StackObj { |
| 603 | private: |
| 604 | Mspace* _mspace; |
| 605 | FromList& _list; |
| 606 | typename Mspace::NodePtr _prev; |
| 607 | public: |
| 608 | typedef typename Mspace::Node Node; |
| 609 | ReinitializeAllReleaseRetiredOp(Mspace* mspace, FromList& list) : |
| 610 | _mspace(mspace), _list(list), _prev(NULL__null) {} |
| 611 | bool process(Node* node); |
| 612 | }; |
| 613 | |
| 614 | template <typename Mspace, typename FromList> |
| 615 | inline bool ReinitializeAllReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) { |
| 616 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 616, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 617 | // assumes some means of exclusive access to node |
| 618 | const bool retired = node->retired(); |
| 619 | node->reinitialize(); |
| 620 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 620, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 621 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 621, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 622 | if (retired) { |
| 623 | _prev = _list.excise(_prev, node); |
| 624 | node->release(); |
| 625 | mspace_release(node, _mspace); |
| 626 | } else { |
| 627 | _prev = node; |
| 628 | } |
| 629 | return true; |
| 630 | } |
| 631 | |
| 632 | #ifdef ASSERT1 |
| 633 | template <typename Node> |
| 634 | inline void assert_migration_state(const Node* old, const Node* new_node, size_t used, size_t requested) { |
| 635 | assert(old != NULL, "invariant")do { if (!(old != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 635, "assert(" "old != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 636 | assert(new_node != NULL, "invariant")do { if (!(new_node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 636, "assert(" "new_node != __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 637 | assert(old->pos() >= old->start(), "invariant")do { if (!(old->pos() >= old->start())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 637, "assert(" "old->pos() >= old->start()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 638 | assert(old->pos() + used <= old->end(), "invariant")do { if (!(old->pos() + used <= old->end())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 638, "assert(" "old->pos() + used <= old->end()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 639 | assert(new_node->free_size() >= (used + requested), "invariant")do { if (!(new_node->free_size() >= (used + requested)) ) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 639, "assert(" "new_node->free_size() >= (used + requested)" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 640 | } |
| 641 | #endif // ASSERT |
| 642 | |
| 643 | template <typename Node> |
| 644 | inline void migrate_outstanding_writes(const Node* old, Node* new_node, size_t used, size_t requested) { |
| 645 | DEBUG_ONLY(assert_migration_state(old, new_node, used, requested);)assert_migration_state(old, new_node, used, requested); |
| 646 | if (used > 0) { |
| 647 | memcpy(new_node->pos(), old->pos(), used); |
| 648 | } |
| 649 | } |
| 650 | |
| 651 | #endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP |
| 1 | /* |
| 2 | * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP |
| 26 | #define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP |
| 27 | |
| 28 | #include "jfr/utilities/jfrConcurrentQueue.hpp" |
| 29 | |
| 30 | #include "jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" |
| 31 | #include "jfr/utilities/jfrVersionSystem.inline.hpp" |
| 32 | |
| 33 | template <typename NodeType, typename AllocPolicy> |
| 34 | JfrConcurrentQueue<NodeType, AllocPolicy>::JfrConcurrentQueue() : _list(NULL__null), _head(), _last(), _tail(), _version_system() { |
| 35 | _head._next = const_cast<NodePtr>(&_tail); |
| 36 | _last._next = const_cast<NodePtr>(&_tail); |
| 37 | } |
| 38 | |
| 39 | template <typename NodeType, typename AllocPolicy> |
| 40 | bool JfrConcurrentQueue<NodeType, AllocPolicy>::initialize() { |
| 41 | assert(_list == NULL, "invariant")do { if (!(_list == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp" , 41, "assert(" "_list == __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 42 | _list = new JfrConcurrentLinkedListHost<JfrConcurrentQueue<NodeType, AllocPolicy>, HeadNode, AllocPolicy>(this); |
| 43 | return _list != NULL__null && _list->initialize(); |
| 44 | } |
| 45 | |
| 46 | template <typename NodeType, typename AllocPolicy> |
| 47 | inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_empty() const { |
| 48 | return Atomic::load_acquire(&_head._next) == &_tail; |
| 49 | } |
| 50 | |
| 51 | template <typename NodeType, typename AllocPolicy> |
| 52 | inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_nonempty() const { |
| 53 | return !is_empty(); |
| 54 | } |
| 55 | |
| 56 | template <typename NodeType, typename AllocPolicy> |
| 57 | void JfrConcurrentQueue<NodeType, AllocPolicy>::add(typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr node) { |
| 58 | _list->insert_tail(node, &_head, &_last, &_tail); |
| 59 | } |
| 60 | |
| 61 | template <typename NodeType, typename AllocPolicy> |
| 62 | typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr JfrConcurrentQueue<NodeType, AllocPolicy>::remove() { |
| 63 | return _list->remove(&_head, &_tail, &_last, false); |
| 64 | } |
| 65 | |
| 66 | template <typename NodeType, typename AllocPolicy> |
| 67 | template <typename Callback> |
| 68 | void JfrConcurrentQueue<NodeType, AllocPolicy>::iterate(Callback& cb) { |
| 69 | _list->iterate(&_head, &_tail, cb); |
| 70 | } |
| 71 | |
| 72 | template <typename NodeType, typename AllocPolicy> |
| 73 | inline JfrVersionSystem::Handle JfrConcurrentQueue<NodeType, AllocPolicy>::get_version_handle() { |
| 74 | return _version_system.get(); |
| 75 | } |
| 76 | |
| 77 | template <typename NodeType, typename AllocPolicy> |
| 78 | bool JfrConcurrentQueue<NodeType, AllocPolicy>::in_list(const NodeType* node) const { |
| 79 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp" , 79, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 80 | return _list->in_list(node, const_cast<NodePtr>(&_head), &_tail); |
| 81 | } |
| 82 | |
| 83 | #endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP |
| 1 | /* | |||
| 2 | * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. | |||
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |||
| 4 | * | |||
| 5 | * This code is free software; you can redistribute it and/or modify it | |||
| 6 | * under the terms of the GNU General Public License version 2 only, as | |||
| 7 | * published by the Free Software Foundation. | |||
| 8 | * | |||
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT | |||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |||
| 12 | * version 2 for more details (a copy is included in the LICENSE file that | |||
| 13 | * accompanied this code). | |||
| 14 | * | |||
| 15 | * You should have received a copy of the GNU General Public License version | |||
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, | |||
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |||
| 18 | * | |||
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |||
| 20 | * or visit www.oracle.com if you need additional information or have any | |||
| 21 | * questions. | |||
| 22 | * | |||
| 23 | */ | |||
| 24 | ||||
| 25 | #ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP | |||
| 26 | #define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP | |||
| 27 | ||||
| 28 | #include "jfr/utilities/jfrConcurrentLinkedListHost.hpp" | |||
| 29 | ||||
| 30 | #include "jfr/utilities/jfrRelation.hpp" | |||
| 31 | #include "jfr/utilities/jfrTypes.hpp" | |||
| 32 | #include "runtime/atomic.hpp" | |||
| 33 | #include "utilities/globalDefinitions.hpp" | |||
| 34 | ||||
| 35 | /* | |||
| 36 | * The removal marker (i.e. the excision bit) is represented by '( )' as part of state description comments: | |||
| 37 | * "node --> next" becomes "(node) --> next", when node is logically deleted. | |||
| 38 | */ | |||
| 39 | template <typename Node> | |||
| 40 | inline Node* mark_for_removal(Node* node) { | |||
| 41 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 41, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 42 | const Node* next = node->_next; | |||
| 43 | assert(next != NULL, "invariant")do { if (!(next != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 43, "assert(" "next != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 44 | Node* const unmasked_next = unmask(next); | |||
| 45 | return next == unmasked_next && cas(&node->_next, unmasked_next, set_excision_bit(unmasked_next)) ? unmasked_next : NULL__null; | |||
| 46 | } | |||
| 47 | ||||
| 48 | /* | |||
| 49 | * The insertion marker (i.e. the insertion bit) is represented by '[ ]' as part of state description comments: | |||
| 50 | * "node --> next" becomes "[node] --> next", in an attempt to convey the node as exlusively reserved. | |||
| 51 | */ | |||
| 52 | template <typename Node> | |||
| 53 | inline bool mark_for_insertion(Node* node, const Node* tail) { | |||
| 54 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 54, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 55 | return node->_next == tail && cas(&node->_next, const_cast<Node*>(tail), set_insertion_bit(tail)); | |||
| 56 | } | |||
| 57 | ||||
| 58 | /* | |||
| 59 | * Find a predecessor and successor node pair where successor covers predecessor (adjacency). | |||
| 60 | */ | |||
| 61 | template <typename Node, typename VersionHandle, template <typename> class SearchPolicy> | |||
| 62 | Node* find_adjacent(Node* head, const Node* tail, Node** predecessor, VersionHandle& version_handle, SearchPolicy<Node>& predicate) { | |||
| 63 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 63, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 64 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 64, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 65 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 65, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 66 | Node* predecessor_next = NULL__null; | |||
| 67 | while (true) { | |||
| 68 | Node* current = head; | |||
| 69 | version_handle->checkout(); | |||
| 70 | Node* next = Atomic::load_acquire(¤t->_next); | |||
| 71 | do { | |||
| 72 | assert(next != NULL, "invariant")do { if (!(next != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 72, "assert(" "next != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 73 | Node* const unmasked_next = unmask(next); | |||
| 74 | // 1A: Locate the first node to keep as predecessor. | |||
| 75 | if (!is_marked_for_removal(next)) { | |||
| 76 | *predecessor = current; | |||
| 77 | predecessor_next = unmasked_next; | |||
| 78 | } | |||
| 79 | // 1B: Locate the next node to keep as successor. | |||
| 80 | current = unmasked_next; | |||
| 81 | if (current == tail) break; | |||
| 82 | next = current->_next; | |||
| 83 | } while (predicate(current, next)); | |||
| 84 | // current represents the successor node from here on out. | |||
| 85 | // 2: Check predecessor and successor node pair for adjacency. | |||
| 86 | if (predecessor_next == current) { | |||
| 87 | // Invariant: predecessor --> successor | |||
| 88 | return current; | |||
| 89 | } | |||
| 90 | // 3: Successor does not (yet) cover predecessor. | |||
| 91 | // Invariant: predecessor --> (logically excised nodes) --> successor | |||
| 92 | // Physically excise one or more logically excised nodes in-between. | |||
| 93 | if (cas(&(*predecessor)->_next, predecessor_next, current)) { | |||
| ||||
| 94 | // Invariant: predecessor --> successor | |||
| 95 | return current; | |||
| 96 | } | |||
| 97 | } | |||
| 98 | } | |||
| 99 | ||||
| 100 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 101 | JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::JfrConcurrentLinkedListHost(Client* client) : _client(client) {} | |||
| 102 | ||||
| 103 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 104 | bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::initialize() { | |||
| 105 | return true; | |||
| 106 | } | |||
| 107 | ||||
| 108 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 109 | void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_head(typename Client::Node* node, | |||
| 110 | typename Client::Node* head, | |||
| 111 | const typename Client::Node* tail) const { | |||
| 112 | Node* predecessor; | |||
| 113 | Node* successor; | |||
| 114 | HeadNode<Node> predicate(node); | |||
| 115 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 116 | while (true) { | |||
| 117 | // Find an adjacent predecessor and successor node pair. | |||
| 118 | successor = find_adjacent<Node, VersionHandle, HeadNode>(head, tail, &predecessor, version_handle, predicate); | |||
| 119 | // Invariant (adjacency): predecessor --> successor | |||
| 120 | // Invariant (optional: key-based total order): predecessor->key() < key && key <= successor->key(). | |||
| 121 | // We can now attempt to insert the new node in-between. | |||
| 122 | node->_next = successor; | |||
| 123 | if (cas(&predecessor->_next, successor, node)) { | |||
| 124 | // Invariant: predecessor --> node --> successor | |||
| 125 | // An insert to head is a benign modification and will not need to be committed to the version control system. | |||
| 126 | return; | |||
| 127 | } | |||
| 128 | } | |||
| 129 | } | |||
| 130 | ||||
| 131 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 132 | void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_tail(typename Client::Node* node, | |||
| 133 | typename Client::Node* head, | |||
| 134 | typename Client::Node* last, | |||
| 135 | const typename Client::Node* tail) const { | |||
| 136 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 136, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 137 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 137, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 138 | assert(last != NULL, "invarinat")do { if (!(last != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 138, "assert(" "last != __null" ") failed", "invarinat"); :: breakpoint(); } } while (0); | |||
| 139 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 139, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 140 | // Mark the new node to be inserted with the insertion marker already. | |||
| 141 | node->_next = set_insertion_bit(const_cast<NodePtr>(tail)); | |||
| 142 | // Invariant: [node]--> tail | |||
| 143 | assert(is_marked_for_insertion(node->_next), "invariant")do { if (!(is_marked_for_insertion(node->_next))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 143, "assert(" "is_marked_for_insertion(node->_next)" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 144 | NodePtr predecessor; | |||
| 145 | LastNode<Node> predicate; | |||
| 146 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 147 | while (true) { | |||
| 148 | // Find an adjacent predecessor and successor node pair, where the successor == tail | |||
| 149 | const NodePtr successor = find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, predicate); | |||
| 150 | assert(successor == tail, "invariant")do { if (!(successor == tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 150, "assert(" "successor == tail" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 151 | // Invariant: predecessor --> successor | |||
| 152 | // We first attempt to mark the predecessor node to signal our intent of performing an insertion. | |||
| 153 | if (mark_for_insertion(predecessor, tail)) { | |||
| 154 | break; | |||
| 155 | } | |||
| 156 | } | |||
| 157 | // Predecessor node is claimed for insertion. | |||
| 158 | // Invariant: [predecessor] --> tail | |||
| 159 | assert(is_marked_for_insertion(predecessor->_next), "invariant")do { if (!(is_marked_for_insertion(predecessor->_next))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 159, "assert(" "is_marked_for_insertion(predecessor->_next)" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 160 | assert(predecessor != head, "invariant")do { if (!(predecessor != head)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 160, "assert(" "predecessor != head" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 161 | if (Atomic::load_acquire(&last->_next) == predecessor) { | |||
| 162 | /* Even after we store the new node into the last->_next field, there is no race | |||
| 163 | because it is also marked with the insertion bit. */ | |||
| 164 | last->_next = node; | |||
| 165 | // Invariant: last --> [node] --> tail | |||
| 166 | OrderAccess::storestore(); | |||
| 167 | // Perform the link with the predecessor node, which by this store becomes visible for removal. | |||
| 168 | predecessor->_next = node; | |||
| 169 | // Invariant: predecessor --> [node] --> tail | |||
| 170 | } else { | |||
| 171 | assert(last == predecessor, "invariant")do { if (!(last == predecessor)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 171, "assert(" "last == predecessor" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 172 | last->_next = node; | |||
| 173 | // Invariant: last --> [node] --> tail | |||
| 174 | OrderAccess::storestore(); | |||
| 175 | /* This implies the list is logically empty from the removal perspective. | |||
| 176 | cas is not needed here because inserts must not come in from the head side | |||
| 177 | concurrently with inserts from tail which are currently blocked by us. | |||
| 178 | Invariant (logical): head --> tail. */ | |||
| 179 | head->_next = node; | |||
| 180 | // Invariant: head --> [node] --> tail | |||
| 181 | } | |||
| 182 | OrderAccess::storestore(); | |||
| 183 | // Publish the inserted node by removing the insertion marker. | |||
| 184 | node->_next = const_cast<NodePtr>(tail); | |||
| 185 | // Invariant: last --> node --> tail (possibly also head --> node --> tail) | |||
| 186 | } | |||
| 187 | ||||
| 188 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 189 | typename Client::Node* JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::remove(typename Client::Node* head, | |||
| 190 | const typename Client::Node* tail, | |||
| 191 | typename Client::Node* last /* NULL */, | |||
| 192 | bool insert_is_head /* true */) { | |||
| 193 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 193, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 194 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 194, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 195 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 195, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 196 | NodePtr predecessor; | |||
| 197 | NodePtr successor; | |||
| 198 | NodePtr successor_next; | |||
| 199 | SearchPolicy<Node> predicate; | |||
| 200 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 201 | while (true) { | |||
| 202 | // Find an adjacent predecessor and successor node pair. | |||
| 203 | successor = find_adjacent<Node, VersionHandle, SearchPolicy>(head, tail, &predecessor, version_handle, predicate); | |||
| 204 | if (successor == tail) { | |||
| 205 | return NULL__null; | |||
| 206 | } | |||
| 207 | // Invariant: predecessor --> successor | |||
| 208 | // Invariant (optional: key-based total order): predecessor->key() < key && key <= successor->key() | |||
| 209 | // It is the successor node that is to be removed. | |||
| 210 | // We first attempt to reserve (logically excise) the successor node. | |||
| 211 | successor_next = mark_for_removal(successor); | |||
| 212 | if (successor_next != NULL__null) { | |||
| 213 | break; | |||
| 214 | } | |||
| 215 | } | |||
| 216 | // Invariant: predecessor --> (successor) --> successor_next | |||
| 217 | // Successor node now logically excised. | |||
| 218 | assert(is_marked_for_removal(successor->_next), "invariant")do { if (!(is_marked_for_removal(successor->_next))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 218, "assert(" "is_marked_for_removal(successor->_next)" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 219 | // Now attempt to physically excise the successor node. | |||
| 220 | // If the cas fails, we can optimize for the slow path if we know we are not performing | |||
| 221 | // insertions from the head. Then a failed cas results not from a new node being inserted, | |||
| 222 | // but only because another thread excised us already. | |||
| 223 | if (!cas(&predecessor->_next, successor, successor_next) && insert_is_head) { | |||
| 224 | // Physically excise using slow path, can be completed asynchronously by other threads. | |||
| 225 | Identity<Node> excise(successor); | |||
| 226 | find_adjacent<Node, VersionHandle, Identity>(head, tail, &predecessor, version_handle, excise); | |||
| 227 | } | |||
| 228 | if (last != NULL__null && Atomic::load_acquire(&last->_next) == successor) { | |||
| 229 | guarantee(!insert_is_head, "invariant")do { if (!(!insert_is_head)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 229, "guarantee(" "!insert_is_head" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 230 | guarantee(successor_next == tail, "invariant")do { if (!(successor_next == tail)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 230, "guarantee(" "successor_next == tail" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 231 | LastNode<Node> excise; | |||
| 232 | find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, excise); | |||
| 233 | // Invariant: successor excised from last list | |||
| 234 | } | |||
| 235 | // Commit the modification back to the version control system. | |||
| 236 | // It blocks until all checkouts for versions earlier than the commit have been released. | |||
| 237 | version_handle->commit(); | |||
| 238 | // At this point we know there can be no references onto the excised node. It is safe, enjoy it. | |||
| 239 | return successor; | |||
| 240 | } | |||
| 241 | ||||
| 242 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 243 | bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::in_list(const typename Client::Node* node, | |||
| 244 | typename Client::Node* head, | |||
| 245 | const typename Client::Node* tail) const { | |||
| 246 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 246, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 247 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 247, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 248 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 248, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 249 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 250 | const Node* current = head; | |||
| 251 | version_handle->checkout(); | |||
| 252 | const Node* next = Atomic::load_acquire(¤t->_next); | |||
| 253 | while (true) { | |||
| 254 | if (!is_marked_for_removal(next)) { | |||
| 255 | if (current == node) { | |||
| 256 | return true; | |||
| 257 | } | |||
| 258 | } | |||
| 259 | current = unmask(next); | |||
| 260 | if (current == tail) break; | |||
| 261 | next = current->_next; | |||
| 262 | } | |||
| 263 | return false; | |||
| 264 | } | |||
| 265 | ||||
| 266 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 267 | template <typename Callback> | |||
| 268 | inline void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::iterate(typename Client::Node* head, | |||
| 269 | const typename Client::Node* tail, | |||
| 270 | Callback& cb) { | |||
| 271 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 271, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 272 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 272, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 273 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 273, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 274 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 275 | NodePtr current = head; | |||
| 276 | version_handle->checkout(); | |||
| 277 | NodePtr next = Atomic::load_acquire(¤t->_next); | |||
| 278 | while (true) { | |||
| 279 | if (!is_marked_for_removal(next)) { | |||
| 280 | if (!cb.process(current)) { | |||
| 281 | return; | |||
| 282 | } | |||
| 283 | } | |||
| 284 | current = unmask(next); | |||
| 285 | if (current == tail) break; | |||
| 286 | next = current->_next; | |||
| 287 | } | |||
| 288 | } | |||
| 289 | ||||
| 290 | #endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP |