| File: | jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp |
| Warning: | line 93, column 9 1st function call argument is an uninitialized value |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
| 1 | /* | |||
| 2 | * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. | |||
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |||
| 4 | * | |||
| 5 | * This code is free software; you can redistribute it and/or modify it | |||
| 6 | * under the terms of the GNU General Public License version 2 only, as | |||
| 7 | * published by the Free Software Foundation. | |||
| 8 | * | |||
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT | |||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |||
| 12 | * version 2 for more details (a copy is included in the LICENSE file that | |||
| 13 | * accompanied this code). | |||
| 14 | * | |||
| 15 | * You should have received a copy of the GNU General Public License version | |||
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, | |||
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |||
| 18 | * | |||
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |||
| 20 | * or visit www.oracle.com if you need additional information or have any | |||
| 21 | * questions. | |||
| 22 | * | |||
| 23 | */ | |||
| 24 | ||||
| 25 | #include "precompiled.hpp" | |||
| 26 | #include "jfr/jfrEvents.hpp" | |||
| 27 | #include "jfr/jni/jfrJavaSupport.hpp" | |||
| 28 | #include "jfr/recorder/jfrRecorder.hpp" | |||
| 29 | #include "jfr/recorder/checkpoint/jfrCheckpointManager.hpp" | |||
| 30 | #include "jfr/recorder/repository/jfrChunkWriter.hpp" | |||
| 31 | #include "jfr/recorder/service/jfrOptionSet.hpp" | |||
| 32 | #include "jfr/recorder/service/jfrPostBox.hpp" | |||
| 33 | #include "jfr/recorder/storage/jfrFullStorage.inline.hpp" | |||
| 34 | #include "jfr/recorder/storage/jfrMemorySpace.inline.hpp" | |||
| 35 | #include "jfr/recorder/storage/jfrStorage.hpp" | |||
| 36 | #include "jfr/recorder/storage/jfrStorageControl.hpp" | |||
| 37 | #include "jfr/recorder/storage/jfrStorageUtils.inline.hpp" | |||
| 38 | #include "jfr/utilities/jfrIterator.hpp" | |||
| 39 | #include "jfr/utilities/jfrLinkedList.inline.hpp" | |||
| 40 | #include "jfr/utilities/jfrTime.hpp" | |||
| 41 | #include "jfr/writers/jfrNativeEventWriter.hpp" | |||
| 42 | #include "logging/log.hpp" | |||
| 43 | #include "runtime/mutexLocker.hpp" | |||
| 44 | #include "runtime/safepoint.hpp" | |||
| 45 | #include "runtime/thread.hpp" | |||
| 46 | ||||
| 47 | typedef JfrStorage::BufferPtr BufferPtr; | |||
| 48 | ||||
| 49 | static JfrStorage* _instance = NULL__null; | |||
| 50 | static JfrStorageControl* _control; | |||
| 51 | ||||
| 52 | JfrStorage& JfrStorage::instance() { | |||
| 53 | return *_instance; | |||
| 54 | } | |||
| 55 | ||||
| 56 | JfrStorage* JfrStorage::create(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) { | |||
| 57 | assert(_instance == NULL, "invariant")do { if (!(_instance == __null)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 57, "assert(" "_instance == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 58 | _instance = new JfrStorage(chunkwriter, post_box); | |||
| 59 | return _instance; | |||
| 60 | } | |||
| 61 | ||||
| 62 | void JfrStorage::destroy() { | |||
| 63 | if (_instance != NULL__null) { | |||
| 64 | delete _instance; | |||
| 65 | _instance = NULL__null; | |||
| 66 | } | |||
| 67 | } | |||
| 68 | ||||
| 69 | JfrStorage::JfrStorage(JfrChunkWriter& chunkwriter, JfrPostBox& post_box) : | |||
| 70 | _control(NULL__null), | |||
| 71 | _global_mspace(NULL__null), | |||
| 72 | _thread_local_mspace(NULL__null), | |||
| 73 | _chunkwriter(chunkwriter), | |||
| 74 | _post_box(post_box) {} | |||
| 75 | ||||
| 76 | JfrStorage::~JfrStorage() { | |||
| 77 | if (_control != NULL__null) { | |||
| ||||
| 78 | delete _control; | |||
| 79 | } | |||
| 80 | if (_global_mspace != NULL__null) { | |||
| 81 | delete _global_mspace; | |||
| 82 | } | |||
| 83 | if (_thread_local_mspace != NULL__null) { | |||
| 84 | delete _thread_local_mspace; | |||
| 85 | } | |||
| 86 | if (_full_list != NULL__null) { | |||
| 87 | delete _full_list; | |||
| 88 | } | |||
| 89 | _instance = NULL__null; | |||
| 90 | } | |||
| 91 | ||||
| 92 | static const size_t thread_local_cache_count = 8; | |||
| 93 | // start to discard data when the only this number of free buffers are left | |||
| 94 | static const size_t in_memory_discard_threshold_delta = 2; | |||
| 95 | ||||
| 96 | bool JfrStorage::initialize() { | |||
| 97 | assert(_control == NULL, "invariant")do { if (!(_control == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 97, "assert(" "_control == __null" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 98 | assert(_global_mspace == NULL, "invariant")do { if (!(_global_mspace == __null)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 98, "assert(" "_global_mspace == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 99 | assert(_thread_local_mspace == NULL, "invariant")do { if (!(_thread_local_mspace == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 99, "assert(" "_thread_local_mspace == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 100 | ||||
| 101 | const size_t num_global_buffers = (size_t)JfrOptionSet::num_global_buffers(); | |||
| 102 | assert(num_global_buffers >= in_memory_discard_threshold_delta, "invariant")do { if (!(num_global_buffers >= in_memory_discard_threshold_delta )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 102, "assert(" "num_global_buffers >= in_memory_discard_threshold_delta" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 103 | const size_t global_buffer_size = (size_t)JfrOptionSet::global_buffer_size(); | |||
| 104 | const size_t thread_buffer_size = (size_t)JfrOptionSet::thread_buffer_size(); | |||
| 105 | ||||
| 106 | _control = new JfrStorageControl(num_global_buffers, num_global_buffers - in_memory_discard_threshold_delta); | |||
| 107 | if (_control == NULL__null) { | |||
| 108 | return false; | |||
| 109 | } | |||
| 110 | _global_mspace = create_mspace<JfrStorageMspace>(global_buffer_size, | |||
| 111 | num_global_buffers, // cache count limit | |||
| 112 | num_global_buffers, // cache_preallocate count | |||
| 113 | false, // preallocate_to_free_list (== preallocate directly to live list) | |||
| 114 | this); | |||
| 115 | if (_global_mspace == NULL__null) { | |||
| 116 | return false; | |||
| 117 | } | |||
| 118 | assert(_global_mspace->live_list_is_nonempty(), "invariant")do { if (!(_global_mspace->live_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 118, "assert(" "_global_mspace->live_list_is_nonempty()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 119 | _thread_local_mspace = create_mspace<JfrThreadLocalMspace>(thread_buffer_size, | |||
| 120 | thread_local_cache_count, // cache count limit | |||
| 121 | thread_local_cache_count, // cache preallocate count | |||
| 122 | true, // preallocate_to_free_list | |||
| 123 | this); | |||
| 124 | if (_thread_local_mspace == NULL__null) { | |||
| 125 | return false; | |||
| 126 | } | |||
| 127 | assert(_thread_local_mspace->free_list_is_nonempty(), "invariant")do { if (!(_thread_local_mspace->free_list_is_nonempty())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 127, "assert(" "_thread_local_mspace->free_list_is_nonempty()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 128 | // The full list will contain nodes pointing to retired global and transient buffers. | |||
| 129 | _full_list = new JfrFullList(*_control); | |||
| 130 | return _full_list != NULL__null && _full_list->initialize(num_global_buffers * 2); | |||
| 131 | } | |||
| 132 | ||||
| 133 | JfrStorageControl& JfrStorage::control() { | |||
| 134 | return *instance()._control; | |||
| 135 | } | |||
| 136 | ||||
| 137 | static void log_allocation_failure(const char* msg, size_t size) { | |||
| 138 | log_warning(jfr)(!(LogImpl<(LogTag::_jfr), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Warning))) ? (void)0 : LogImpl<(LogTag ::_jfr), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>::write<LogLevel ::Warning>("Unable to allocate " SIZE_FORMAT"%" "l" "u" " bytes of %s.", size, msg); | |||
| 139 | } | |||
| 140 | ||||
| 141 | BufferPtr JfrStorage::acquire_thread_local(Thread* thread, size_t size /* 0 */) { | |||
| 142 | BufferPtr buffer = mspace_acquire_to_live_list(size, instance()._thread_local_mspace, thread); | |||
| 143 | if (buffer == NULL__null) { | |||
| 144 | log_allocation_failure("thread local_memory", size); | |||
| 145 | return NULL__null; | |||
| 146 | } | |||
| 147 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 147, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 148 | return buffer; | |||
| 149 | } | |||
| 150 | ||||
| 151 | BufferPtr JfrStorage::acquire_transient(size_t size, Thread* thread) { | |||
| 152 | BufferPtr buffer = mspace_allocate_transient_lease(size, instance()._thread_local_mspace, thread); | |||
| 153 | if (buffer == NULL__null) { | |||
| 154 | log_allocation_failure("transient memory", size); | |||
| 155 | return NULL__null; | |||
| 156 | } | |||
| 157 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 157, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 158 | assert(buffer->transient(), "invariant")do { if (!(buffer->transient())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 158, "assert(" "buffer->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 159 | assert(buffer->lease(), "invariant")do { if (!(buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 159, "assert(" "buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 160 | return buffer; | |||
| 161 | } | |||
| 162 | ||||
| 163 | static BufferPtr acquire_lease(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) { | |||
| 164 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 164, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 165 | while (true) { | |||
| 166 | BufferPtr buffer = mspace_acquire_lease_with_retry(size, mspace, retry_count, thread); | |||
| 167 | if (buffer == NULL__null && storage_instance.control().should_discard()) { | |||
| 168 | storage_instance.discard_oldest(thread); | |||
| 169 | continue; | |||
| 170 | } | |||
| 171 | return buffer; | |||
| 172 | } | |||
| 173 | } | |||
| 174 | ||||
| 175 | static BufferPtr acquire_promotion_buffer(size_t size, JfrStorageMspace* mspace, JfrStorage& storage_instance, size_t retry_count, Thread* thread) { | |||
| 176 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 176, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 177 | while (true) { | |||
| 178 | BufferPtr buffer= mspace_acquire_live_with_retry(size, mspace, retry_count, thread); | |||
| 179 | if (buffer == NULL__null && storage_instance.control().should_discard()) { | |||
| 180 | storage_instance.discard_oldest(thread); | |||
| 181 | continue; | |||
| 182 | } | |||
| 183 | return buffer; | |||
| 184 | } | |||
| 185 | } | |||
| 186 | ||||
| 187 | static const size_t lease_retry = 10; | |||
| 188 | ||||
| 189 | BufferPtr JfrStorage::acquire_large(size_t size, Thread* thread) { | |||
| 190 | JfrStorage& storage_instance = instance(); | |||
| 191 | const size_t max_elem_size = storage_instance._global_mspace->min_element_size(); // min is also max | |||
| 192 | // if not too large and capacity is still available, ask for a lease from the global system | |||
| 193 | if (size < max_elem_size && storage_instance.control().is_global_lease_allowed()) { | |||
| 194 | BufferPtr const buffer = acquire_lease(size, storage_instance._global_mspace, storage_instance, lease_retry, thread); | |||
| 195 | if (buffer != NULL__null) { | |||
| 196 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 196, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 197 | assert(!buffer->transient(), "invariant")do { if (!(!buffer->transient())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 197, "assert(" "!buffer->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 198 | assert(buffer->lease(), "invariant")do { if (!(buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 198, "assert(" "buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 199 | storage_instance.control().increment_leased(); | |||
| 200 | return buffer; | |||
| 201 | } | |||
| 202 | } | |||
| 203 | return acquire_transient(size, thread); | |||
| 204 | } | |||
| 205 | ||||
| 206 | static void write_data_loss_event(JfrBuffer* buffer, u8 unflushed_size, Thread* thread) { | |||
| 207 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 207, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 208 | assert(buffer->empty(), "invariant")do { if (!(buffer->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 208, "assert(" "buffer->empty()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 209 | const u8 total_data_loss = thread->jfr_thread_local()->add_data_lost(unflushed_size); | |||
| 210 | if (EventDataLoss::is_enabled()) { | |||
| 211 | JfrNativeEventWriter writer(buffer, thread); | |||
| 212 | writer.begin_event_write(false); | |||
| 213 | writer.write<u8>(EventDataLoss::eventId); | |||
| 214 | writer.write(JfrTicks::now()); | |||
| 215 | writer.write(unflushed_size); | |||
| 216 | writer.write(total_data_loss); | |||
| 217 | writer.end_event_write(false); | |||
| 218 | } | |||
| 219 | } | |||
| 220 | ||||
| 221 | static void write_data_loss(BufferPtr buffer, Thread* thread) { | |||
| 222 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 222, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 223 | const size_t unflushed_size = buffer->unflushed_size(); | |||
| 224 | buffer->reinitialize(); | |||
| 225 | if (unflushed_size == 0) { | |||
| 226 | return; | |||
| 227 | } | |||
| 228 | write_data_loss_event(buffer, unflushed_size, thread); | |||
| 229 | } | |||
| 230 | ||||
| 231 | static const size_t promotion_retry = 100; | |||
| 232 | ||||
| 233 | bool JfrStorage::flush_regular_buffer(BufferPtr buffer, Thread* thread) { | |||
| 234 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 234, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 235 | assert(!buffer->lease(), "invariant")do { if (!(!buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 235, "assert(" "!buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 236 | assert(!buffer->transient(), "invariant")do { if (!(!buffer->transient())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 236, "assert(" "!buffer->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 237 | const size_t unflushed_size = buffer->unflushed_size(); | |||
| 238 | if (unflushed_size == 0) { | |||
| 239 | buffer->reinitialize(); | |||
| 240 | assert(buffer->empty(), "invariant")do { if (!(buffer->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 240, "assert(" "buffer->empty()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 241 | return true; | |||
| 242 | } | |||
| 243 | ||||
| 244 | if (buffer->excluded()) { | |||
| 245 | const bool thread_is_excluded = thread->jfr_thread_local()->is_excluded(); | |||
| 246 | buffer->reinitialize(thread_is_excluded); | |||
| 247 | assert(buffer->empty(), "invariant")do { if (!(buffer->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 247, "assert(" "buffer->empty()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 248 | if (!thread_is_excluded) { | |||
| 249 | // state change from exclusion to inclusion requires a thread checkpoint | |||
| 250 | JfrCheckpointManager::write_thread_checkpoint(thread); | |||
| 251 | } | |||
| 252 | return true; | |||
| 253 | } | |||
| 254 | ||||
| 255 | BufferPtr const promotion_buffer = acquire_promotion_buffer(unflushed_size, _global_mspace, *this, promotion_retry, thread); | |||
| 256 | if (promotion_buffer == NULL__null) { | |||
| 257 | write_data_loss(buffer, thread); | |||
| 258 | return false; | |||
| 259 | } | |||
| 260 | assert(promotion_buffer->acquired_by_self(), "invariant")do { if (!(promotion_buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 260, "assert(" "promotion_buffer->acquired_by_self()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 261 | assert(promotion_buffer->free_size() >= unflushed_size, "invariant")do { if (!(promotion_buffer->free_size() >= unflushed_size )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 261, "assert(" "promotion_buffer->free_size() >= unflushed_size" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 262 | buffer->move(promotion_buffer, unflushed_size); | |||
| 263 | assert(buffer->empty(), "invariant")do { if (!(buffer->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 263, "assert(" "buffer->empty()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 264 | return true; | |||
| 265 | } | |||
| 266 | ||||
| 267 | /* | |||
| 268 | * 1. If the buffer was a "lease" from the global system, release back. | |||
| 269 | * 2. If the buffer is transient (temporal dynamically allocated), retire and register full. | |||
| 270 | * | |||
| 271 | * The buffer is effectively invalidated for the thread post-return, | |||
| 272 | * and the caller should take means to ensure that it is not referenced any longer. | |||
| 273 | */ | |||
| 274 | void JfrStorage::release_large(BufferPtr buffer, Thread* thread) { | |||
| 275 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 275, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 276 | assert(buffer->lease(), "invariant")do { if (!(buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 276, "assert(" "buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 277 | assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 277, "assert(" "buffer->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 278 | buffer->clear_lease(); | |||
| 279 | if (buffer->transient()) { | |||
| 280 | buffer->set_retired(); | |||
| 281 | register_full(buffer, thread); | |||
| 282 | } else { | |||
| 283 | buffer->release(); | |||
| 284 | control().decrement_leased(); | |||
| 285 | } | |||
| 286 | } | |||
| 287 | ||||
| 288 | void JfrStorage::register_full(BufferPtr buffer, Thread* thread) { | |||
| 289 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 289, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 290 | assert(buffer->acquired_by(thread), "invariant")do { if (!(buffer->acquired_by(thread))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 290, "assert(" "buffer->acquired_by(thread)" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 291 | assert(buffer->retired(), "invariant")do { if (!(buffer->retired())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 291, "assert(" "buffer->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 292 | if (_full_list->add(buffer)) { | |||
| 293 | _post_box.post(MSG_FULLBUFFER); | |||
| 294 | } | |||
| 295 | } | |||
| 296 | ||||
| 297 | // don't use buffer on return, it is gone | |||
| 298 | void JfrStorage::release(BufferPtr buffer, Thread* thread) { | |||
| 299 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 299, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 300 | assert(!buffer->lease(), "invariant")do { if (!(!buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 300, "assert(" "!buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 301 | assert(!buffer->transient(), "invariant")do { if (!(!buffer->transient())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 301, "assert(" "!buffer->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 302 | assert(!buffer->retired(), "invariant")do { if (!(!buffer->retired())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 302, "assert(" "!buffer->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 303 | if (!buffer->empty()) { | |||
| 304 | if (!flush_regular_buffer(buffer, thread)) { | |||
| 305 | buffer->reinitialize(); | |||
| 306 | } | |||
| 307 | } | |||
| 308 | assert(buffer->empty(), "invariant")do { if (!(buffer->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 308, "assert(" "buffer->empty()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 309 | assert(buffer->identity() != NULL, "invariant")do { if (!(buffer->identity() != __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 309, "assert(" "buffer->identity() != __null" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 310 | buffer->clear_excluded(); | |||
| 311 | buffer->set_retired(); | |||
| 312 | } | |||
| 313 | ||||
| 314 | void JfrStorage::release_thread_local(BufferPtr buffer, Thread* thread) { | |||
| 315 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 315, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 316 | JfrStorage& storage_instance = instance(); | |||
| 317 | storage_instance.release(buffer, thread); | |||
| 318 | } | |||
| 319 | ||||
| 320 | static void log_discard(size_t pre_full_count, size_t post_full_count, size_t amount) { | |||
| 321 | if (log_is_enabled(Debug, jfr, system)(LogImpl<(LogTag::_jfr), (LogTag::_system), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Debug))) { | |||
| 322 | const size_t number_of_discards = pre_full_count - post_full_count; | |||
| 323 | if (number_of_discards > 0) { | |||
| 324 | log_debug(jfr, system)(!(LogImpl<(LogTag::_jfr), (LogTag::_system), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Debug))) ? (void)0 : LogImpl<(LogTag ::_jfr), (LogTag::_system), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>::write<LogLevel ::Debug>("Cleared " SIZE_FORMAT"%" "l" "u" " full buffer(s) of " SIZE_FORMAT"%" "l" "u"" bytes.", number_of_discards, amount); | |||
| 325 | log_debug(jfr, system)(!(LogImpl<(LogTag::_jfr), (LogTag::_system), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Debug))) ? (void)0 : LogImpl<(LogTag ::_jfr), (LogTag::_system), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>::write<LogLevel ::Debug>("Current number of full buffers " SIZE_FORMAT"%" "l" "u" "", number_of_discards); | |||
| 326 | } | |||
| 327 | } | |||
| 328 | } | |||
| 329 | ||||
| 330 | void JfrStorage::discard_oldest(Thread* thread) { | |||
| 331 | if (JfrBuffer_lock->try_lock()) { | |||
| 332 | if (!control().should_discard()) { | |||
| 333 | // another thread handled it | |||
| 334 | return; | |||
| 335 | } | |||
| 336 | const size_t num_full_pre_discard = control().full_count(); | |||
| 337 | size_t discarded_size = 0; | |||
| 338 | while (_full_list->is_nonempty()) { | |||
| 339 | BufferPtr oldest = _full_list->remove(); | |||
| 340 | assert(oldest != NULL, "invariant")do { if (!(oldest != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 340, "assert(" "oldest != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 341 | assert(oldest->identity() != NULL, "invariant")do { if (!(oldest->identity() != __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 341, "assert(" "oldest->identity() != __null" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 342 | discarded_size += oldest->discard(); | |||
| 343 | assert(oldest->unflushed_size() == 0, "invariant")do { if (!(oldest->unflushed_size() == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 343, "assert(" "oldest->unflushed_size() == 0" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 344 | if (oldest->transient()) { | |||
| 345 | mspace_release(oldest, _thread_local_mspace); | |||
| 346 | continue; | |||
| 347 | } | |||
| 348 | oldest->reinitialize(); | |||
| 349 | assert(!oldest->retired(), "invariant")do { if (!(!oldest->retired())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 349, "assert(" "!oldest->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 350 | oldest->release(); // publish | |||
| 351 | break; | |||
| 352 | } | |||
| 353 | JfrBuffer_lock->unlock(); | |||
| 354 | log_discard(num_full_pre_discard, control().full_count(), discarded_size); | |||
| 355 | } | |||
| 356 | } | |||
| 357 | ||||
| 358 | #ifdef ASSERT1 | |||
| 359 | typedef const BufferPtr ConstBufferPtr; | |||
| 360 | ||||
| 361 | static void assert_flush_precondition(ConstBufferPtr cur, size_t used, bool native, const Thread* t) { | |||
| 362 | assert(t != NULL, "invariant")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 362, "assert(" "t != __null" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 363 | assert(cur != NULL, "invariant")do { if (!(cur != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 363, "assert(" "cur != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 364 | assert(cur->pos() + used <= cur->end(), "invariant")do { if (!(cur->pos() + used <= cur->end())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 364, "assert(" "cur->pos() + used <= cur->end()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 365 | assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant")do { if (!(native ? t->jfr_thread_local()->native_buffer () == cur : t->jfr_thread_local()->java_buffer() == cur )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 365, "assert(" "native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 366 | } | |||
| 367 | ||||
| 368 | static void assert_flush_regular_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, const Thread* t) { | |||
| 369 | assert(t != NULL, "invariant")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 369, "assert(" "t != __null" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 370 | assert(cur != NULL, "invariant")do { if (!(cur != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 370, "assert(" "cur != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 371 | assert(!cur->lease(), "invariant")do { if (!(!cur->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 371, "assert(" "!cur->lease()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 372 | assert(cur_pos != NULL, "invariant")do { if (!(cur_pos != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 372, "assert(" "cur_pos != __null" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 373 | assert(req >= used, "invariant")do { if (!(req >= used)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 373, "assert(" "req >= used" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 374 | } | |||
| 375 | ||||
| 376 | static void assert_provision_large_precondition(ConstBufferPtr cur, size_t used, size_t req, const Thread* t) { | |||
| 377 | assert(cur != NULL, "invariant")do { if (!(cur != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 377, "assert(" "cur != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 378 | assert(t != NULL, "invariant")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 378, "assert(" "t != __null" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 379 | assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant")do { if (!(t->jfr_thread_local()->shelved_buffer() != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 379, "assert(" "t->jfr_thread_local()->shelved_buffer() != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 380 | assert(req >= used, "invariant")do { if (!(req >= used)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 380, "assert(" "req >= used" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 381 | } | |||
| 382 | ||||
| 383 | static void assert_flush_large_precondition(ConstBufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { | |||
| 384 | assert(t != NULL, "invariant")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 384, "assert(" "t != __null" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 385 | assert(cur != NULL, "invariant")do { if (!(cur != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 385, "assert(" "cur != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 386 | assert(cur->lease(), "invariant")do { if (!(cur->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 386, "assert(" "cur->lease()" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 387 | assert(!cur->excluded(), "invariant")do { if (!(!cur->excluded())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 387, "assert(" "!cur->excluded()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 388 | assert(cur_pos != NULL, "invariant")do { if (!(cur_pos != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 388, "assert(" "cur_pos != __null" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 389 | assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant")do { if (!(native ? t->jfr_thread_local()->native_buffer () == cur : t->jfr_thread_local()->java_buffer() == cur )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 389, "assert(" "native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 390 | assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant")do { if (!(t->jfr_thread_local()->shelved_buffer() != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 390, "assert(" "t->jfr_thread_local()->shelved_buffer() != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 391 | assert(req >= used, "invariant")do { if (!(req >= used)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 391, "assert(" "req >= used" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 392 | assert(cur != t->jfr_thread_local()->shelved_buffer(), "invariant")do { if (!(cur != t->jfr_thread_local()->shelved_buffer ())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 392, "assert(" "cur != t->jfr_thread_local()->shelved_buffer()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 393 | } | |||
| 394 | #endif // ASSERT | |||
| 395 | ||||
| 396 | BufferPtr JfrStorage::flush(BufferPtr cur, size_t used, size_t req, bool native, Thread* t) { | |||
| 397 | debug_only(assert_flush_precondition(cur, used, native, t);)assert_flush_precondition(cur, used, native, t); | |||
| 398 | const u1* const cur_pos = cur->pos(); | |||
| 399 | req += used; | |||
| 400 | // requested size now encompass the outstanding used size | |||
| 401 | return cur->lease() ? instance().flush_large(cur, cur_pos, used, req, native, t) : | |||
| 402 | instance().flush_regular(cur, cur_pos, used, req, native, t); | |||
| 403 | } | |||
| 404 | ||||
| 405 | BufferPtr JfrStorage::flush_regular(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { | |||
| 406 | debug_only(assert_flush_regular_precondition(cur, cur_pos, used, req, t);)assert_flush_regular_precondition(cur, cur_pos, used, req, t) ; | |||
| 407 | // A flush is needed before memmove since a non-large buffer is thread stable | |||
| 408 | // (thread local). The flush will not modify memory in addresses above pos() | |||
| 409 | // which is where the "used / uncommitted" data resides. It is therefore both | |||
| 410 | // possible and valid to migrate data after the flush. This is however only | |||
| 411 | // the case for stable thread local buffers; it is not the case for large buffers. | |||
| 412 | flush_regular_buffer(cur, t); | |||
| 413 | if (cur->excluded()) { | |||
| 414 | return cur; | |||
| 415 | } | |||
| 416 | if (cur->free_size() >= req) { | |||
| 417 | // simplest case, no switching of buffers | |||
| 418 | if (used > 0) { | |||
| 419 | // source and destination may overlap so memmove must be used instead of memcpy | |||
| 420 | memmove(cur->pos(), (void*)cur_pos, used); | |||
| 421 | } | |||
| 422 | assert(native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur, "invariant")do { if (!(native ? t->jfr_thread_local()->native_buffer () == cur : t->jfr_thread_local()->java_buffer() == cur )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 422, "assert(" "native ? t->jfr_thread_local()->native_buffer() == cur : t->jfr_thread_local()->java_buffer() == cur" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 423 | return cur; | |||
| 424 | } | |||
| 425 | // Going for a "larger-than-regular" buffer. | |||
| 426 | // Shelve the current buffer to make room for a temporary lease. | |||
| 427 | assert(t->jfr_thread_local()->shelved_buffer() == NULL, "invariant")do { if (!(t->jfr_thread_local()->shelved_buffer() == __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 427, "assert(" "t->jfr_thread_local()->shelved_buffer() == __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 428 | t->jfr_thread_local()->shelve_buffer(cur); | |||
| 429 | return provision_large(cur, cur_pos, used, req, native, t); | |||
| 430 | } | |||
| 431 | ||||
| 432 | static BufferPtr store_buffer_to_thread_local(BufferPtr buffer, JfrThreadLocal* jfr_thread_local, bool native) { | |||
| 433 | assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 433, "assert(" "buffer != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 434 | if (native) { | |||
| 435 | jfr_thread_local->set_native_buffer(buffer); | |||
| 436 | } else { | |||
| 437 | jfr_thread_local->set_java_buffer(buffer); | |||
| 438 | } | |||
| 439 | return buffer; | |||
| 440 | } | |||
| 441 | ||||
| 442 | static BufferPtr restore_shelved_buffer(bool native, Thread* t) { | |||
| 443 | JfrThreadLocal* const tl = t->jfr_thread_local(); | |||
| 444 | BufferPtr shelved = tl->shelved_buffer(); | |||
| 445 | assert(shelved != NULL, "invariant")do { if (!(shelved != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 445, "assert(" "shelved != __null" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 446 | tl->shelve_buffer(NULL__null); | |||
| 447 | // restore shelved buffer back as primary | |||
| 448 | return store_buffer_to_thread_local(shelved, tl, native); | |||
| 449 | } | |||
| 450 | ||||
| 451 | BufferPtr JfrStorage::flush_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { | |||
| 452 | debug_only(assert_flush_large_precondition(cur, cur_pos, used, req, native, t);)assert_flush_large_precondition(cur, cur_pos, used, req, native , t); | |||
| 453 | // Can the "regular" buffer (now shelved) accommodate the requested size? | |||
| 454 | BufferPtr shelved = t->jfr_thread_local()->shelved_buffer(); | |||
| 455 | assert(shelved != NULL, "invariant")do { if (!(shelved != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 455, "assert(" "shelved != __null" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 456 | if (shelved->free_size() >= req) { | |||
| 457 | if (req > 0) { | |||
| 458 | memcpy(shelved->pos(), (void*)cur_pos, (size_t)used); | |||
| 459 | } | |||
| 460 | // release and invalidate | |||
| 461 | release_large(cur, t); | |||
| 462 | return restore_shelved_buffer(native, t); | |||
| 463 | } | |||
| 464 | // regular too small | |||
| 465 | return provision_large(cur, cur_pos, used, req, native, t); | |||
| 466 | } | |||
| 467 | ||||
| 468 | static BufferPtr large_fail(BufferPtr cur, bool native, JfrStorage& storage_instance, Thread* t) { | |||
| 469 | assert(cur != NULL, "invariant")do { if (!(cur != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 469, "assert(" "cur != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 470 | assert(t != NULL, "invariant")do { if (!(t != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 470, "assert(" "t != __null" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 471 | if (cur->lease()) { | |||
| 472 | storage_instance.release_large(cur, t); | |||
| 473 | } | |||
| 474 | return restore_shelved_buffer(native, t); | |||
| 475 | } | |||
| 476 | ||||
| 477 | // Always returns a non-null buffer. | |||
| 478 | // If accommodating the large request fails, the shelved buffer is returned | |||
| 479 | // even though it might be smaller than the requested size. | |||
| 480 | // Caller needs to ensure if the size was successfully accommodated. | |||
| 481 | BufferPtr JfrStorage::provision_large(BufferPtr cur, const u1* const cur_pos, size_t used, size_t req, bool native, Thread* t) { | |||
| 482 | debug_only(assert_provision_large_precondition(cur, used, req, t);)assert_provision_large_precondition(cur, used, req, t); | |||
| 483 | assert(t->jfr_thread_local()->shelved_buffer() != NULL, "invariant")do { if (!(t->jfr_thread_local()->shelved_buffer() != __null )) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 483, "assert(" "t->jfr_thread_local()->shelved_buffer() != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 484 | BufferPtr const buffer = acquire_large(req, t); | |||
| 485 | if (buffer == NULL__null) { | |||
| 486 | // unable to allocate and serve the request | |||
| 487 | return large_fail(cur, native, *this, t); | |||
| 488 | } | |||
| 489 | // ok managed to acquire a "large" buffer for the requested size | |||
| 490 | assert(buffer->free_size() >= req, "invariant")do { if (!(buffer->free_size() >= req)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 490, "assert(" "buffer->free_size() >= req" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 491 | assert(buffer->lease(), "invariant")do { if (!(buffer->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 491, "assert(" "buffer->lease()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 492 | // transfer outstanding data | |||
| 493 | memcpy(buffer->pos(), (void*)cur_pos, used); | |||
| 494 | if (cur->lease()) { | |||
| 495 | release_large(cur, t); | |||
| 496 | // don't use current anymore, it is gone | |||
| 497 | } | |||
| 498 | return store_buffer_to_thread_local(buffer, t->jfr_thread_local(), native); | |||
| 499 | } | |||
| 500 | ||||
| 501 | typedef UnBufferedWriteToChunk<JfrBuffer> WriteOperation; | |||
| 502 | typedef MutexedWriteOp<WriteOperation> MutexedWriteOperation; | |||
| 503 | typedef ConcurrentWriteOp<WriteOperation> ConcurrentWriteOperation; | |||
| 504 | ||||
| 505 | typedef Excluded<JfrBuffer, true> NonExcluded; | |||
| 506 | typedef PredicatedConcurrentWriteOp<WriteOperation, NonExcluded> ConcurrentNonExcludedWriteOperation; | |||
| 507 | ||||
| 508 | typedef ScavengingReleaseOp<JfrThreadLocalMspace, JfrThreadLocalMspace::LiveList> ReleaseThreadLocalOperation; | |||
| 509 | typedef CompositeOperation<ConcurrentNonExcludedWriteOperation, ReleaseThreadLocalOperation> ConcurrentWriteReleaseThreadLocalOperation; | |||
| 510 | ||||
| 511 | size_t JfrStorage::write() { | |||
| 512 | const size_t full_elements = write_full(); | |||
| 513 | WriteOperation wo(_chunkwriter); | |||
| 514 | NonExcluded ne; | |||
| 515 | ConcurrentNonExcludedWriteOperation cnewo(wo, ne); | |||
| 516 | ReleaseThreadLocalOperation rtlo(_thread_local_mspace, _thread_local_mspace->live_list()); | |||
| 517 | ConcurrentWriteReleaseThreadLocalOperation tlop(&cnewo, &rtlo); | |||
| 518 | process_live_list(tlop, _thread_local_mspace); | |||
| 519 | assert(_global_mspace->free_list_is_empty(), "invariant")do { if (!(_global_mspace->free_list_is_empty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 519, "assert(" "_global_mspace->free_list_is_empty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 520 | assert(_global_mspace->live_list_is_nonempty(), "invariant")do { if (!(_global_mspace->live_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 520, "assert(" "_global_mspace->live_list_is_nonempty()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 521 | process_live_list(cnewo, _global_mspace); | |||
| 522 | return full_elements + wo.elements(); | |||
| 523 | } | |||
| 524 | ||||
| 525 | size_t JfrStorage::write_at_safepoint() { | |||
| 526 | assert(SafepointSynchronize::is_at_safepoint(), "invariant")do { if (!(SafepointSynchronize::is_at_safepoint())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 526, "assert(" "SafepointSynchronize::is_at_safepoint()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 527 | const size_t full_elements = write_full(); | |||
| 528 | WriteOperation wo(_chunkwriter); | |||
| 529 | NonExcluded ne; | |||
| 530 | ConcurrentNonExcludedWriteOperation cnewo(wo, ne); // concurrent because of gc's | |||
| 531 | process_live_list(cnewo, _thread_local_mspace); | |||
| 532 | assert(_global_mspace->free_list_is_empty(), "invariant")do { if (!(_global_mspace->free_list_is_empty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 532, "assert(" "_global_mspace->free_list_is_empty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 533 | assert(_global_mspace->live_list_is_nonempty(), "invariant")do { if (!(_global_mspace->live_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 533, "assert(" "_global_mspace->live_list_is_nonempty()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 534 | process_live_list(cnewo, _global_mspace); | |||
| 535 | return full_elements + wo.elements(); | |||
| 536 | } | |||
| 537 | ||||
| 538 | typedef DiscardOp<DefaultDiscarder<JfrStorage::Buffer> > DiscardOperation; | |||
| 539 | typedef CompositeOperation<DiscardOperation, ReleaseThreadLocalOperation> DiscardReleaseThreadLocalOperation; | |||
| 540 | ||||
| 541 | size_t JfrStorage::clear() { | |||
| 542 | const size_t full_elements = clear_full(); | |||
| 543 | DiscardOperation discarder(concurrent); // concurrent discard mode | |||
| 544 | ReleaseThreadLocalOperation rtlo(_thread_local_mspace, _thread_local_mspace->live_list()); | |||
| 545 | DiscardReleaseThreadLocalOperation tldo(&discarder, &rtlo); | |||
| 546 | process_live_list(tldo, _thread_local_mspace); | |||
| 547 | assert(_global_mspace->free_list_is_empty(), "invariant")do { if (!(_global_mspace->free_list_is_empty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 547, "assert(" "_global_mspace->free_list_is_empty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 548 | assert(_global_mspace->live_list_is_nonempty(), "invariant")do { if (!(_global_mspace->live_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 548, "assert(" "_global_mspace->live_list_is_nonempty()" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 549 | process_live_list(discarder, _global_mspace); | |||
| 550 | return full_elements + discarder.elements(); | |||
| 551 | } | |||
| 552 | ||||
| 553 | template <typename Processor> | |||
| 554 | static size_t process_full(Processor& processor, JfrFullList* list, JfrStorageControl& control) { | |||
| 555 | assert(list != NULL, "invariant")do { if (!(list != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 555, "assert(" "list != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 556 | assert(list->is_nonempty(), "invariant")do { if (!(list->is_nonempty())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 556, "assert(" "list->is_nonempty()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 557 | size_t count = 0; | |||
| 558 | do { | |||
| 559 | BufferPtr full = list->remove(); | |||
| 560 | if (full == NULL__null) break; | |||
| 561 | assert(full->retired(), "invariant")do { if (!(full->retired())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 561, "assert(" "full->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 562 | processor.process(full); | |||
| 563 | // at this point, the buffer is already live or destroyed | |||
| 564 | ++count; | |||
| 565 | } while (list->is_nonempty()); | |||
| 566 | return count; | |||
| 567 | } | |||
| 568 | ||||
| 569 | static void log(size_t count, size_t amount, bool clear = false) { | |||
| 570 | if (log_is_enabled(Debug, jfr, system)(LogImpl<(LogTag::_jfr), (LogTag::_system), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Debug))) { | |||
| 571 | if (count > 0) { | |||
| 572 | log_debug(jfr, system)(!(LogImpl<(LogTag::_jfr), (LogTag::_system), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG) >::is_level(LogLevel::Debug))) ? (void)0 : LogImpl<(LogTag ::_jfr), (LogTag::_system), (LogTag::__NO_TAG), (LogTag::__NO_TAG ), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>::write<LogLevel ::Debug>("%s " SIZE_FORMAT"%" "l" "u" " full buffer(s) of " SIZE_FORMAT"%" "l" "u"" B of data%s", | |||
| 573 | clear ? "Discarded" : "Wrote", count, amount, clear ? "." : " to chunk."); | |||
| 574 | } | |||
| 575 | } | |||
| 576 | } | |||
| 577 | ||||
| 578 | typedef ReleaseOp<JfrThreadLocalMspace> ReleaseFullOperation; | |||
| 579 | typedef CompositeOperation<MutexedWriteOperation, ReleaseFullOperation> WriteFullOperation; | |||
| 580 | ||||
| 581 | // full writer | |||
| 582 | // Assumption is retired only; exclusive access | |||
| 583 | // MutexedWriter -> ReleaseOp | |||
| 584 | // | |||
| 585 | size_t JfrStorage::write_full() { | |||
| 586 | assert(_chunkwriter.is_valid(), "invariant")do { if (!(_chunkwriter.is_valid())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrStorage.cpp" , 586, "assert(" "_chunkwriter.is_valid()" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 587 | if (_full_list->is_empty()) { | |||
| 588 | return 0; | |||
| 589 | } | |||
| 590 | WriteOperation wo(_chunkwriter); | |||
| 591 | MutexedWriteOperation writer(wo); // a retired buffer implies mutexed access | |||
| 592 | ReleaseFullOperation rfo(_thread_local_mspace); | |||
| 593 | WriteFullOperation wfo(&writer, &rfo); | |||
| 594 | const size_t count = process_full(wfo, _full_list, control()); | |||
| 595 | if (count != 0) { | |||
| 596 | log(count, writer.size()); | |||
| 597 | } | |||
| 598 | return count; | |||
| 599 | } | |||
| 600 | ||||
| 601 | size_t JfrStorage::clear_full() { | |||
| 602 | if (_full_list->is_empty()) { | |||
| 603 | return 0; | |||
| 604 | } | |||
| 605 | DiscardOperation discarder(mutexed); // a retired buffer implies mutexed access | |||
| 606 | const size_t count = process_full(discarder, _full_list, control()); | |||
| 607 | if (count != 0) { | |||
| 608 | log(count, discarder.size()); | |||
| 609 | } | |||
| 610 | return count; | |||
| 611 | } |
| 1 | /* |
| 2 | * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP |
| 26 | #define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP |
| 27 | |
| 28 | #include "jfr/recorder/storage/jfrMemorySpace.hpp" |
| 29 | |
| 30 | #include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp" |
| 31 | #include "runtime/atomic.hpp" |
| 32 | #include "runtime/os.hpp" |
| 33 | |
| 34 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 35 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>:: |
| 36 | JfrMemorySpace(size_t min_element_size, size_t free_list_cache_count_limit, Client* client) : |
| 37 | _free_list(), |
| 38 | _live_list_epoch_0(), |
| 39 | _live_list_epoch_1(), |
| 40 | _client(client), |
| 41 | _min_element_size(min_element_size), |
| 42 | _free_list_cache_count_limit(free_list_cache_count_limit), |
| 43 | _free_list_cache_count(0) {} |
| 44 | |
| 45 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 46 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::~JfrMemorySpace() { |
| 47 | while (_live_list_epoch_0.is_nonempty()) { |
| 48 | deallocate(_live_list_epoch_0.remove()); |
| 49 | } |
| 50 | while (_live_list_epoch_1.is_nonempty()) { |
| 51 | deallocate(_live_list_epoch_1.remove()); |
| 52 | } |
| 53 | while (_free_list.is_nonempty()) { |
| 54 | deallocate(_free_list.remove()); |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 59 | bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::initialize(size_t cache_prealloc_count, bool prealloc_to_free_list) { |
| 60 | if (!(_free_list.initialize() && _live_list_epoch_0.initialize() && _live_list_epoch_1.initialize())) { |
| 61 | return false; |
| 62 | } |
| 63 | // pre-allocate elements to be cached in the requested list |
| 64 | for (size_t i = 0; i < cache_prealloc_count; ++i) { |
| 65 | NodePtr const node = allocate(_min_element_size); |
| 66 | if (node == NULL__null) { |
| 67 | return false; |
| 68 | } |
| 69 | if (prealloc_to_free_list) { |
| 70 | add_to_free_list(node); |
| 71 | } else { |
| 72 | add_to_live_list(node); |
| 73 | } |
| 74 | } |
| 75 | return true; |
| 76 | } |
| 77 | |
| 78 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 79 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::should_populate_free_list_cache() const { |
| 80 | return !is_free_list_cache_limited() || _free_list_cache_count < _free_list_cache_count_limit; |
| 81 | } |
| 82 | |
| 83 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 84 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::is_free_list_cache_limited() const { |
| 85 | return _free_list_cache_count_limit != JFR_MSPACE_UNLIMITED_CACHE_SIZE; |
| 86 | } |
| 87 | |
| 88 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 89 | inline size_t JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::min_element_size() const { |
| 90 | return _min_element_size; |
| 91 | } |
| 92 | |
| 93 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 94 | inline FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() { |
| 95 | return _free_list; |
| 96 | } |
| 97 | |
| 98 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 99 | inline const FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() const { |
| 100 | return _free_list; |
| 101 | } |
| 102 | |
| 103 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 104 | inline FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) { |
| 105 | if (epoch_aware) { |
| 106 | return previous_epoch ? previous_epoch_list() : current_epoch_list(); |
| 107 | } |
| 108 | return _live_list_epoch_0; |
| 109 | } |
| 110 | |
| 111 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 112 | inline const FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) const { |
| 113 | if (epoch_aware) { |
| 114 | return previous_epoch ? previous_epoch_list() : current_epoch_list(); |
| 115 | } |
| 116 | return _live_list_epoch_0; |
| 117 | } |
| 118 | |
| 119 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 120 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_empty() const { |
| 121 | return _free_list.is_empty(); |
| 122 | } |
| 123 | |
| 124 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 125 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_nonempty() const { |
| 126 | return !free_list_is_empty(); |
| 127 | } |
| 128 | |
| 129 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 130 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_empty(bool previous_epoch) const { |
| 131 | return live_list().is_empty(); |
| 132 | } |
| 133 | |
| 134 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 135 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_nonempty(bool previous_epoch) const { |
| 136 | return live_list().is_nonempty(); |
| 137 | } |
| 138 | |
| 139 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 140 | bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_free_list(const typename FreeListType::Node* node) const { |
| 141 | return _free_list.in_list(node); |
| 142 | } |
| 143 | |
| 144 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 145 | inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 146 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) const { |
| 147 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 147, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 148 | return epoch == 0 ? _live_list_epoch_0 : _live_list_epoch_1; |
| 149 | } |
| 150 | |
| 151 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 152 | inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 153 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) { |
| 154 | return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>( |
| 155 | const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->epoch_list_selector(epoch)); |
| 156 | } |
| 157 | |
| 158 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 159 | inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 160 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() const { |
| 161 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 161, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 162 | return epoch_list_selector(JfrTraceIdEpoch::current()); |
| 163 | } |
| 164 | |
| 165 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 166 | inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 167 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() { |
| 168 | return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>( |
| 169 | const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->current_epoch_list()); |
| 170 | } |
| 171 | |
| 172 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 173 | inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 174 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() const { |
| 175 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 175, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 176 | return epoch_list_selector(JfrTraceIdEpoch::previous()); |
| 177 | } |
| 178 | |
| 179 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 180 | inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList& |
| 181 | JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() { |
| 182 | return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>( |
| 183 | const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->previous_epoch_list()); |
| 184 | } |
| 185 | |
| 186 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 187 | bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_live_list(const typename FreeListType::Node* node, bool previous_epoch) const { |
| 188 | return live_list(previous_epoch).in_list(node); |
| 189 | } |
| 190 | |
| 191 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 192 | inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_current_epoch_list(const typename FreeListType::Node* node) const { |
| 193 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 193, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 194 | return current_epoch_list().in_list(node); |
| 195 | } |
| 196 | |
| 197 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 198 | inline bool JfrMemorySpace< Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_previous_epoch_list(const typename FreeListType::Node* node) const { |
| 199 | assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 199, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint (); } } while (0); |
| 200 | return previous_epoch_list().in_list(node); |
| 201 | } |
| 202 | |
| 203 | // allocations are even multiples of the mspace min size |
| 204 | static inline size_t align_allocation_size(size_t requested_size, size_t min_element_size) { |
| 205 | u8 alloc_size_bytes = min_element_size; |
| 206 | while (requested_size > alloc_size_bytes) { |
| 207 | alloc_size_bytes <<= 1; |
| 208 | } |
| 209 | return (size_t)alloc_size_bytes; |
| 210 | } |
| 211 | |
| 212 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 213 | inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::allocate(size_t size) { |
| 214 | const size_t aligned_size_bytes = align_allocation_size(size, _min_element_size); |
| 215 | void* const allocation = JfrCHeapObj::new_array<u1>(aligned_size_bytes + sizeof(Node)); |
| 216 | if (allocation == NULL__null) { |
| 217 | return NULL__null; |
| 218 | } |
| 219 | NodePtr node = new (allocation) Node(); |
| 220 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 220, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 221 | if (!node->initialize(sizeof(Node), aligned_size_bytes)) { |
| 222 | JfrCHeapObj::free(node, aligned_size_bytes + sizeof(Node)); |
| 223 | return NULL__null; |
| 224 | } |
| 225 | return node; |
| 226 | } |
| 227 | |
| 228 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 229 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::deallocate(typename FreeListType::NodePtr node) { |
| 230 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 230, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 231 | assert(!in_free_list(node), "invariant")do { if (!(!in_free_list(node))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 231, "assert(" "!in_free_list(node)" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 232 | assert(!_live_list_epoch_0.in_list(node), "invariant")do { if (!(!_live_list_epoch_0.in_list(node))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 232, "assert(" "!_live_list_epoch_0.in_list(node)" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 233 | assert(!_live_list_epoch_1.in_list(node), "invariant")do { if (!(!_live_list_epoch_1.in_list(node))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 233, "assert(" "!_live_list_epoch_1.in_list(node)" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 234 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 234, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 235 | JfrCHeapObj::free(node, node->total_size()); |
| 236 | } |
| 237 | |
| 238 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 239 | inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::acquire(size_t size, bool free_list, Thread* thread, bool previous_epoch) { |
| 240 | return RetrievalPolicy<JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware> >::acquire(this, free_list, thread, size, previous_epoch); |
| 241 | } |
| 242 | |
| 243 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 244 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::release(typename FreeListType::NodePtr node) { |
| 245 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 245, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 246 | if (node->transient()) { |
| 247 | deallocate(node); |
| 248 | return; |
| 249 | } |
| 250 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 250, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 251 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 251, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 252 | assert(node->identity() == NULL, "invariant")do { if (!(node->identity() == __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 252, "assert(" "node->identity() == __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 253 | if (should_populate_free_list_cache()) { |
| 254 | add_to_free_list(node); |
| 255 | } else { |
| 256 | deallocate(node); |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 261 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_free_list(typename FreeListType::NodePtr node) { |
| 262 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 262, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 263 | _free_list.add(node); |
| 264 | if (is_free_list_cache_limited()) { |
| 265 | Atomic::inc(&_free_list_cache_count); |
| 266 | } |
| 267 | } |
| 268 | |
| 269 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 270 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_live_list(typename FreeListType::NodePtr node, bool previous_epoch) { |
| 271 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 271, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 272 | live_list(previous_epoch).add(node); |
| 273 | } |
| 274 | |
| 275 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 276 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::decrement_free_list_count() { |
| 277 | if (is_free_list_cache_limited()) { |
| 278 | Atomic::dec(&_free_list_cache_count); |
| 279 | } |
| 280 | } |
| 281 | |
| 282 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 283 | template <typename Callback> |
| 284 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_free_list(Callback& callback) { |
| 285 | return _free_list.iterate(callback); |
| 286 | } |
| 287 | |
| 288 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 289 | template <typename Callback> |
| 290 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_live_list(Callback& callback, bool previous_epoch) { |
| 291 | if (epoch_aware) { |
| 292 | live_list(previous_epoch).iterate(callback); |
| 293 | return; |
| 294 | } |
| 295 | _live_list_epoch_0.iterate(callback); |
| 296 | } |
| 297 | |
| 298 | template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware> |
| 299 | inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::register_full(typename FreeListType::NodePtr node, Thread* thread) { |
| 300 | _client->register_full(node, thread); |
| 301 | } |
| 302 | |
| 303 | template <typename Mspace, typename Client> |
| 304 | static inline Mspace* create_mspace(size_t min_element_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count, bool prealloc_to_free_list, Client* cb) { |
| 305 | Mspace* const mspace = new Mspace(min_element_size, free_list_cache_count_limit, cb); |
| 306 | if (mspace != NULL__null) { |
| 307 | mspace->initialize(cache_prealloc_count, prealloc_to_free_list); |
| 308 | } |
| 309 | return mspace; |
| 310 | } |
| 311 | |
| 312 | template <typename Mspace> |
| 313 | inline typename Mspace::NodePtr mspace_allocate(size_t size, Mspace* mspace) { |
| 314 | return mspace->allocate(size); |
| 315 | } |
| 316 | |
| 317 | template <typename Mspace> |
| 318 | inline typename Mspace::NodePtr mspace_allocate_acquired(size_t size, Mspace* mspace, Thread* thread) { |
| 319 | typename Mspace::NodePtr node = mspace_allocate(size, mspace); |
| 320 | if (node == NULL__null) return NULL__null; |
| 321 | node->set_identity(thread); |
| 322 | return node; |
| 323 | } |
| 324 | |
| 325 | template <typename Mspace> |
| 326 | inline typename Mspace::NodePtr mspace_allocate_transient(size_t size, Mspace* mspace, Thread* thread) { |
| 327 | typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread); |
| 328 | if (node == NULL__null) return NULL__null; |
| 329 | assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 329, "assert(" "node->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 330 | node->set_transient(); |
| 331 | return node; |
| 332 | } |
| 333 | |
| 334 | template <typename Mspace> |
| 335 | inline typename Mspace::NodePtr mspace_allocate_transient_lease(size_t size, Mspace* mspace, Thread* thread) { |
| 336 | typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread); |
| 337 | if (node == NULL__null) return NULL__null; |
| 338 | assert(node->transient(), "invariant")do { if (!(node->transient())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 338, "assert(" "node->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 339 | node->set_lease(); |
| 340 | return node; |
| 341 | } |
| 342 | |
| 343 | template <typename Mspace> |
| 344 | inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_free(size_t size, Mspace* mspace, Thread* thread) { |
| 345 | typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread); |
| 346 | if (node == NULL__null) return NULL__null; |
| 347 | assert(node->lease(), "invariant")do { if (!(node->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 347, "assert(" "node->lease()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 348 | mspace->add_to_free_list(node); |
| 349 | return node; |
| 350 | } |
| 351 | |
| 352 | template <typename Mspace> |
| 353 | inline typename Mspace::NodePtr mspace_acquire_free(size_t size, Mspace* mspace, Thread* thread) { |
| 354 | return mspace->acquire(size, true, thread); |
| 355 | } |
| 356 | |
| 357 | template <typename Mspace> |
| 358 | inline typename Mspace::NodePtr mspace_acquire_free_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) { |
| 359 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 359, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 360 | for (size_t i = 0; i < retry_count; ++i) { |
| 361 | typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread); |
| 362 | if (node != NULL__null) { |
| 363 | return node; |
| 364 | } |
| 365 | } |
| 366 | return NULL__null; |
| 367 | } |
| 368 | |
| 369 | template <typename Mspace> |
| 370 | inline typename Mspace::NodePtr mspace_allocate_to_live_list(size_t size, Mspace* mspace, Thread* thread) { |
| 371 | typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread); |
| 372 | if (node == NULL__null) return NULL__null; |
| 373 | assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 373, "assert(" "node->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 374 | mspace->add_to_live_list(node); |
| 375 | return node; |
| 376 | } |
| 377 | |
| 378 | template <typename Mspace> |
| 379 | inline typename Mspace::NodePtr mspace_allocate_transient_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 380 | typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread); |
| 381 | if (node == NULL__null) return NULL__null; |
| 382 | assert(node->transient(), "invariant")do { if (!(node->transient())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 382, "assert(" "node->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 383 | mspace->add_to_live_list(node, previous_epoch); |
| 384 | return node; |
| 385 | } |
| 386 | |
| 387 | template <typename Mspace> |
| 388 | inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 389 | typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread); |
| 390 | if (node == NULL__null) return NULL__null; |
| 391 | assert(node->lease(), "invariant")do { if (!(node->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 391, "assert(" "node->lease()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 392 | mspace->add_to_live_list(node, previous_epoch); |
| 393 | return node; |
| 394 | } |
| 395 | |
| 396 | template <typename Mspace> |
| 397 | inline typename Mspace::NodePtr mspace_acquire_free_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 398 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 398, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 399 | typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread); |
| 400 | if (node == NULL__null) { |
| 401 | return NULL__null; |
| 402 | } |
| 403 | assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 403, "assert(" "node->acquired_by_self()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 404 | mspace->add_to_live_list(node, previous_epoch); |
| 405 | return node; |
| 406 | } |
| 407 | |
| 408 | template <typename Mspace> |
| 409 | inline typename Mspace::NodePtr mspace_acquire_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 410 | if (size <= mspace->min_element_size()) { |
| 411 | typename Mspace::NodePtr node = mspace_acquire_free_to_live_list(size, mspace, thread, previous_epoch); |
| 412 | if (node != NULL__null) { |
| 413 | return node; |
| 414 | } |
| 415 | } |
| 416 | return mspace_allocate_to_live_list(size, mspace, thread); |
| 417 | } |
| 418 | |
| 419 | template <typename Mspace> |
| 420 | inline typename Mspace::NodePtr mspace_acquire_live(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) { |
| 421 | return mspace->acquire(size, false, thread, previous_epoch); |
| 422 | } |
| 423 | |
| 424 | template <typename Mspace> |
| 425 | inline typename Mspace::NodePtr mspace_acquire_live_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) { |
| 426 | assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 426, "assert(" "size <= mspace->min_element_size()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 427 | for (size_t i = 0; i < retry_count; ++i) { |
| 428 | typename Mspace::NodePtr const node = mspace_acquire_live(size, mspace, thread, previous_epoch); |
| 429 | if (node != NULL__null) { |
| 430 | return node; |
| 431 | } |
| 432 | } |
| 433 | return NULL__null; |
| 434 | } |
| 435 | |
| 436 | template <typename Mspace> |
| 437 | inline typename Mspace::NodePtr mspace_acquire_lease_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) { |
| 438 | typename Mspace::NodePtr node = mspace_acquire_live_with_retry(size, mspace, retry_count, thread, previous_epoch); |
| 439 | if (node != NULL__null) { |
| 440 | node->set_lease(); |
| 441 | } |
| 442 | return node; |
| 443 | } |
| 444 | |
| 445 | template <typename Mspace> |
| 446 | inline void mspace_release(typename Mspace::NodePtr node, Mspace* mspace) { |
| 447 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 447, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 448 | assert(node->unflushed_size() == 0, "invariant")do { if (!(node->unflushed_size() == 0)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 448, "assert(" "node->unflushed_size() == 0" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 449 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 449, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 450 | mspace->release(node); |
| 451 | } |
| 452 | |
| 453 | template <typename Callback, typename Mspace> |
| 454 | inline void process_live_list(Callback& callback, Mspace* mspace, bool previous_epoch = false) { |
| 455 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 455, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 456 | mspace->iterate_live_list(callback, previous_epoch); |
| 457 | } |
| 458 | |
| 459 | template <typename Callback, typename Mspace> |
| 460 | inline void process_free_list(Callback& callback, Mspace* mspace) { |
| 461 | assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 461, "assert(" "mspace != __null" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 462 | assert(mspace->free_list_is_nonempty(), "invariant")do { if (!(mspace->free_list_is_nonempty())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 462, "assert(" "mspace->free_list_is_nonempty()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 463 | mspace->iterate_free_list(callback); |
| 464 | } |
| 465 | |
| 466 | template <typename Mspace> |
| 467 | class ReleaseOp : public StackObj { |
| 468 | private: |
| 469 | Mspace* _mspace; |
| 470 | bool _previous_epoch; |
| 471 | public: |
| 472 | typedef typename Mspace::Node Node; |
| 473 | ReleaseOp(Mspace* mspace) : _mspace(mspace) {} |
| 474 | bool process(typename Mspace::NodePtr node); |
| 475 | size_t processed() const { return 0; } |
| 476 | }; |
| 477 | |
| 478 | template <typename Mspace> |
| 479 | inline bool ReleaseOp<Mspace>::process(typename Mspace::NodePtr node) { |
| 480 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 480, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 481 | // assumes some means of exclusive access to the node |
| 482 | if (node->transient()) { |
| 483 | // make sure the transient node is already detached |
| 484 | _mspace->release(node); |
| 485 | return true; |
| 486 | } |
| 487 | node->reinitialize(); |
| 488 | if (node->identity() != NULL__null) { |
| 489 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 489, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 490 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 490, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 491 | node->release(); // publish |
| 492 | } |
| 493 | return true; |
| 494 | } |
| 495 | |
| 496 | template <typename Mspace, typename List> |
| 497 | class ReleaseWithExcisionOp : public ReleaseOp<Mspace> { |
| 498 | private: |
| 499 | List& _list; |
| 500 | typename List::NodePtr _prev; |
| 501 | size_t _count; |
| 502 | size_t _amount; |
| 503 | public: |
| 504 | ReleaseWithExcisionOp(Mspace* mspace, List& list) : |
| 505 | ReleaseOp<Mspace>(mspace), _list(list), _prev(NULL__null), _count(0), _amount(0) {} |
| 506 | bool process(typename List::NodePtr node); |
| 507 | size_t processed() const { return _count; } |
| 508 | size_t amount() const { return _amount; } |
| 509 | }; |
| 510 | |
| 511 | template <typename Mspace, typename List> |
| 512 | inline bool ReleaseWithExcisionOp<Mspace, List>::process(typename List::NodePtr node) { |
| 513 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 513, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 514 | if (node->transient()) { |
| 515 | _prev = _list.excise(_prev, node); |
| 516 | } else { |
| 517 | _prev = node; |
| 518 | } |
| 519 | return ReleaseOp<Mspace>::process(node); |
| 520 | } |
| 521 | |
| 522 | template <typename Mspace, typename List> |
| 523 | class ScavengingReleaseOp : public StackObj { |
| 524 | protected: |
| 525 | Mspace* _mspace; |
| 526 | List& _list; |
| 527 | typename List::NodePtr _prev; |
| 528 | size_t _count; |
| 529 | size_t _amount; |
| 530 | bool excise_with_release(typename List::NodePtr node); |
| 531 | public: |
| 532 | typedef typename List::Node Node; |
| 533 | ScavengingReleaseOp(Mspace* mspace, List& list) : |
| 534 | _mspace(mspace), _list(list), _prev(NULL__null), _count(0), _amount(0) {} |
| 535 | bool process(typename List::NodePtr node); |
| 536 | size_t processed() const { return _count; } |
| 537 | size_t amount() const { return _amount; } |
| 538 | }; |
| 539 | |
| 540 | template <typename Mspace, typename List> |
| 541 | inline bool ScavengingReleaseOp<Mspace, List>::process(typename List::NodePtr node) { |
| 542 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 542, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 543 | assert(!node->transient(), "invariant")do { if (!(!node->transient())) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 543, "assert(" "!node->transient()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 544 | if (node->retired()) { |
| 545 | return excise_with_release(node); |
| 546 | } |
| 547 | _prev = node; |
| 548 | return true; |
| 549 | } |
| 550 | |
| 551 | template <typename Mspace, typename List> |
| 552 | inline bool ScavengingReleaseOp<Mspace, List>::excise_with_release(typename List::NodePtr node) { |
| 553 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 553, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 554 | assert(node->retired(), "invariant")do { if (!(node->retired())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 554, "assert(" "node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 555 | _prev = _list.excise(_prev, node); |
| 556 | if (node->transient()) { |
| 557 | _mspace->deallocate(node); |
| 558 | return true; |
| 559 | } |
| 560 | assert(node->identity() != NULL, "invariant")do { if (!(node->identity() != __null)) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 560, "assert(" "node->identity() != __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 561 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 561, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 562 | assert(!node->lease(), "invariant")do { if (!(!node->lease())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 562, "assert(" "!node->lease()" ") failed", "invariant") ; ::breakpoint(); } } while (0); |
| 563 | assert(!node->excluded(), "invariant")do { if (!(!node->excluded())) { (*g_assert_poison) = 'X'; ; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 563, "assert(" "!node->excluded()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 564 | ++_count; |
| 565 | _amount += node->total_size(); |
| 566 | node->clear_retired(); |
| 567 | node->release(); |
| 568 | mspace_release(node, _mspace); |
| 569 | return true; |
| 570 | } |
| 571 | |
| 572 | template <typename Mspace, typename FromList> |
| 573 | class ReleaseRetiredOp : public StackObj { |
| 574 | private: |
| 575 | Mspace* _mspace; |
| 576 | FromList& _list; |
| 577 | typename Mspace::NodePtr _prev; |
| 578 | public: |
| 579 | typedef typename Mspace::Node Node; |
| 580 | ReleaseRetiredOp(Mspace* mspace, FromList& list) : |
| 581 | _mspace(mspace), _list(list), _prev(NULL__null) {} |
| 582 | bool process(Node* node); |
| 583 | }; |
| 584 | |
| 585 | template <typename Mspace, typename FromList> |
| 586 | inline bool ReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) { |
| 587 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 587, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 588 | if (node->retired()) { |
| 589 | _prev = _list.excise(_prev, node); |
| 590 | node->reinitialize(); |
| 591 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 591, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 592 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 592, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 593 | node->release(); |
| 594 | mspace_release(node, _mspace); |
| 595 | } else { |
| 596 | _prev = node; |
| 597 | } |
| 598 | return true; |
| 599 | } |
| 600 | |
| 601 | template <typename Mspace, typename FromList> |
| 602 | class ReinitializeAllReleaseRetiredOp : public StackObj { |
| 603 | private: |
| 604 | Mspace* _mspace; |
| 605 | FromList& _list; |
| 606 | typename Mspace::NodePtr _prev; |
| 607 | public: |
| 608 | typedef typename Mspace::Node Node; |
| 609 | ReinitializeAllReleaseRetiredOp(Mspace* mspace, FromList& list) : |
| 610 | _mspace(mspace), _list(list), _prev(NULL__null) {} |
| 611 | bool process(Node* node); |
| 612 | }; |
| 613 | |
| 614 | template <typename Mspace, typename FromList> |
| 615 | inline bool ReinitializeAllReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) { |
| 616 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 616, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 617 | // assumes some means of exclusive access to node |
| 618 | const bool retired = node->retired(); |
| 619 | node->reinitialize(); |
| 620 | assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 620, "assert(" "node->empty()" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 621 | assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 621, "assert(" "!node->retired()" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 622 | if (retired) { |
| 623 | _prev = _list.excise(_prev, node); |
| 624 | node->release(); |
| 625 | mspace_release(node, _mspace); |
| 626 | } else { |
| 627 | _prev = node; |
| 628 | } |
| 629 | return true; |
| 630 | } |
| 631 | |
| 632 | #ifdef ASSERT1 |
| 633 | template <typename Node> |
| 634 | inline void assert_migration_state(const Node* old, const Node* new_node, size_t used, size_t requested) { |
| 635 | assert(old != NULL, "invariant")do { if (!(old != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 635, "assert(" "old != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 636 | assert(new_node != NULL, "invariant")do { if (!(new_node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 636, "assert(" "new_node != __null" ") failed", "invariant" ); ::breakpoint(); } } while (0); |
| 637 | assert(old->pos() >= old->start(), "invariant")do { if (!(old->pos() >= old->start())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 637, "assert(" "old->pos() >= old->start()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 638 | assert(old->pos() + used <= old->end(), "invariant")do { if (!(old->pos() + used <= old->end())) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 638, "assert(" "old->pos() + used <= old->end()" ") failed" , "invariant"); ::breakpoint(); } } while (0); |
| 639 | assert(new_node->free_size() >= (used + requested), "invariant")do { if (!(new_node->free_size() >= (used + requested)) ) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp" , 639, "assert(" "new_node->free_size() >= (used + requested)" ") failed", "invariant"); ::breakpoint(); } } while (0); |
| 640 | } |
| 641 | #endif // ASSERT |
| 642 | |
| 643 | template <typename Node> |
| 644 | inline void migrate_outstanding_writes(const Node* old, Node* new_node, size_t used, size_t requested) { |
| 645 | DEBUG_ONLY(assert_migration_state(old, new_node, used, requested);)assert_migration_state(old, new_node, used, requested); |
| 646 | if (used > 0) { |
| 647 | memcpy(new_node->pos(), old->pos(), used); |
| 648 | } |
| 649 | } |
| 650 | |
| 651 | #endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP |
| 1 | /* |
| 2 | * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved. |
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 | * |
| 5 | * This code is free software; you can redistribute it and/or modify it |
| 6 | * under the terms of the GNU General Public License version 2 only, as |
| 7 | * published by the Free Software Foundation. |
| 8 | * |
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT |
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License |
| 12 | * version 2 for more details (a copy is included in the LICENSE file that |
| 13 | * accompanied this code). |
| 14 | * |
| 15 | * You should have received a copy of the GNU General Public License version |
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, |
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. |
| 18 | * |
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
| 20 | * or visit www.oracle.com if you need additional information or have any |
| 21 | * questions. |
| 22 | * |
| 23 | */ |
| 24 | |
| 25 | #ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP |
| 26 | #define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP |
| 27 | |
| 28 | #include "jfr/utilities/jfrConcurrentQueue.hpp" |
| 29 | |
| 30 | #include "jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" |
| 31 | #include "jfr/utilities/jfrVersionSystem.inline.hpp" |
| 32 | |
| 33 | template <typename NodeType, typename AllocPolicy> |
| 34 | JfrConcurrentQueue<NodeType, AllocPolicy>::JfrConcurrentQueue() : _list(NULL__null), _head(), _last(), _tail(), _version_system() { |
| 35 | _head._next = const_cast<NodePtr>(&_tail); |
| 36 | _last._next = const_cast<NodePtr>(&_tail); |
| 37 | } |
| 38 | |
| 39 | template <typename NodeType, typename AllocPolicy> |
| 40 | bool JfrConcurrentQueue<NodeType, AllocPolicy>::initialize() { |
| 41 | assert(_list == NULL, "invariant")do { if (!(_list == __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp" , 41, "assert(" "_list == __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 42 | _list = new JfrConcurrentLinkedListHost<JfrConcurrentQueue<NodeType, AllocPolicy>, HeadNode, AllocPolicy>(this); |
| 43 | return _list != NULL__null && _list->initialize(); |
| 44 | } |
| 45 | |
| 46 | template <typename NodeType, typename AllocPolicy> |
| 47 | inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_empty() const { |
| 48 | return Atomic::load_acquire(&_head._next) == &_tail; |
| 49 | } |
| 50 | |
| 51 | template <typename NodeType, typename AllocPolicy> |
| 52 | inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_nonempty() const { |
| 53 | return !is_empty(); |
| 54 | } |
| 55 | |
| 56 | template <typename NodeType, typename AllocPolicy> |
| 57 | void JfrConcurrentQueue<NodeType, AllocPolicy>::add(typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr node) { |
| 58 | _list->insert_tail(node, &_head, &_last, &_tail); |
| 59 | } |
| 60 | |
| 61 | template <typename NodeType, typename AllocPolicy> |
| 62 | typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr JfrConcurrentQueue<NodeType, AllocPolicy>::remove() { |
| 63 | return _list->remove(&_head, &_tail, &_last, false); |
| 64 | } |
| 65 | |
| 66 | template <typename NodeType, typename AllocPolicy> |
| 67 | template <typename Callback> |
| 68 | void JfrConcurrentQueue<NodeType, AllocPolicy>::iterate(Callback& cb) { |
| 69 | _list->iterate(&_head, &_tail, cb); |
| 70 | } |
| 71 | |
| 72 | template <typename NodeType, typename AllocPolicy> |
| 73 | inline JfrVersionSystem::Handle JfrConcurrentQueue<NodeType, AllocPolicy>::get_version_handle() { |
| 74 | return _version_system.get(); |
| 75 | } |
| 76 | |
| 77 | template <typename NodeType, typename AllocPolicy> |
| 78 | bool JfrConcurrentQueue<NodeType, AllocPolicy>::in_list(const NodeType* node) const { |
| 79 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp" , 79, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); |
| 80 | return _list->in_list(node, const_cast<NodePtr>(&_head), &_tail); |
| 81 | } |
| 82 | |
| 83 | #endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP |
| 1 | /* | |||
| 2 | * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved. | |||
| 3 | * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |||
| 4 | * | |||
| 5 | * This code is free software; you can redistribute it and/or modify it | |||
| 6 | * under the terms of the GNU General Public License version 2 only, as | |||
| 7 | * published by the Free Software Foundation. | |||
| 8 | * | |||
| 9 | * This code is distributed in the hope that it will be useful, but WITHOUT | |||
| 10 | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |||
| 11 | * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |||
| 12 | * version 2 for more details (a copy is included in the LICENSE file that | |||
| 13 | * accompanied this code). | |||
| 14 | * | |||
| 15 | * You should have received a copy of the GNU General Public License version | |||
| 16 | * 2 along with this work; if not, write to the Free Software Foundation, | |||
| 17 | * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |||
| 18 | * | |||
| 19 | * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |||
| 20 | * or visit www.oracle.com if you need additional information or have any | |||
| 21 | * questions. | |||
| 22 | * | |||
| 23 | */ | |||
| 24 | ||||
| 25 | #ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP | |||
| 26 | #define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP | |||
| 27 | ||||
| 28 | #include "jfr/utilities/jfrConcurrentLinkedListHost.hpp" | |||
| 29 | ||||
| 30 | #include "jfr/utilities/jfrRelation.hpp" | |||
| 31 | #include "jfr/utilities/jfrTypes.hpp" | |||
| 32 | #include "runtime/atomic.hpp" | |||
| 33 | #include "utilities/globalDefinitions.hpp" | |||
| 34 | ||||
| 35 | /* | |||
| 36 | * The removal marker (i.e. the excision bit) is represented by '( )' as part of state description comments: | |||
| 37 | * "node --> next" becomes "(node) --> next", when node is logically deleted. | |||
| 38 | */ | |||
| 39 | template <typename Node> | |||
| 40 | inline Node* mark_for_removal(Node* node) { | |||
| 41 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 41, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 42 | const Node* next = node->_next; | |||
| 43 | assert(next != NULL, "invariant")do { if (!(next != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 43, "assert(" "next != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 44 | Node* const unmasked_next = unmask(next); | |||
| 45 | return next == unmasked_next && cas(&node->_next, unmasked_next, set_excision_bit(unmasked_next)) ? unmasked_next : NULL__null; | |||
| 46 | } | |||
| 47 | ||||
| 48 | /* | |||
| 49 | * The insertion marker (i.e. the insertion bit) is represented by '[ ]' as part of state description comments: | |||
| 50 | * "node --> next" becomes "[node] --> next", in an attempt to convey the node as exlusively reserved. | |||
| 51 | */ | |||
| 52 | template <typename Node> | |||
| 53 | inline bool mark_for_insertion(Node* node, const Node* tail) { | |||
| 54 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 54, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 55 | return node->_next == tail && cas(&node->_next, const_cast<Node*>(tail), set_insertion_bit(tail)); | |||
| 56 | } | |||
| 57 | ||||
| 58 | /* | |||
| 59 | * Find a predecessor and successor node pair where successor covers predecessor (adjacency). | |||
| 60 | */ | |||
| 61 | template <typename Node, typename VersionHandle, template <typename> class SearchPolicy> | |||
| 62 | Node* find_adjacent(Node* head, const Node* tail, Node** predecessor, VersionHandle& version_handle, SearchPolicy<Node>& predicate) { | |||
| 63 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 63, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 64 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 64, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 65 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 65, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 66 | Node* predecessor_next = NULL__null; | |||
| 67 | while (true) { | |||
| 68 | Node* current = head; | |||
| 69 | version_handle->checkout(); | |||
| 70 | Node* next = Atomic::load_acquire(¤t->_next); | |||
| 71 | do { | |||
| 72 | assert(next != NULL, "invariant")do { if (!(next != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 72, "assert(" "next != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 73 | Node* const unmasked_next = unmask(next); | |||
| 74 | // 1A: Locate the first node to keep as predecessor. | |||
| 75 | if (!is_marked_for_removal(next)) { | |||
| 76 | *predecessor = current; | |||
| 77 | predecessor_next = unmasked_next; | |||
| 78 | } | |||
| 79 | // 1B: Locate the next node to keep as successor. | |||
| 80 | current = unmasked_next; | |||
| 81 | if (current == tail) break; | |||
| 82 | next = current->_next; | |||
| 83 | } while (predicate(current, next)); | |||
| 84 | // current represents the successor node from here on out. | |||
| 85 | // 2: Check predecessor and successor node pair for adjacency. | |||
| 86 | if (predecessor_next == current) { | |||
| 87 | // Invariant: predecessor --> successor | |||
| 88 | return current; | |||
| 89 | } | |||
| 90 | // 3: Successor does not (yet) cover predecessor. | |||
| 91 | // Invariant: predecessor --> (logically excised nodes) --> successor | |||
| 92 | // Physically excise one or more logically excised nodes in-between. | |||
| 93 | if (cas(&(*predecessor)->_next, predecessor_next, current)) { | |||
| ||||
| 94 | // Invariant: predecessor --> successor | |||
| 95 | return current; | |||
| 96 | } | |||
| 97 | } | |||
| 98 | } | |||
| 99 | ||||
| 100 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 101 | JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::JfrConcurrentLinkedListHost(Client* client) : _client(client) {} | |||
| 102 | ||||
| 103 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 104 | bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::initialize() { | |||
| 105 | return true; | |||
| 106 | } | |||
| 107 | ||||
| 108 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 109 | void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_head(typename Client::Node* node, | |||
| 110 | typename Client::Node* head, | |||
| 111 | const typename Client::Node* tail) const { | |||
| 112 | Node* predecessor; | |||
| 113 | Node* successor; | |||
| 114 | HeadNode<Node> predicate(node); | |||
| 115 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 116 | while (true) { | |||
| 117 | // Find an adjacent predecessor and successor node pair. | |||
| 118 | successor = find_adjacent<Node, VersionHandle, HeadNode>(head, tail, &predecessor, version_handle, predicate); | |||
| 119 | // Invariant (adjacency): predecessor --> successor | |||
| 120 | // Invariant (optional: key-based total order): predecessor->key() < key && key <= successor->key(). | |||
| 121 | // We can now attempt to insert the new node in-between. | |||
| 122 | node->_next = successor; | |||
| 123 | if (cas(&predecessor->_next, successor, node)) { | |||
| 124 | // Invariant: predecessor --> node --> successor | |||
| 125 | // An insert to head is a benign modification and will not need to be committed to the version control system. | |||
| 126 | return; | |||
| 127 | } | |||
| 128 | } | |||
| 129 | } | |||
| 130 | ||||
| 131 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 132 | void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_tail(typename Client::Node* node, | |||
| 133 | typename Client::Node* head, | |||
| 134 | typename Client::Node* last, | |||
| 135 | const typename Client::Node* tail) const { | |||
| 136 | assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 136, "assert(" "node != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 137 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 137, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 138 | assert(last != NULL, "invarinat")do { if (!(last != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 138, "assert(" "last != __null" ") failed", "invarinat"); :: breakpoint(); } } while (0); | |||
| 139 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 139, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 140 | // Mark the new node to be inserted with the insertion marker already. | |||
| 141 | node->_next = set_insertion_bit(const_cast<NodePtr>(tail)); | |||
| 142 | // Invariant: [node]--> tail | |||
| 143 | assert(is_marked_for_insertion(node->_next), "invariant")do { if (!(is_marked_for_insertion(node->_next))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 143, "assert(" "is_marked_for_insertion(node->_next)" ") failed" , "invariant"); ::breakpoint(); } } while (0); | |||
| 144 | NodePtr predecessor; | |||
| 145 | LastNode<Node> predicate; | |||
| 146 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 147 | while (true) { | |||
| 148 | // Find an adjacent predecessor and successor node pair, where the successor == tail | |||
| 149 | const NodePtr successor = find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, predicate); | |||
| 150 | assert(successor == tail, "invariant")do { if (!(successor == tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 150, "assert(" "successor == tail" ") failed", "invariant") ; ::breakpoint(); } } while (0); | |||
| 151 | // Invariant: predecessor --> successor | |||
| 152 | // We first attempt to mark the predecessor node to signal our intent of performing an insertion. | |||
| 153 | if (mark_for_insertion(predecessor, tail)) { | |||
| 154 | break; | |||
| 155 | } | |||
| 156 | } | |||
| 157 | // Predecessor node is claimed for insertion. | |||
| 158 | // Invariant: [predecessor] --> tail | |||
| 159 | assert(is_marked_for_insertion(predecessor->_next), "invariant")do { if (!(is_marked_for_insertion(predecessor->_next))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 159, "assert(" "is_marked_for_insertion(predecessor->_next)" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 160 | assert(predecessor != head, "invariant")do { if (!(predecessor != head)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 160, "assert(" "predecessor != head" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 161 | if (Atomic::load_acquire(&last->_next) == predecessor) { | |||
| 162 | /* Even after we store the new node into the last->_next field, there is no race | |||
| 163 | because it is also marked with the insertion bit. */ | |||
| 164 | last->_next = node; | |||
| 165 | // Invariant: last --> [node] --> tail | |||
| 166 | OrderAccess::storestore(); | |||
| 167 | // Perform the link with the predecessor node, which by this store becomes visible for removal. | |||
| 168 | predecessor->_next = node; | |||
| 169 | // Invariant: predecessor --> [node] --> tail | |||
| 170 | } else { | |||
| 171 | assert(last == predecessor, "invariant")do { if (!(last == predecessor)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 171, "assert(" "last == predecessor" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 172 | last->_next = node; | |||
| 173 | // Invariant: last --> [node] --> tail | |||
| 174 | OrderAccess::storestore(); | |||
| 175 | /* This implies the list is logically empty from the removal perspective. | |||
| 176 | cas is not needed here because inserts must not come in from the head side | |||
| 177 | concurrently with inserts from tail which are currently blocked by us. | |||
| 178 | Invariant (logical): head --> tail. */ | |||
| 179 | head->_next = node; | |||
| 180 | // Invariant: head --> [node] --> tail | |||
| 181 | } | |||
| 182 | OrderAccess::storestore(); | |||
| 183 | // Publish the inserted node by removing the insertion marker. | |||
| 184 | node->_next = const_cast<NodePtr>(tail); | |||
| 185 | // Invariant: last --> node --> tail (possibly also head --> node --> tail) | |||
| 186 | } | |||
| 187 | ||||
| 188 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 189 | typename Client::Node* JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::remove(typename Client::Node* head, | |||
| 190 | const typename Client::Node* tail, | |||
| 191 | typename Client::Node* last /* NULL */, | |||
| 192 | bool insert_is_head /* true */) { | |||
| 193 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 193, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 194 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 194, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 195 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 195, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 196 | NodePtr predecessor; | |||
| 197 | NodePtr successor; | |||
| 198 | NodePtr successor_next; | |||
| 199 | SearchPolicy<Node> predicate; | |||
| 200 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 201 | while (true) { | |||
| 202 | // Find an adjacent predecessor and successor node pair. | |||
| 203 | successor = find_adjacent<Node, VersionHandle, SearchPolicy>(head, tail, &predecessor, version_handle, predicate); | |||
| 204 | if (successor == tail) { | |||
| 205 | return NULL__null; | |||
| 206 | } | |||
| 207 | // Invariant: predecessor --> successor | |||
| 208 | // Invariant (optional: key-based total order): predecessor->key() < key && key <= successor->key() | |||
| 209 | // It is the successor node that is to be removed. | |||
| 210 | // We first attempt to reserve (logically excise) the successor node. | |||
| 211 | successor_next = mark_for_removal(successor); | |||
| 212 | if (successor_next != NULL__null) { | |||
| 213 | break; | |||
| 214 | } | |||
| 215 | } | |||
| 216 | // Invariant: predecessor --> (successor) --> successor_next | |||
| 217 | // Successor node now logically excised. | |||
| 218 | assert(is_marked_for_removal(successor->_next), "invariant")do { if (!(is_marked_for_removal(successor->_next))) { (*g_assert_poison ) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 218, "assert(" "is_marked_for_removal(successor->_next)" ") failed", "invariant"); ::breakpoint(); } } while (0); | |||
| 219 | // Now attempt to physically excise the successor node. | |||
| 220 | // If the cas fails, we can optimize for the slow path if we know we are not performing | |||
| 221 | // insertions from the head. Then a failed cas results not from a new node being inserted, | |||
| 222 | // but only because another thread excised us already. | |||
| 223 | if (!cas(&predecessor->_next, successor, successor_next) && insert_is_head) { | |||
| 224 | // Physically excise using slow path, can be completed asynchronously by other threads. | |||
| 225 | Identity<Node> excise(successor); | |||
| 226 | find_adjacent<Node, VersionHandle, Identity>(head, tail, &predecessor, version_handle, excise); | |||
| 227 | } | |||
| 228 | if (last != NULL__null && Atomic::load_acquire(&last->_next) == successor) { | |||
| 229 | guarantee(!insert_is_head, "invariant")do { if (!(!insert_is_head)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 229, "guarantee(" "!insert_is_head" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 230 | guarantee(successor_next == tail, "invariant")do { if (!(successor_next == tail)) { (*g_assert_poison) = 'X' ;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 230, "guarantee(" "successor_next == tail" ") failed", "invariant" ); ::breakpoint(); } } while (0); | |||
| 231 | LastNode<Node> excise; | |||
| 232 | find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, excise); | |||
| 233 | // Invariant: successor excised from last list | |||
| 234 | } | |||
| 235 | // Commit the modification back to the version control system. | |||
| 236 | // It blocks until all checkouts for versions earlier than the commit have been released. | |||
| 237 | version_handle->commit(); | |||
| 238 | // At this point we know there can be no references onto the excised node. It is safe, enjoy it. | |||
| 239 | return successor; | |||
| 240 | } | |||
| 241 | ||||
| 242 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 243 | bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::in_list(const typename Client::Node* node, | |||
| 244 | typename Client::Node* head, | |||
| 245 | const typename Client::Node* tail) const { | |||
| 246 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 246, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 247 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 247, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 248 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 248, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 249 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 250 | const Node* current = head; | |||
| 251 | version_handle->checkout(); | |||
| 252 | const Node* next = Atomic::load_acquire(¤t->_next); | |||
| 253 | while (true) { | |||
| 254 | if (!is_marked_for_removal(next)) { | |||
| 255 | if (current == node) { | |||
| 256 | return true; | |||
| 257 | } | |||
| 258 | } | |||
| 259 | current = unmask(next); | |||
| 260 | if (current == tail) break; | |||
| 261 | next = current->_next; | |||
| 262 | } | |||
| 263 | return false; | |||
| 264 | } | |||
| 265 | ||||
| 266 | template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy> | |||
| 267 | template <typename Callback> | |||
| 268 | inline void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::iterate(typename Client::Node* head, | |||
| 269 | const typename Client::Node* tail, | |||
| 270 | Callback& cb) { | |||
| 271 | assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 271, "assert(" "head != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 272 | assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 272, "assert(" "tail != __null" ") failed", "invariant"); :: breakpoint(); } } while (0); | |||
| 273 | assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error ("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp" , 273, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint (); } } while (0); | |||
| 274 | VersionHandle version_handle = _client->get_version_handle(); | |||
| 275 | NodePtr current = head; | |||
| 276 | version_handle->checkout(); | |||
| 277 | NodePtr next = Atomic::load_acquire(¤t->_next); | |||
| 278 | while (true) { | |||
| 279 | if (!is_marked_for_removal(next)) { | |||
| 280 | if (!cb.process(current)) { | |||
| 281 | return; | |||
| 282 | } | |||
| 283 | } | |||
| 284 | current = unmask(next); | |||
| 285 | if (current == tail) break; | |||
| 286 | next = current->_next; | |||
| 287 | } | |||
| 288 | } | |||
| 289 | ||||
| 290 | #endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP |