/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp

Bug Summary

File:	jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp
Warning:	line 93, column 9 1st function call argument is an uninitialized value

Annotated Source Code

Press '?' to see keyboard shortcuts

Show analyzer invocation

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name jfrTraceIdKlassQueue.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -mthread-model posix -fno-delete-null-pointer-checks -mframe-pointer=all -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/libjvm/objs/precompiled -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D _GNU_SOURCE -D _REENTRANT -D LIBC=gnu -D LINUX -D VM_LITTLE_ENDIAN -D _LP64=1 -D ASSERT -D CHECK_UNHANDLED_OOPS -D TARGET_ARCH_x86 -D INCLUDE_SUFFIX_OS=_linux -D INCLUDE_SUFFIX_CPU=_x86 -D INCLUDE_SUFFIX_COMPILER=_gcc -D TARGET_COMPILER_gcc -D AMD64 -D HOTSPOT_LIB_ARCH="amd64" -D COMPILER1 -D COMPILER2 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc/adfiles -I /home/daniel/Projects/java/jdk/src/hotspot/share -I /home/daniel/Projects/java/jdk/src/hotspot/os/linux -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix -I /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86 -I /home/daniel/Projects/java/jdk/src/hotspot/os_cpu/linux_x86 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc -I /home/daniel/Projects/java/jdk/src/hotspot/share/precompiled -I /home/daniel/Projects/java/jdk/src/hotspot/share/include -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix/include -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/support/modules_include/java.base -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/support/modules_include/java.base/linux -I /home/daniel/Projects/java/jdk/src/java.base/share/native/libjimage -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc/adfiles -I /home/daniel/Projects/java/jdk/src/hotspot/share -I /home/daniel/Projects/java/jdk/src/hotspot/os/linux -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix -I /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86 -I /home/daniel/Projects/java/jdk/src/hotspot/os_cpu/linux_x86 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc -D _FORTIFY_SOURCE=2 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -Wno-format-zero-length -Wno-unused-parameter -Wno-unused -Wno-parentheses -Wno-comment -Wno-unknown-pragmas -Wno-address -Wno-delete-non-virtual-dtor -Wno-char-subscripts -Wno-array-bounds -Wno-int-in-bool-context -Wno-ignored-qualifiers -Wno-missing-field-initializers -Wno-implicit-fallthrough -Wno-empty-body -Wno-strict-overflow -Wno-sequence-point -Wno-maybe-uninitialized -Wno-misleading-indentation -Wno-cast-function-type -Wno-shift-negative-value -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /home/daniel/Projects/java/jdk/make/hotspot -ferror-limit 19 -fmessage-length 0 -fvisibility hidden -stack-protector 1 -fno-rtti -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -o /home/daniel/Projects/java/scan/2021-12-21-193737-8510-1 -x c++ /home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp

/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp

→

1/*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24 
25#include "precompiled.hpp"
26#include "jfr/recorder/checkpoint/types/traceid/jfrTraceId.inline.hpp"
27#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
28#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.hpp"
29#include "jfr/support/jfrKlassUnloading.hpp"
30#include "jfr/support/jfrThreadLocal.hpp"
31#include "jfr/utilities/jfrEpochQueue.inline.hpp"
32#include "jfr/utilities/jfrTypes.hpp"
33#include "memory/metaspace.hpp"
34#include "oops/compressedOops.hpp"
35#include "utilities/macros.hpp"
36 
37#ifdef VM_LITTLE_ENDIAN1
38static const u1      UNLOADED_BIT = 1;
39static const u1      UNCOMPRESSED_BIT = 1 << 1;
40static const u1      METADATA_SHIFT = UNCOMPRESSED_BIT;
41static const traceid UNLOADED = UNLOADED_BIT;
42static const traceid UNCOMPRESSED = UNCOMPRESSED_BIT;
43static const juint   UNLOADED_NARROW = UNLOADED_BIT;
44static const juint   UNCOMPRESSED_NARROW = UNCOMPRESSED_BIT;
45#else
46static const u1      UNLOADED_BIT = 1 << 7;
47static const u1      UNCOMPRESSED_BIT = 1 << 6;
48static const traceid UNLOADED = (traceid)UNLOADED_BIT << 56;
49static const traceid UNCOMPRESSED = (traceid)UNCOMPRESSED_BIT << 56;
50static const traceid METADATA_MASK = ~(UNCOMPRESSED | UNLOADED);
51static const juint   UNLOADED_NARROW = (juint)UNLOADED_BIT << 24;
52static const juint   UNCOMPRESSED_NARROW = (juint)UNCOMPRESSED_BIT << 24;
53static const juint   METADATA_MASK_NARROW = ~(UNCOMPRESSED_NARROW | UNLOADED_NARROW);
54#endif
55 
56struct JfrEpochQueueKlassElement {
57  traceid id;
58  const Klass* klass;
59};
60 
61struct JfrEpochQueueNarrowKlassElement {
62  u4 id;
63  narrowKlass compressed_klass;
64};
65 
66static const size_t ELEMENT_SIZE = sizeof(JfrEpochQueueKlassElement);
67static const size_t NARROW_ELEMENT_SIZE = sizeof(JfrEpochQueueNarrowKlassElement);
68static const size_t THRESHOLD_SHIFT = 30;
69 
70// If the traceid value is less than this threshold (1 073 741 824),
71// compress the element for more effective queue storage.
72static const traceid uncompressed_threshold = ((traceid)1) << THRESHOLD_SHIFT;
73 
74static size_t element_size(bool compressed) {
75  return compressed ? NARROW_ELEMENT_SIZE : ELEMENT_SIZE;
76}
77 
78static bool can_compress_element(traceid id) {
79  return Metaspace::using_class_space() && id < uncompressed_threshold;
80}
81 
82static size_t element_size(const Klass* klass) {
83  assert(klass != NULL, "invariant")do { if (!(klass != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 83, "assert(" "klass != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
84  return element_size(can_compress_element(JfrTraceId::load_raw(klass)));
85}
86 
87static bool is_unloaded(traceid id, bool previous_epoch) {
88  return JfrKlassUnloading::is_unloaded(id, previous_epoch);
89}
90 
91static narrowKlass encode(const Klass* klass) {
92  return CompressedKlassPointers::encode(const_cast<Klass*>(klass));
93}
94 
95static const Klass* decode(narrowKlass klass) {
96  return CompressedKlassPointers::decode(klass);
97}
98 
99static traceid unmask_id(traceid id, bool compressed) {
100#ifdef VM_LITTLE_ENDIAN1
101  return id >> METADATA_SHIFT;
102#else
103  return compressed ? id & METADATA_MASK_NARROW : id & METADATA_MASK;
104#endif
105}
106 
107static traceid read_compressed_element(const u1* pos, const Klass** klass) {
108  const JfrEpochQueueNarrowKlassElement* element = (const JfrEpochQueueNarrowKlassElement*)pos;
109  *klass = decode(element->compressed_klass);
110  return unmask_id(element->id, true);
111}
112 
113static traceid read_uncompressed_element(const u1* pos, const Klass** klass) {
114  const JfrEpochQueueKlassElement* element = (const JfrEpochQueueKlassElement*)pos;
115  *klass = element->klass;
116  return unmask_id(element->id, false);
117}
118 
119static traceid read_element(const u1* pos, const Klass** klass, bool compressed) {
120  assert(pos != NULL, "invariant")do { if (!(pos != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 120, "assert(" "pos != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
121  return compressed ? read_compressed_element(pos, klass) : read_uncompressed_element(pos, klass);
122}
123 
124template <typename T>
125static inline void store_traceid(T* element, traceid id, bool uncompressed) {
126#ifdef VM_LITTLE_ENDIAN1
127  id <<= METADATA_SHIFT;
128#endif
129  element->id = uncompressed ? id | UNCOMPRESSED : id;
130}
131 
132static void store_compressed_element(traceid id, const Klass* klass, u1* pos) {
133  assert(can_compress_element(id), "invariant")do { if (!(can_compress_element(id))) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 133, "assert(" "can_compress_element(id)" ") failed", "invariant"
); ::breakpoint(); } } while (0);
134  JfrEpochQueueNarrowKlassElement* const element = new (pos) JfrEpochQueueNarrowKlassElement();
135  store_traceid(element, id, false);
136  element->compressed_klass = encode(klass);
137}
138 
139static void store_uncompressed_element(traceid id, const Klass* klass, u1* pos) {
140  JfrEpochQueueKlassElement* const element = new (pos) JfrEpochQueueKlassElement();
141  store_traceid(element, id, true);
142  element->klass = klass;
143}
144 
145static void store_element(const Klass* klass, u1* pos) {
146  assert(pos != NULL, "invariant")do { if (!(pos != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 146, "assert(" "pos != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
147  assert(klass != NULL, "invariant")do { if (!(klass != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 147, "assert(" "klass != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
148  const traceid id = JfrTraceId::load_raw(klass);
149  if (can_compress_element(id)) {
150    store_compressed_element(id, klass, pos);
151    return;
152  }
153  store_uncompressed_element(id, klass, pos);
154}
155 
156static void set_unloaded(const u1* pos) {
157  *(const_cast<u1*>(pos)) |= UNLOADED_BIT;
158}
159 
160static bool is_unloaded(const u1* pos) {
161  return (*pos & UNLOADED_BIT) == UNLOADED_BIT;
162}
163 
164static bool is_compressed(const u1* pos) {
165  return (*pos & UNCOMPRESSED_BIT) == 0;
166}
167 
168// this is an optimization to clear out elements
169// by short-curcuiting the callback loop.
170static bool _clear = false;
171 
172template <typename Buffer>
173size_t JfrEpochQueueKlassPolicy<Buffer>::operator()(const u1* pos, KlassFunctor& callback, bool previous_epoch) {
174  assert(pos != NULL, "invariant")do { if (!(pos != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 174, "assert(" "pos != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
175  const bool compressed = is_compressed(pos);
176  const size_t size = ::element_size(compressed);
177  if (_clear || is_unloaded(pos)) {
178    return size;
179  }
180  const Klass* klass;
181  const traceid id = read_element(pos, &klass, compressed);
182  assert(id > 0, "invariant")do { if (!(id > 0)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 182, "assert(" "id > 0" ") failed", "invariant"); ::breakpoint
(); } } while (0);
183  if (is_unloaded(id, previous_epoch)) {
184    set_unloaded(pos);
185    return size;
186  }
187  assert(klass != NULL, "invariant")do { if (!(klass != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 187, "assert(" "klass != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
188  callback(const_cast<Klass*>(klass));
189  return size;
190}
191 
192template <typename Buffer>
193void JfrEpochQueueKlassPolicy<Buffer>::store_element(const Klass* klass, Buffer* buffer) {
194  assert(klass != NULL, "invariant")do { if (!(klass != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 194, "assert(" "klass != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
195  assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 195, "assert(" "buffer != __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
196  assert(buffer->free_size() >= ::element_size(klass), "invariant")do { if (!(buffer->free_size() >= ::element_size(klass)
)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 196, "assert(" "buffer->free_size() >= ::element_size(klass)"
 ") failed", "invariant"); ::breakpoint(); } } while (0);
197  ::store_element(klass, buffer->pos());
198}
199 
200template <typename Buffer>
201inline size_t JfrEpochQueueKlassPolicy<Buffer>::element_size(const Klass* klass) {
202  assert(klass != NULL, "invariant")do { if (!(klass != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 202, "assert(" "klass != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
203  return ::element_size(klass);
204}
205 
206template <typename Buffer>
207inline Buffer* JfrEpochQueueKlassPolicy<Buffer>::thread_local_storage(Thread* thread) const {
208  assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 208, "assert(" "thread != __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
209  JfrThreadLocal* tl = thread->jfr_thread_local();
210  return JfrTraceIdEpoch::epoch() ? tl->_load_barrier_buffer_epoch_1 : tl->_load_barrier_buffer_epoch_0;
211}
212 
213template <typename Buffer>
214inline void JfrEpochQueueKlassPolicy<Buffer>::set_thread_local_storage(Buffer* buffer, Thread* thread) {
215  assert(thread != NULL, "invariant")do { if (!(thread != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 215, "assert(" "thread != __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
216  JfrThreadLocal* tl = thread->jfr_thread_local();
217  if (JfrTraceIdEpoch::epoch()) {
218    tl->_load_barrier_buffer_epoch_1 = buffer;
219  } else {
220    tl->_load_barrier_buffer_epoch_0 = buffer;
221  }
222}
223 
224JfrTraceIdKlassQueue::JfrTraceIdKlassQueue() : _queue() {}
225 
226JfrTraceIdKlassQueue::~JfrTraceIdKlassQueue() {
227  delete _queue;
1
Calling '~JfrEpochQueue'→
228}
229 
230bool JfrTraceIdKlassQueue::initialize(size_t min_elem_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count) {
231  assert(_queue == NULL, "invariant")do { if (!(_queue == __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 231, "assert(" "_queue == __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
232  _queue = new JfrEpochQueue<JfrEpochQueueKlassPolicy>();
233  return _queue != NULL__null && _queue->initialize(min_elem_size, free_list_cache_count_limit, cache_prealloc_count);
234}
235 
236void JfrTraceIdKlassQueue::clear() {
237  if (_queue != NULL__null) {
238    _clear = true;
239    KlassFunctor functor(NULL__null);
240    _queue->iterate(functor, true);
241    _clear = false;
242  }
243}
244 
245void JfrTraceIdKlassQueue::enqueue(const Klass* klass) {
246  assert(klass != NULL, "invariant")do { if (!(klass != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/checkpoint/types/traceid/jfrTraceIdKlassQueue.cpp"
, 246, "assert(" "klass != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
247  _queue->enqueue(klass);
248}
249 
250JfrBuffer* JfrTraceIdKlassQueue::get_enqueue_buffer(Thread* thread) {
251  return _queue->thread_local_storage(thread);
252}
253 
254JfrBuffer* JfrTraceIdKlassQueue::renew_enqueue_buffer(size_t size, Thread* thread) {
255  return _queue->renew(size, thread);
256}
257 
258void JfrTraceIdKlassQueue::iterate(klass_callback callback, bool previous_epoch) {
259  assert_locked_or_safepoint(ClassLoaderDataGraph_lock);
260  KlassFunctor functor(callback);
261  _queue->iterate(functor, previous_epoch);
262}

←

/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp

→

1/*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24 
25#ifndef SHARE_JFR_UTILITIES_JFREPOCHQUEUE_INLINE_HPP
26#define SHARE_JFR_UTILITIES_JFREPOCHQUEUE_INLINE_HPP
27 
28#include "jfr/utilities/jfrEpochQueue.hpp"
29#include "jfr/recorder/storage/jfrEpochStorage.inline.hpp"
30#include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
31#include "runtime/thread.inline.hpp"
32 
33template <template <typename> class ElementPolicy>
34JfrEpochQueue<ElementPolicy>::JfrEpochQueue() : _storage(NULL__null) {}
35 
36template <template <typename> class ElementPolicy>
37JfrEpochQueue<ElementPolicy>::~JfrEpochQueue() {
38  delete _storage;
2
←
Calling '~JfrEpochStorageHost'→
39}
40 
41template <template <typename> class ElementPolicy>
42bool JfrEpochQueue<ElementPolicy>::initialize(size_t min_buffer_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count) {
43  assert(_storage == NULL, "invariant")do { if (!(_storage == __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 43, "assert(" "_storage == __null" ") failed", "invariant")
; ::breakpoint(); } } while (0);
44  _storage = new JfrEpochStorage();
45  return _storage != NULL__null && _storage->initialize(min_buffer_size, free_list_cache_count_limit, cache_prealloc_count);
46}
47 
48template <template <typename> class ElementPolicy>
49inline typename JfrEpochQueue<ElementPolicy>::BufferPtr
50JfrEpochQueue<ElementPolicy>::renew(size_t size, Thread* thread) {
51  assert(thread != nullptr, "invariant")do { if (!(thread != nullptr)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 51, "assert(" "thread != nullptr" ") failed", "invariant");
 ::breakpoint(); } } while (0);
52  BufferPtr buffer = this->thread_local_storage(thread);
53  if (buffer != nullptr) {
54    _storage->release(buffer);
55  }
56  buffer = _storage->acquire(size, thread);
57  assert(buffer != nullptr, "invariant")do { if (!(buffer != nullptr)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 57, "assert(" "buffer != nullptr" ") failed", "invariant");
 ::breakpoint(); } } while (0);
58  assert(buffer->free_size() >= size, "invariant")do { if (!(buffer->free_size() >= size)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 58, "assert(" "buffer->free_size() >= size" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
59  this->set_thread_local_storage(buffer, thread);
60  assert(this->thread_local_storage(thread) == buffer, "invariant")do { if (!(this->thread_local_storage(thread) == buffer)) {
 (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 60, "assert(" "this->thread_local_storage(thread) == buffer"
 ") failed", "invariant"); ::breakpoint(); } } while (0);
61  return buffer;
62}
63 
64template <template <typename> class ElementPolicy>
65inline typename JfrEpochQueue<ElementPolicy>::BufferPtr
66JfrEpochQueue<ElementPolicy>::storage_for_element(JfrEpochQueue<ElementPolicy>::TypePtr t, size_t element_size) {
67  assert(this->element_size(t) == element_size, "invariant")do { if (!(this->element_size(t) == element_size)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 67, "assert(" "this->element_size(t) == element_size" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
68  Thread* const thread = Thread::current();
69  BufferPtr buffer = this->thread_local_storage(thread);
70  if (buffer == nullptr) {
71    buffer = _storage->acquire(element_size, thread);
72    this->set_thread_local_storage(buffer, thread);
73  } else if (buffer->free_size() < element_size) {
74    _storage->release(buffer);
75    buffer = _storage->acquire(element_size, thread);
76    this->set_thread_local_storage(buffer, thread);
77  }
78  assert(buffer->free_size() >= element_size, "invariant")do { if (!(buffer->free_size() >= element_size)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 78, "assert(" "buffer->free_size() >= element_size" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
79  assert(this->thread_local_storage(thread) == buffer, "invariant")do { if (!(this->thread_local_storage(thread) == buffer)) {
 (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 79, "assert(" "this->thread_local_storage(thread) == buffer"
 ") failed", "invariant"); ::breakpoint(); } } while (0);
80  return buffer;
81}
82 
83template <template <typename> class ElementPolicy>
84void JfrEpochQueue<ElementPolicy>::enqueue(JfrEpochQueue<ElementPolicy>::TypePtr t) {
85  assert(t != nullptr, "invariant")do { if (!(t != nullptr)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 85, "assert(" "t != nullptr" ") failed", "invariant"); ::breakpoint
(); } } while (0);
86  size_t element_size = this->element_size(t);
87  BufferPtr buffer = storage_for_element(t, element_size);
88  assert(buffer != nullptr, "invariant")do { if (!(buffer != nullptr)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 88, "assert(" "buffer != nullptr" ") failed", "invariant");
 ::breakpoint(); } } while (0);
89  this->store_element(t, buffer);
90  buffer->set_pos(element_size);
91}
92 
93template <template <typename> class ElementPolicy>
94template <typename Callback>
95JfrEpochQueue<ElementPolicy>::ElementDispatch<Callback>::ElementDispatch(Callback& callback, JfrEpochQueue<ElementPolicy>& queue) :
96  _callback(callback), _queue(queue) {}
97 
98template <template <typename> class ElementPolicy>
99template <typename Callback>
100size_t JfrEpochQueue<ElementPolicy>::ElementDispatch<Callback>::operator()(const u1* element, bool previous_epoch) {
101  assert(element != nullptr, "invariant")do { if (!(element != nullptr)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrEpochQueue.inline.hpp"
, 101, "assert(" "element != nullptr" ") failed", "invariant"
); ::breakpoint(); } } while (0);
102  return _queue(element, _callback, previous_epoch);
103}
104 
105template <template <typename> class ElementPolicy>
106template <typename Callback>
107void JfrEpochQueue<ElementPolicy>::iterate(Callback& callback, bool previous_epoch) {
108  typedef ElementDispatch<Callback> ElementDispatcher;
109  typedef EpochDispatchOp<ElementDispatcher> QueueDispatcher;
110  ElementDispatcher element_dispatcher(callback, *this);
111  QueueDispatcher dispatch(element_dispatcher, previous_epoch);
112  _storage->iterate(dispatch, previous_epoch);
113  DEBUG_ONLY(_storage->verify_previous_empty();)_storage->verify_previous_empty();
114}
115 
116#endif // SHARE_JFR_UTILITIES_JFREPOCHQUEUE_INLINE_HPP

←

/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp

→

1/*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24 
25#ifndef SHARE_JFR_RECORDER_STORAGE_JFREPOCHSTORAGE_INLINE_HPP
26#define SHARE_JFR_RECORDER_STORAGE_JFREPOCHSTORAGE_INLINE_HPP
27 
28#include "jfr/recorder/storage/jfrEpochStorage.hpp"
29 
30#include "jfr/recorder/storage/jfrMemorySpace.inline.hpp"
31#include "jfr/recorder/storage/jfrStorageUtils.inline.hpp"
32#include "jfr/utilities/jfrConcurrentQueue.inline.hpp"
33#include "jfr/utilities/jfrLinkedList.inline.hpp"
34#include "logging/log.hpp"
35 
36template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
37JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::JfrEpochStorageHost() : _mspace(NULL__null) {}
38 
39template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
40JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::~JfrEpochStorageHost() {
41  delete _mspace;
3
←
Calling '~JfrMemorySpace'→
42}
43 
44template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
45bool JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::initialize(size_t min_elem_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count) {
46  assert(_mspace == NULL, "invariant")do { if (!(_mspace == __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp"
, 46, "assert(" "_mspace == __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
47  _mspace = new EpochMspace(min_elem_size, free_list_cache_count_limit, this);
48  return _mspace != NULL__null && _mspace->initialize(cache_prealloc_count);
49}
50 
51template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
52inline NodeType* JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::acquire(size_t size, Thread* thread) {
53  BufferPtr buffer = mspace_acquire_to_live_list(size, _mspace, thread);
54  if (buffer == NULL__null) {
55    log_warning(jfr)(!(LogImpl<(LogTag::_jfr), (LogTag::__NO_TAG), (LogTag::__NO_TAG
), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG)
>::is_level(LogLevel::Warning))) ? (void)0 : LogImpl<(LogTag
::_jfr), (LogTag::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG
), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>::write<LogLevel
::Warning>("Unable to allocate " SIZE_FORMAT"%" "l" "u" " bytes of %s.", _mspace->min_element_size(), "epoch storage");
56    return NULL__null;
57  }
58  assert(buffer->acquired_by_self(), "invariant")do { if (!(buffer->acquired_by_self())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp"
, 58, "assert(" "buffer->acquired_by_self()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
59  return buffer;
60}
61 
62template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
63void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::release(NodeType* buffer) {
64  assert(buffer != NULL, "invariant")do { if (!(buffer != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp"
, 64, "assert(" "buffer != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
65  buffer->set_retired();
66}
67 
68template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
69void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::register_full(NodeType* buffer, Thread* thread) {
70  // nothing here at the moment
71}
72 
73template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
74template <typename Functor>
75void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::iterate(Functor& functor, bool previous_epoch) {
76  typedef ReinitializeAllReleaseRetiredOp<EpochMspace, typename EpochMspace::LiveList> PreviousEpochReleaseOperation;
77  typedef CompositeOperation<Functor, PreviousEpochReleaseOperation> PreviousEpochOperation;
78  typedef ReleaseRetiredOp<EpochMspace, typename EpochMspace::LiveList> CurrentEpochReleaseOperation;
79  typedef CompositeOperation<Functor, CurrentEpochReleaseOperation> CurrentEpochOperation;
80  if (previous_epoch) {
81    PreviousEpochReleaseOperation pero(_mspace, _mspace->live_list(true));
82    PreviousEpochOperation peo(&functor, &pero);
83    process_live_list(peo, _mspace, true); // previous epoch list
84    return;
85  }
86  if (EagerReclaim) {
87    CurrentEpochReleaseOperation cero(_mspace, _mspace->live_list());
88    CurrentEpochOperation ceo(&functor, &cero);
89    process_live_list(ceo, _mspace, false); // current epoch list
90    return;
91  }
92  process_live_list(functor, _mspace, false); // current epoch list
93}
94 
95#ifdef ASSERT1
96 
97template <typename Mspace>
98class EmptyVerifier {
99 private:
100  Mspace* _mspace;
101 public:
102  typedef typename Mspace::Node Node;
103  typedef typename Mspace::NodePtr NodePtr;
104  EmptyVerifier(Mspace* mspace) : _mspace(mspace) {}
105  bool process(NodePtr node) {
106    assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp"
, 106, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
107    assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrEpochStorage.inline.hpp"
, 107, "assert(" "node->empty()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
108    return true;
109  }
110};
111 
112template <typename NodeType, template <typename> class RetrievalPolicy, bool EagerReclaim>
113void JfrEpochStorageHost<NodeType, RetrievalPolicy, EagerReclaim>::verify_previous_empty() const {
114  typedef EmptyVerifier<JfrEpochStorage::Mspace> VerifyEmptyMspace;
115  VerifyEmptyMspace vem(_mspace);
116  process_live_list(vem, _mspace, true);
117}
118 
119#endif // ASSERT
120 
121#endif // SHARE_JFR_RECORDER_STORAGE_JFREPOCHSTORAGE_INLINE_HPP

←

/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp

→

1/*
2 * Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24 
25#ifndef SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP
26#define SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP
27 
28#include "jfr/recorder/storage/jfrMemorySpace.hpp"
29 
30#include "jfr/recorder/checkpoint/types/traceid/jfrTraceIdEpoch.hpp"
31#include "runtime/atomic.hpp"
32#include "runtime/os.hpp"
33 
34template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
35JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::
36JfrMemorySpace(size_t min_element_size, size_t free_list_cache_count_limit, Client* client) :
37  _free_list(),
38  _live_list_epoch_0(),
39  _live_list_epoch_1(),
40  _client(client),
41  _min_element_size(min_element_size),
42  _free_list_cache_count_limit(free_list_cache_count_limit),
43  _free_list_cache_count(0) {}
44 
45template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
46JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::~JfrMemorySpace() {
47  while (_live_list_epoch_0.is_nonempty()) {
4
←
Loop condition is false. Execution continues on line 50→
48    deallocate(_live_list_epoch_0.remove());
49  }
50  while (_live_list_epoch_1.is_nonempty()) {
5
←
Loop condition is false. Execution continues on line 53→
51    deallocate(_live_list_epoch_1.remove());
52  }
53  while (_free_list.is_nonempty()) {
6
←
Loop condition is true.  Entering loop body→
54    deallocate(_free_list.remove());
7
←
Calling 'JfrConcurrentQueue::remove'→
55  }
56}
57 
58template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
59bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::initialize(size_t cache_prealloc_count, bool prealloc_to_free_list) {
60  if (!(_free_list.initialize() && _live_list_epoch_0.initialize() && _live_list_epoch_1.initialize())) {
61    return false;
62  }
63  // pre-allocate elements to be cached in the requested list
64  for (size_t i = 0; i < cache_prealloc_count; ++i) {
65    NodePtr const node = allocate(_min_element_size);
66    if (node == NULL__null) {
67      return false;
68    }
69    if (prealloc_to_free_list) {
70      add_to_free_list(node);
71    } else {
72      add_to_live_list(node);
73    }
74  }
75  return true;
76}
77 
78template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
79inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::should_populate_free_list_cache() const {
80  return !is_free_list_cache_limited() || _free_list_cache_count < _free_list_cache_count_limit;
81}
82 
83template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
84inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::is_free_list_cache_limited() const {
85  return _free_list_cache_count_limit != JFR_MSPACE_UNLIMITED_CACHE_SIZE;
86}
87 
88template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
89inline size_t JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::min_element_size() const {
90  return _min_element_size;
91}
92 
93template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
94inline FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() {
95  return _free_list;
96}
97 
98template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
99inline const FreeListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list() const {
100  return _free_list;
101}
102 
103template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
104inline FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) {
105  if (epoch_aware) {
106    return previous_epoch ? previous_epoch_list() : current_epoch_list();
107  }
108  return _live_list_epoch_0;
109}
110 
111template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
112inline const FullListType& JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list(bool previous_epoch) const {
113  if (epoch_aware) {
114    return previous_epoch ? previous_epoch_list() : current_epoch_list();
115  }
116  return _live_list_epoch_0;
117}
118 
119template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
120inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_empty() const {
121  return _free_list.is_empty();
122}
123 
124template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
125inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::free_list_is_nonempty() const {
126  return !free_list_is_empty();
127}
128 
129template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
130inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_empty(bool previous_epoch) const {
131  return live_list().is_empty();
132}
133 
134template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
135inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::live_list_is_nonempty(bool previous_epoch) const {
136  return live_list().is_nonempty();
137}
138 
139template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
140bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_free_list(const typename FreeListType::Node* node) const {
141  return _free_list.in_list(node);
142}
143 
144template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
145inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
146JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) const {
147  assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 147, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint
(); } } while (0);
148  return epoch == 0 ? _live_list_epoch_0 : _live_list_epoch_1;
149}
150 
151template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
152inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
153JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::epoch_list_selector(u1 epoch) {
154  return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>(
155    const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->epoch_list_selector(epoch));
156}
157 
158template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
159inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
160JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() const {
161  assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 161, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint
(); } } while (0);
162  return epoch_list_selector(JfrTraceIdEpoch::current());
163}
164 
165template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
166inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
167JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::current_epoch_list() {
168  return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>(
169    const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->current_epoch_list());
170}
171 
172template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
173inline const typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
174JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() const {
175  assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 175, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint
(); } } while (0);
176  return epoch_list_selector(JfrTraceIdEpoch::previous());
177}
178 
179template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
180inline typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&
181JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::previous_epoch_list() {
182  return const_cast<typename JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::LiveList&>(
183    const_cast<const JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>*>(this)->previous_epoch_list());
184}
185 
186template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
187bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_live_list(const typename FreeListType::Node* node, bool previous_epoch) const {
188  return live_list(previous_epoch).in_list(node);
189}
190 
191template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
192inline bool JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_current_epoch_list(const typename FreeListType::Node* node) const {
193  assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 193, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint
(); } } while (0);
194  return current_epoch_list().in_list(node);
195}
196 
197template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
198inline bool JfrMemorySpace< Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::in_previous_epoch_list(const typename FreeListType::Node* node) const {
199  assert(epoch_aware, "invariant")do { if (!(epoch_aware)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 199, "assert(" "epoch_aware" ") failed", "invariant"); ::breakpoint
(); } } while (0);
200  return previous_epoch_list().in_list(node);
201}
202 
203// allocations are even multiples of the mspace min size
204static inline size_t align_allocation_size(size_t requested_size, size_t min_element_size) {
205  u8 alloc_size_bytes = min_element_size;
206  while (requested_size > alloc_size_bytes) {
207    alloc_size_bytes <<= 1;
208  }
209  return (size_t)alloc_size_bytes;
210}
211 
212template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
213inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::allocate(size_t size) {
214  const size_t aligned_size_bytes = align_allocation_size(size, _min_element_size);
215  void* const allocation = JfrCHeapObj::new_array<u1>(aligned_size_bytes + sizeof(Node));
216  if (allocation == NULL__null) {
217    return NULL__null;
218  }
219  NodePtr node = new (allocation) Node();
220  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 220, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
221  if (!node->initialize(sizeof(Node), aligned_size_bytes)) {
222    JfrCHeapObj::free(node, aligned_size_bytes + sizeof(Node));
223    return NULL__null;
224  }
225  return node;
226}
227 
228template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
229inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::deallocate(typename FreeListType::NodePtr node) {
230  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 230, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
231  assert(!in_free_list(node), "invariant")do { if (!(!in_free_list(node))) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 231, "assert(" "!in_free_list(node)" ") failed", "invariant"
); ::breakpoint(); } } while (0);
232  assert(!_live_list_epoch_0.in_list(node), "invariant")do { if (!(!_live_list_epoch_0.in_list(node))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 232, "assert(" "!_live_list_epoch_0.in_list(node)" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
233  assert(!_live_list_epoch_1.in_list(node), "invariant")do { if (!(!_live_list_epoch_1.in_list(node))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 233, "assert(" "!_live_list_epoch_1.in_list(node)" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
234  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 234, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
235  JfrCHeapObj::free(node, node->total_size());
236}
237 
238template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
239inline typename FreeListType::NodePtr JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::acquire(size_t size, bool free_list, Thread* thread, bool previous_epoch) {
240  return RetrievalPolicy<JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware> >::acquire(this, free_list, thread, size, previous_epoch);
241}
242 
243template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
244inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::release(typename FreeListType::NodePtr node) {
245  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 245, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
246  if (node->transient()) {
247    deallocate(node);
248    return;
249  }
250  assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 250, "assert(" "node->empty()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
251  assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 251, "assert(" "!node->retired()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
252  assert(node->identity() == NULL, "invariant")do { if (!(node->identity() == __null)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 252, "assert(" "node->identity() == __null" ") failed", "invariant"
); ::breakpoint(); } } while (0);
253  if (should_populate_free_list_cache()) {
254    add_to_free_list(node);
255  } else {
256    deallocate(node);
257  }
258}
259 
260template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
261inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_free_list(typename FreeListType::NodePtr node) {
262  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 262, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
263  _free_list.add(node);
264  if (is_free_list_cache_limited()) {
265    Atomic::inc(&_free_list_cache_count);
266  }
267}
268 
269template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
270inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::add_to_live_list(typename FreeListType::NodePtr node, bool previous_epoch) {
271  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 271, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
272  live_list(previous_epoch).add(node);
273}
274 
275template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
276inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::decrement_free_list_count() {
277  if (is_free_list_cache_limited()) {
278    Atomic::dec(&_free_list_cache_count);
279  }
280}
281 
282template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
283template <typename Callback>
284inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_free_list(Callback& callback) {
285  return _free_list.iterate(callback);
286}
287 
288template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
289template <typename Callback>
290inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::iterate_live_list(Callback& callback, bool previous_epoch) {
291  if (epoch_aware) {
292    live_list(previous_epoch).iterate(callback);
293    return;
294  }
295  _live_list_epoch_0.iterate(callback);
296}
297 
298template <typename Client, template <typename> class RetrievalPolicy, typename FreeListType, typename FullListType, bool epoch_aware>
299inline void JfrMemorySpace<Client, RetrievalPolicy, FreeListType, FullListType, epoch_aware>::register_full(typename FreeListType::NodePtr node, Thread* thread) {
300  _client->register_full(node, thread);
301}
302 
303template <typename Mspace, typename Client>
304static inline Mspace* create_mspace(size_t min_element_size, size_t free_list_cache_count_limit, size_t cache_prealloc_count, bool prealloc_to_free_list, Client* cb) {
305  Mspace* const mspace = new Mspace(min_element_size, free_list_cache_count_limit, cb);
306  if (mspace != NULL__null) {
307    mspace->initialize(cache_prealloc_count, prealloc_to_free_list);
308  }
309  return mspace;
310}
311 
312template <typename Mspace>
313inline typename Mspace::NodePtr mspace_allocate(size_t size, Mspace* mspace) {
314  return mspace->allocate(size);
315}
316 
317template <typename Mspace>
318inline typename Mspace::NodePtr mspace_allocate_acquired(size_t size, Mspace* mspace, Thread* thread) {
319  typename Mspace::NodePtr node = mspace_allocate(size, mspace);
320  if (node == NULL__null) return NULL__null;
321  node->set_identity(thread);
322  return node;
323}
324 
325template <typename Mspace>
326inline typename Mspace::NodePtr mspace_allocate_transient(size_t size, Mspace* mspace, Thread* thread) {
327  typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread);
328  if (node == NULL__null) return NULL__null;
329  assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison)
 = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 329, "assert(" "node->acquired_by_self()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
330  node->set_transient();
331  return node;
332}
333 
334template <typename Mspace>
335inline typename Mspace::NodePtr mspace_allocate_transient_lease(size_t size, Mspace* mspace, Thread* thread) {
336  typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread);
337  if (node == NULL__null) return NULL__null;
338  assert(node->transient(), "invariant")do { if (!(node->transient())) { (*g_assert_poison) = 'X';
; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 338, "assert(" "node->transient()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
339  node->set_lease();
340  return node;
341}
342 
343template <typename Mspace>
344inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_free(size_t size, Mspace* mspace, Thread* thread) {
345  typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread);
346  if (node == NULL__null) return NULL__null;
347  assert(node->lease(), "invariant")do { if (!(node->lease())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 347, "assert(" "node->lease()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
348  mspace->add_to_free_list(node);
349  return node;
350}
351 
352template <typename Mspace>
353inline typename Mspace::NodePtr mspace_acquire_free(size_t size, Mspace* mspace, Thread* thread) {
354  return mspace->acquire(size, true, thread);
355}
356 
357template <typename Mspace>
358inline typename Mspace::NodePtr mspace_acquire_free_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread) {
359  assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 359, "assert(" "size <= mspace->min_element_size()" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
360  for (size_t i = 0; i < retry_count; ++i) {
361    typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread);
362    if (node != NULL__null) {
363      return node;
364    }
365  }
366  return NULL__null;
367}
368 
369template <typename Mspace>
370inline typename Mspace::NodePtr mspace_allocate_to_live_list(size_t size, Mspace* mspace, Thread* thread) {
371  typename Mspace::NodePtr node = mspace_allocate_acquired(size, mspace, thread);
372  if (node == NULL__null) return NULL__null;
373  assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison)
 = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 373, "assert(" "node->acquired_by_self()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
374  mspace->add_to_live_list(node);
375  return node;
376}
377 
378template <typename Mspace>
379inline typename Mspace::NodePtr mspace_allocate_transient_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
380  typename Mspace::NodePtr node = mspace_allocate_transient(size, mspace, thread);
381  if (node == NULL__null) return NULL__null;
382  assert(node->transient(), "invariant")do { if (!(node->transient())) { (*g_assert_poison) = 'X';
; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 382, "assert(" "node->transient()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
383  mspace->add_to_live_list(node, previous_epoch);
384  return node;
385}
386 
387template <typename Mspace>
388inline typename Mspace::NodePtr mspace_allocate_transient_lease_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
389  typename Mspace::NodePtr node = mspace_allocate_transient_lease(size, mspace, thread);
390  if (node == NULL__null) return NULL__null;
391  assert(node->lease(), "invariant")do { if (!(node->lease())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 391, "assert(" "node->lease()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
392  mspace->add_to_live_list(node, previous_epoch);
393  return node;
394}
395 
396template <typename Mspace>
397inline typename Mspace::NodePtr mspace_acquire_free_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
398  assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 398, "assert(" "size <= mspace->min_element_size()" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
399  typename Mspace::NodePtr node = mspace_acquire_free(size, mspace, thread);
400  if (node == NULL__null) {
401    return NULL__null;
402  }
403  assert(node->acquired_by_self(), "invariant")do { if (!(node->acquired_by_self())) { (*g_assert_poison)
 = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 403, "assert(" "node->acquired_by_self()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
404  mspace->add_to_live_list(node, previous_epoch);
405  return node;
406}
407 
408template <typename Mspace>
409inline typename Mspace::NodePtr mspace_acquire_to_live_list(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
410  if (size <= mspace->min_element_size()) {
411    typename Mspace::NodePtr node = mspace_acquire_free_to_live_list(size, mspace, thread, previous_epoch);
412    if (node != NULL__null) {
413      return node;
414    }
415  }
416  return mspace_allocate_to_live_list(size, mspace, thread);
417}
418 
419template <typename Mspace>
420inline typename Mspace::NodePtr mspace_acquire_live(size_t size, Mspace* mspace, Thread* thread, bool previous_epoch = false) {
421  return mspace->acquire(size, false, thread, previous_epoch);
422}
423 
424template <typename Mspace>
425inline typename Mspace::NodePtr mspace_acquire_live_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) {
426  assert(size <= mspace->min_element_size(), "invariant")do { if (!(size <= mspace->min_element_size())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 426, "assert(" "size <= mspace->min_element_size()" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
427  for (size_t i = 0; i < retry_count; ++i) {
428    typename Mspace::NodePtr const node = mspace_acquire_live(size, mspace, thread, previous_epoch);
429    if (node != NULL__null) {
430      return node;
431    }
432  }
433  return NULL__null;
434}
435 
436template <typename Mspace>
437inline typename Mspace::NodePtr mspace_acquire_lease_with_retry(size_t size, Mspace* mspace, size_t retry_count, Thread* thread, bool previous_epoch = false) {
438  typename Mspace::NodePtr node = mspace_acquire_live_with_retry(size, mspace, retry_count, thread, previous_epoch);
439  if (node != NULL__null) {
440    node->set_lease();
441  }
442  return node;
443}
444 
445template <typename Mspace>
446inline void mspace_release(typename Mspace::NodePtr node, Mspace* mspace) {
447  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 447, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
448  assert(node->unflushed_size() == 0, "invariant")do { if (!(node->unflushed_size() == 0)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 448, "assert(" "node->unflushed_size() == 0" ") failed",
 "invariant"); ::breakpoint(); } } while (0);
449  assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 449, "assert(" "mspace != __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
450  mspace->release(node);
451}
452 
453template <typename Callback, typename Mspace>
454inline void process_live_list(Callback& callback, Mspace* mspace, bool previous_epoch = false) {
455  assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 455, "assert(" "mspace != __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
456  mspace->iterate_live_list(callback, previous_epoch);
457}
458 
459template <typename Callback, typename Mspace>
460inline void process_free_list(Callback& callback, Mspace* mspace) {
461  assert(mspace != NULL, "invariant")do { if (!(mspace != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 461, "assert(" "mspace != __null" ") failed", "invariant");
 ::breakpoint(); } } while (0);
462  assert(mspace->free_list_is_nonempty(), "invariant")do { if (!(mspace->free_list_is_nonempty())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 462, "assert(" "mspace->free_list_is_nonempty()" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
463  mspace->iterate_free_list(callback);
464}
465 
466template <typename Mspace>
467class ReleaseOp : public StackObj {
468 private:
469  Mspace* _mspace;
470  bool _previous_epoch;
471 public:
472  typedef typename Mspace::Node Node;
473  ReleaseOp(Mspace* mspace) : _mspace(mspace) {}
474  bool process(typename Mspace::NodePtr node);
475  size_t processed() const { return 0; }
476};
477 
478template <typename Mspace>
479inline bool ReleaseOp<Mspace>::process(typename Mspace::NodePtr node) {
480  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 480, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
481  // assumes some means of exclusive access to the node
482  if (node->transient()) {
483    // make sure the transient node is already detached
484    _mspace->release(node);
485    return true;
486  }
487  node->reinitialize();
488  if (node->identity() != NULL__null) {
489    assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 489, "assert(" "node->empty()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
490    assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 490, "assert(" "!node->retired()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
491    node->release(); // publish
492  }
493  return true;
494}
495 
496template <typename Mspace, typename List>
497class ReleaseWithExcisionOp : public ReleaseOp<Mspace> {
498 private:
499  List& _list;
500  typename List::NodePtr _prev;
501  size_t _count;
502  size_t _amount;
503 public:
504  ReleaseWithExcisionOp(Mspace* mspace, List& list) :
505    ReleaseOp<Mspace>(mspace), _list(list), _prev(NULL__null), _count(0), _amount(0) {}
506  bool process(typename List::NodePtr node);
507  size_t processed() const { return _count; }
508  size_t amount() const { return _amount; }
509};
510 
511template <typename Mspace, typename List>
512inline bool ReleaseWithExcisionOp<Mspace, List>::process(typename List::NodePtr node) {
513  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 513, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
514  if (node->transient()) {
515    _prev = _list.excise(_prev, node);
516  } else {
517    _prev = node;
518  }
519  return ReleaseOp<Mspace>::process(node);
520}
521 
522template <typename Mspace, typename List>
523class ScavengingReleaseOp : public StackObj {
524 protected:
525  Mspace* _mspace;
526  List& _list;
527  typename List::NodePtr _prev;
528  size_t _count;
529  size_t _amount;
530    bool excise_with_release(typename List::NodePtr node);
531 public:
532  typedef typename List::Node Node;
533  ScavengingReleaseOp(Mspace* mspace, List& list) :
534    _mspace(mspace), _list(list), _prev(NULL__null), _count(0), _amount(0) {}
535  bool process(typename List::NodePtr node);
536  size_t processed() const { return _count; }
537  size_t amount() const { return _amount; }
538};
539 
540template <typename Mspace, typename List>
541inline bool ScavengingReleaseOp<Mspace, List>::process(typename List::NodePtr node) {
542  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 542, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
543  assert(!node->transient(), "invariant")do { if (!(!node->transient())) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 543, "assert(" "!node->transient()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
544  if (node->retired()) {
545    return excise_with_release(node);
546  }
547  _prev = node;
548  return true;
549}
550 
551template <typename Mspace, typename List>
552inline bool ScavengingReleaseOp<Mspace, List>::excise_with_release(typename List::NodePtr node) {
553  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 553, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
554  assert(node->retired(), "invariant")do { if (!(node->retired())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 554, "assert(" "node->retired()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
555  _prev = _list.excise(_prev, node);
556  if (node->transient()) {
557    _mspace->deallocate(node);
558    return true;
559  }
560  assert(node->identity() != NULL, "invariant")do { if (!(node->identity() != __null)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 560, "assert(" "node->identity() != __null" ") failed", "invariant"
); ::breakpoint(); } } while (0);
561  assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 561, "assert(" "node->empty()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
562  assert(!node->lease(), "invariant")do { if (!(!node->lease())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 562, "assert(" "!node->lease()" ") failed", "invariant")
; ::breakpoint(); } } while (0);
563  assert(!node->excluded(), "invariant")do { if (!(!node->excluded())) { (*g_assert_poison) = 'X';
; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 563, "assert(" "!node->excluded()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
564  ++_count;
565  _amount += node->total_size();
566  node->clear_retired();
567  node->release();
568  mspace_release(node, _mspace);
569  return true;
570}
571 
572template <typename Mspace, typename FromList>
573class ReleaseRetiredOp : public StackObj {
574private:
575  Mspace* _mspace;
576  FromList& _list;
577  typename Mspace::NodePtr _prev;
578public:
579  typedef typename Mspace::Node Node;
580  ReleaseRetiredOp(Mspace* mspace, FromList& list) :
581    _mspace(mspace), _list(list), _prev(NULL__null) {}
582  bool process(Node* node);
583};
584 
585template <typename Mspace, typename FromList>
586inline bool ReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) {
587  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 587, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
588  if (node->retired()) {
589    _prev = _list.excise(_prev, node);
590    node->reinitialize();
591    assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 591, "assert(" "node->empty()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
592    assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 592, "assert(" "!node->retired()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
593    node->release();
594    mspace_release(node, _mspace);
595  } else {
596    _prev = node;
597  }
598  return true;
599}
600 
601template <typename Mspace, typename FromList>
602class ReinitializeAllReleaseRetiredOp : public StackObj {
603private:
604  Mspace* _mspace;
605  FromList& _list;
606  typename Mspace::NodePtr _prev;
607 public:
608  typedef typename Mspace::Node Node;
609  ReinitializeAllReleaseRetiredOp(Mspace* mspace, FromList& list) :
610    _mspace(mspace), _list(list), _prev(NULL__null) {}
611  bool process(Node* node);
612};
613 
614template <typename Mspace, typename FromList>
615inline bool ReinitializeAllReleaseRetiredOp<Mspace, FromList>::process(typename Mspace::Node* node) {
616  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 616, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
617  // assumes some means of exclusive access to node
618  const bool retired = node->retired();
619  node->reinitialize();
620  assert(node->empty(), "invariant")do { if (!(node->empty())) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 620, "assert(" "node->empty()" ") failed", "invariant");
 ::breakpoint(); } } while (0);
621  assert(!node->retired(), "invariant")do { if (!(!node->retired())) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 621, "assert(" "!node->retired()" ") failed", "invariant"
); ::breakpoint(); } } while (0);
622  if (retired) {
623    _prev = _list.excise(_prev, node);
624    node->release();
625    mspace_release(node, _mspace);
626  } else {
627    _prev = node;
628  }
629  return true;
630}
631 
632#ifdef ASSERT1
633template <typename Node>
634inline void assert_migration_state(const Node* old, const Node* new_node, size_t used, size_t requested) {
635  assert(old != NULL, "invariant")do { if (!(old != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 635, "assert(" "old != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
636  assert(new_node != NULL, "invariant")do { if (!(new_node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 636, "assert(" "new_node != __null" ") failed", "invariant"
); ::breakpoint(); } } while (0);
637  assert(old->pos() >= old->start(), "invariant")do { if (!(old->pos() >= old->start())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 637, "assert(" "old->pos() >= old->start()" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
638  assert(old->pos() + used <= old->end(), "invariant")do { if (!(old->pos() + used <= old->end())) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 638, "assert(" "old->pos() + used <= old->end()" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
639  assert(new_node->free_size() >= (used + requested), "invariant")do { if (!(new_node->free_size() >= (used + requested))
) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/recorder/storage/jfrMemorySpace.inline.hpp"
, 639, "assert(" "new_node->free_size() >= (used + requested)"
 ") failed", "invariant"); ::breakpoint(); } } while (0);
640}
641#endif // ASSERT
642 
643template <typename Node>
644inline void migrate_outstanding_writes(const Node* old, Node* new_node, size_t used, size_t requested) {
645  DEBUG_ONLY(assert_migration_state(old, new_node, used, requested);)assert_migration_state(old, new_node, used, requested);
646  if (used > 0) {
647    memcpy(new_node->pos(), old->pos(), used);
648  }
649}
650 
651#endif // SHARE_JFR_RECORDER_STORAGE_JFRMEMORYSPACE_INLINE_HPP

←

/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp

→

1/*
2 * Copyright (c) 2020, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24 
25#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP
26#define SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP
27 
28#include "jfr/utilities/jfrConcurrentQueue.hpp"
29 
30#include "jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
31#include "jfr/utilities/jfrVersionSystem.inline.hpp"
32 
33template <typename NodeType, typename AllocPolicy>
34JfrConcurrentQueue<NodeType, AllocPolicy>::JfrConcurrentQueue() : _list(NULL__null), _head(), _last(), _tail(), _version_system() {
35  _head._next = const_cast<NodePtr>(&_tail);
36  _last._next = const_cast<NodePtr>(&_tail);
37}
38 
39template <typename NodeType, typename AllocPolicy>
40bool JfrConcurrentQueue<NodeType, AllocPolicy>::initialize() {
41  assert(_list == NULL, "invariant")do { if (!(_list == __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp"
, 41, "assert(" "_list == __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
42  _list = new JfrConcurrentLinkedListHost<JfrConcurrentQueue<NodeType, AllocPolicy>, HeadNode, AllocPolicy>(this);
43  return _list != NULL__null && _list->initialize();
44}
45 
46template <typename NodeType, typename AllocPolicy>
47inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_empty() const {
48  return Atomic::load_acquire(&_head._next) == &_tail;
49}
50 
51template <typename NodeType, typename AllocPolicy>
52inline bool JfrConcurrentQueue<NodeType, AllocPolicy>::is_nonempty() const {
53  return !is_empty();
54}
55 
56template <typename NodeType, typename AllocPolicy>
57void JfrConcurrentQueue<NodeType, AllocPolicy>::add(typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr node) {
58  _list->insert_tail(node, &_head, &_last, &_tail);
59}
60 
61template <typename NodeType, typename AllocPolicy>
62typename JfrConcurrentQueue<NodeType, AllocPolicy>::NodePtr JfrConcurrentQueue<NodeType, AllocPolicy>::remove() {
63  return _list->remove(&_head, &_tail, &_last, false);
8
←
Calling 'JfrConcurrentLinkedListHost::remove'→
64}
65 
66template <typename NodeType, typename AllocPolicy>
67template <typename Callback>
68void JfrConcurrentQueue<NodeType, AllocPolicy>::iterate(Callback& cb) {
69  _list->iterate(&_head, &_tail, cb);
70}
71 
72template <typename NodeType, typename AllocPolicy>
73inline JfrVersionSystem::Handle JfrConcurrentQueue<NodeType, AllocPolicy>::get_version_handle() {
74  return _version_system.get();
75}
76 
77template <typename NodeType, typename AllocPolicy>
78bool JfrConcurrentQueue<NodeType, AllocPolicy>::in_list(const NodeType* node) const {
79  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentQueue.inline.hpp"
, 79, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
80  return _list->in_list(node, const_cast<NodePtr>(&_head), &_tail);
81}
82 
83#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTQUEUE_INLINE_HPP

←

/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp

1/*
2 * Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24 
25#ifndef SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP
26#define SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP
27 
28#include "jfr/utilities/jfrConcurrentLinkedListHost.hpp"
29 
30#include "jfr/utilities/jfrRelation.hpp"
31#include "jfr/utilities/jfrTypes.hpp"
32#include "runtime/atomic.hpp"
33#include "utilities/globalDefinitions.hpp"
34 
35/*
36 * The removal marker (i.e. the excision bit) is represented by '( )' as part of state description comments:
37 * "node --> next" becomes "(node) --> next", when node is logically deleted.
38 */
39template <typename Node>
40inline Node* mark_for_removal(Node* node) {
41  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 41, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
42  const Node* next = node->_next;
43  assert(next != NULL, "invariant")do { if (!(next != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 43, "assert(" "next != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
44  Node* const unmasked_next = unmask(next);
45  return next == unmasked_next && cas(&node->_next, unmasked_next, set_excision_bit(unmasked_next)) ? unmasked_next : NULL__null;
46}
47 
48/*
49 * The insertion marker (i.e. the insertion bit) is represented by '[ ]' as part of state description comments:
50 * "node --> next" becomes "[node] --> next", in an attempt to convey the node as exlusively reserved.
51 */
52template <typename Node>
53inline bool mark_for_insertion(Node* node, const Node* tail) {
54  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 54, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
55  return node->_next == tail && cas(&node->_next, const_cast<Node*>(tail), set_insertion_bit(tail));
56}
57 
58/*
59 * Find a predecessor and successor node pair where successor covers predecessor (adjacency).
60 */
61template <typename Node, typename VersionHandle, template <typename> class SearchPolicy>
62Node* find_adjacent(Node* head, const Node* tail, Node** predecessor, VersionHandle& version_handle, SearchPolicy<Node>& predicate) {
63  assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 63, "assert(" "head != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
18
←
Taking false branch→
19
←
Loop condition is false.  Exiting loop→
64  assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 64, "assert(" "tail != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
20
←
Taking false branch→
21
←
Loop condition is false.  Exiting loop→
65  assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 65, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint
(); } } while (0);
22
←
Taking false branch→
23
←
Loop condition is false.  Exiting loop→
66  Node* predecessor_next = NULL__null;
67  while (true) {
24
←
Loop condition is true.  Entering loop body→
68    Node* current = head;
69    version_handle->checkout();
70    Node* next = Atomic::load_acquire(&current->_next);
71    do {
72      assert(next != NULL, "invariant")do { if (!(next != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 72, "assert(" "next != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
25
←
Assuming the condition is true→
26
←
Taking false branch→
27
←
Loop condition is false.  Exiting loop→
73      Node* const unmasked_next = unmask(next);
74      // 1A: Locate the first node to keep as predecessor.
75      if (!is_marked_for_removal(next)) {
28
←
Assuming the condition is false→
29
←
Taking false branch→
76        *predecessor = current;
77        predecessor_next = unmasked_next;
78      }
79      // 1B: Locate the next node to keep as successor.
80      current = unmasked_next;
81      if (current == tail) break;
30
←
Assuming 'current' is equal to 'tail'→
31
←
Taking true branch→
32
←
 Execution continues on line 86→
82      next = current->_next;
83    } while (predicate(current, next));
84    // current represents the successor node from here on out.
85    // 2: Check predecessor and successor node pair for adjacency.
86    if (predecessor_next == current) {
33
←
Assuming 'predecessor_next' is not equal to 'current'→
34
←
Taking false branch→
87      // Invariant: predecessor --> successor
88      return current;
89    }
90    // 3: Successor does not (yet) cover predecessor.
91    // Invariant: predecessor --> (logically excised nodes) --> successor
92    // Physically excise one or more logically excised nodes in-between.
93    if (cas(&(*predecessor)->_next, predecessor_next, current)) {
35
←
1st function call argument is an uninitialized value
94      // Invariant: predecessor --> successor
95      return current;
96    }
97  }
98}
99 
100template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
101JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::JfrConcurrentLinkedListHost(Client* client) : _client(client) {}
102 
103template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
104bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::initialize() {
105  return true;
106}
107 
108template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
109void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_head(typename Client::Node* node,
110                                                                                 typename Client::Node* head,
111                                                                                 const typename Client::Node* tail) const {
112  Node* predecessor;
113  Node* successor;
114  HeadNode<Node> predicate(node);
115  VersionHandle version_handle = _client->get_version_handle();
116  while (true) {
117    // Find an adjacent predecessor and successor node pair.
118    successor = find_adjacent<Node, VersionHandle, HeadNode>(head, tail, &predecessor, version_handle, predicate);
119    // Invariant (adjacency): predecessor --> successor
120    // Invariant (optional: key-based total order): predecessor->key() < key  && key <= successor->key().
121    // We can now attempt to insert the new node in-between.
122    node->_next = successor;
123    if (cas(&predecessor->_next, successor, node)) {
124      // Invariant: predecessor --> node --> successor
125      // An insert to head is a benign modification and will not need to be committed to the version control system.
126      return;
127    }
128  }
129}
130 
131template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
132void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::insert_tail(typename Client::Node* node,
133                                                                                 typename Client::Node* head,
134                                                                                 typename Client::Node* last,
135                                                                                 const typename Client::Node* tail) const {
136  assert(node != NULL, "invariant")do { if (!(node != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 136, "assert(" "node != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
137  assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 137, "assert(" "head != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
138  assert(last != NULL, "invarinat")do { if (!(last != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 138, "assert(" "last != __null" ") failed", "invarinat"); ::
breakpoint(); } } while (0);
139  assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 139, "assert(" "tail != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
140  // Mark the new node to be inserted with the insertion marker already.
141  node->_next = set_insertion_bit(const_cast<NodePtr>(tail));
142  // Invariant: [node]--> tail
143  assert(is_marked_for_insertion(node->_next), "invariant")do { if (!(is_marked_for_insertion(node->_next))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 143, "assert(" "is_marked_for_insertion(node->_next)" ") failed"
, "invariant"); ::breakpoint(); } } while (0);
144  NodePtr predecessor;
145  LastNode<Node> predicate;
146  VersionHandle version_handle = _client->get_version_handle();
147  while (true) {
148    // Find an adjacent predecessor and successor node pair, where the successor == tail
149    const NodePtr successor = find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, predicate);
150    assert(successor == tail, "invariant")do { if (!(successor == tail)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 150, "assert(" "successor == tail" ") failed", "invariant")
; ::breakpoint(); } } while (0);
151    // Invariant: predecessor --> successor
152    // We first attempt to mark the predecessor node to signal our intent of performing an insertion.
153    if (mark_for_insertion(predecessor, tail)) {
154      break;
155    }
156  }
157  // Predecessor node is claimed for insertion.
158  // Invariant: [predecessor] --> tail
159  assert(is_marked_for_insertion(predecessor->_next), "invariant")do { if (!(is_marked_for_insertion(predecessor->_next))) {
 (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 159, "assert(" "is_marked_for_insertion(predecessor->_next)"
 ") failed", "invariant"); ::breakpoint(); } } while (0);
160  assert(predecessor != head, "invariant")do { if (!(predecessor != head)) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 160, "assert(" "predecessor != head" ") failed", "invariant"
); ::breakpoint(); } } while (0);
161  if (Atomic::load_acquire(&last->_next) == predecessor) {
162    /* Even after we store the new node into the last->_next field, there is no race
163       because it is also marked with the insertion bit. */
164    last->_next = node;
165    // Invariant: last --> [node] --> tail
166    OrderAccess::storestore();
167    // Perform the link with the predecessor node, which by this store becomes visible for removal.
168    predecessor->_next = node;
169    // Invariant: predecessor --> [node] --> tail
170  } else {
171    assert(last == predecessor, "invariant")do { if (!(last == predecessor)) { (*g_assert_poison) = 'X';;
 report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 171, "assert(" "last == predecessor" ") failed", "invariant"
); ::breakpoint(); } } while (0);
172    last->_next = node;
173    // Invariant: last --> [node] --> tail
174    OrderAccess::storestore();
175    /* This implies the list is logically empty from the removal perspective.
176       cas is not needed here because inserts must not come in from the head side
177       concurrently with inserts from tail which are currently blocked by us.
178       Invariant (logical): head --> tail. */
179    head->_next = node;
180    // Invariant: head --> [node] --> tail
181  }
182  OrderAccess::storestore();
183  // Publish the inserted node by removing the insertion marker.
184  node->_next = const_cast<NodePtr>(tail);
185  // Invariant: last --> node --> tail (possibly also head --> node --> tail)
186}
187 
188template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
189typename Client::Node* JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::remove(typename Client::Node* head,
190                                                                                              const typename Client::Node* tail,
191                                                                                              typename Client::Node* last /* NULL */,
192                                                                                              bool insert_is_head /* true */) {
193  assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 193, "assert(" "head != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
9
←
Taking false branch→
10
←
Loop condition is false.  Exiting loop→
194  assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 194, "assert(" "tail != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
11
←
Taking false branch→
12
←
Loop condition is false.  Exiting loop→
195  assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 195, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint
(); } } while (0);
13
←
Taking false branch→
14
←
Loop condition is false.  Exiting loop→
196  NodePtr predecessor;
15
←
'predecessor' declared without an initial value→
197  NodePtr successor;
198  NodePtr successor_next;
199  SearchPolicy<Node> predicate;
200  VersionHandle version_handle = _client->get_version_handle();
201  while (true) {
16
←
Loop condition is true.  Entering loop body→
202    // Find an adjacent predecessor and successor node pair.
203    successor = find_adjacent<Node, VersionHandle, SearchPolicy>(head, tail, &predecessor, version_handle, predicate);
17
←
Calling 'find_adjacent<JfrBuffer, RefCountHandle<JfrVersionSystem::Node>, HeadNode>'→
204    if (successor == tail) {
205      return NULL__null;
206    }
207    // Invariant: predecessor --> successor
208    // Invariant (optional: key-based total order): predecessor->key() < key  && key <= successor->key()
209    // It is the successor node that is to be removed.
210    // We first attempt to reserve (logically excise) the successor node.
211    successor_next = mark_for_removal(successor);
212    if (successor_next != NULL__null) {
213      break;
214    }
215  }
216  // Invariant: predecessor --> (successor) --> successor_next
217  // Successor node now logically excised.
218  assert(is_marked_for_removal(successor->_next), "invariant")do { if (!(is_marked_for_removal(successor->_next))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 218, "assert(" "is_marked_for_removal(successor->_next)"
 ") failed", "invariant"); ::breakpoint(); } } while (0);
219  // Now attempt to physically excise the successor node.
220  // If the cas fails, we can optimize for the slow path if we know we are not performing
221  // insertions from the head. Then a failed cas results not from a new node being inserted,
222  // but only because another thread excised us already.
223  if (!cas(&predecessor->_next, successor, successor_next) && insert_is_head) {
224    // Physically excise using slow path, can be completed asynchronously by other threads.
225    Identity<Node> excise(successor);
226    find_adjacent<Node, VersionHandle, Identity>(head, tail, &predecessor, version_handle, excise);
227  }
228  if (last != NULL__null && Atomic::load_acquire(&last->_next) == successor) {
229    guarantee(!insert_is_head, "invariant")do { if (!(!insert_is_head)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 229, "guarantee(" "!insert_is_head" ") failed", "invariant"
); ::breakpoint(); } } while (0);
230    guarantee(successor_next == tail, "invariant")do { if (!(successor_next == tail)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 230, "guarantee(" "successor_next == tail" ") failed", "invariant"
); ::breakpoint(); } } while (0);
231    LastNode<Node> excise;
232    find_adjacent<Node, VersionHandle, LastNode>(last, tail, &predecessor, version_handle, excise);
233    // Invariant: successor excised from last list
234  }
235  // Commit the modification back to the version control system.
236  // It blocks until all checkouts for versions earlier than the commit have been released.
237  version_handle->commit();
238  // At this point we know there can be no references onto the excised node. It is safe, enjoy it.
239  return successor;
240}
241 
242template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
243bool JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::in_list(const typename Client::Node* node,
244                                                                             typename Client::Node* head,
245                                                                             const typename Client::Node* tail) const {
246  assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 246, "assert(" "head != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
247  assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 247, "assert(" "tail != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
248  assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 248, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint
(); } } while (0);
249  VersionHandle version_handle = _client->get_version_handle();
250  const Node* current = head;
251  version_handle->checkout();
252  const Node* next = Atomic::load_acquire(&current->_next);
253  while (true) {
254    if (!is_marked_for_removal(next)) {
255      if (current == node) {
256        return true;
257      }
258    }
259    current = unmask(next);
260    if (current == tail) break;
261    next = current->_next;
262  }
263  return false;
264}
265 
266template <typename Client, template <typename> class SearchPolicy, typename AllocPolicy>
267template <typename Callback>
268inline void JfrConcurrentLinkedListHost<Client, SearchPolicy, AllocPolicy>::iterate(typename Client::Node* head,
269                                                                                    const typename Client::Node* tail,
270                                                                                    Callback& cb) {
271  assert(head != NULL, "invariant")do { if (!(head != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 271, "assert(" "head != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
272  assert(tail != NULL, "invariant")do { if (!(tail != __null)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 272, "assert(" "tail != __null" ") failed", "invariant"); ::
breakpoint(); } } while (0);
273  assert(head != tail, "invariant")do { if (!(head != tail)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/jfr/utilities/jfrConcurrentLinkedListHost.inline.hpp"
, 273, "assert(" "head != tail" ") failed", "invariant"); ::breakpoint
(); } } while (0);
274  VersionHandle version_handle = _client->get_version_handle();
275  NodePtr current = head;
276  version_handle->checkout();
277  NodePtr next = Atomic::load_acquire(&current->_next);
278  while (true) {
279    if (!is_marked_for_removal(next)) {
280      if (!cb.process(current)) {
281        return;
282      }
283    }
284    current = unmask(next);
285    if (current == tail) break;
286    next = current->_next;
287  }
288}
289 
290#endif // SHARE_JFR_UTILITIES_JFRCONCURRENTLINKEDLISTHOST_INLINE_HPP