Bug Summary

File:jdk/src/hotspot/share/gc/g1/g1CardSet.cpp
Warning:line 680, column 3
Undefined or garbage value returned to caller

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name g1CardSet.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -mrelocation-model pic -pic-level 2 -mthread-model posix -fno-delete-null-pointer-checks -mframe-pointer=all -relaxed-aliasing -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/libjvm/objs/precompiled -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -D __STDC_CONSTANT_MACROS -D _GNU_SOURCE -D _REENTRANT -D LIBC=gnu -D LINUX -D VM_LITTLE_ENDIAN -D _LP64=1 -D ASSERT -D CHECK_UNHANDLED_OOPS -D TARGET_ARCH_x86 -D INCLUDE_SUFFIX_OS=_linux -D INCLUDE_SUFFIX_CPU=_x86 -D INCLUDE_SUFFIX_COMPILER=_gcc -D TARGET_COMPILER_gcc -D AMD64 -D HOTSPOT_LIB_ARCH="amd64" -D COMPILER1 -D COMPILER2 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc/adfiles -I /home/daniel/Projects/java/jdk/src/hotspot/share -I /home/daniel/Projects/java/jdk/src/hotspot/os/linux -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix -I /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86 -I /home/daniel/Projects/java/jdk/src/hotspot/os_cpu/linux_x86 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc -I /home/daniel/Projects/java/jdk/src/hotspot/share/precompiled -I /home/daniel/Projects/java/jdk/src/hotspot/share/include -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix/include -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/support/modules_include/java.base -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/support/modules_include/java.base/linux -I /home/daniel/Projects/java/jdk/src/java.base/share/native/libjimage -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc/adfiles -I /home/daniel/Projects/java/jdk/src/hotspot/share -I /home/daniel/Projects/java/jdk/src/hotspot/os/linux -I /home/daniel/Projects/java/jdk/src/hotspot/os/posix -I /home/daniel/Projects/java/jdk/src/hotspot/cpu/x86 -I /home/daniel/Projects/java/jdk/src/hotspot/os_cpu/linux_x86 -I /home/daniel/Projects/java/jdk/build/linux-x86_64-server-fastdebug/hotspot/variant-server/gensrc -D _FORTIFY_SOURCE=2 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/x86_64-linux-gnu/c++/7.5.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/7.5.0/../../../../include/c++/7.5.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O3 -Wno-format-zero-length -Wno-unused-parameter -Wno-unused -Wno-parentheses -Wno-comment -Wno-unknown-pragmas -Wno-address -Wno-delete-non-virtual-dtor -Wno-char-subscripts -Wno-array-bounds -Wno-int-in-bool-context -Wno-ignored-qualifiers -Wno-missing-field-initializers -Wno-implicit-fallthrough -Wno-empty-body -Wno-strict-overflow -Wno-sequence-point -Wno-maybe-uninitialized -Wno-misleading-indentation -Wno-cast-function-type -Wno-shift-negative-value -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /home/daniel/Projects/java/jdk/make/hotspot -ferror-limit 19 -fmessage-length 0 -fvisibility hidden -stack-protector 1 -fno-rtti -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -faddrsig -o /home/daniel/Projects/java/scan/2021-12-21-193737-8510-1 -x c++ /home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp

/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp

→
1/*
2 * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#include "precompiled.hpp"
26#include "gc/g1/g1CardSet.inline.hpp"
27#include "gc/g1/g1CardSetContainers.inline.hpp"
28#include "gc/g1/g1CardSetMemory.inline.hpp"
29#include "gc/g1/g1FromCardCache.hpp"
30#include "gc/g1/heapRegion.inline.hpp"
31#include "memory/allocation.inline.hpp"
32#include "runtime/atomic.hpp"
33#include "runtime/globals_extension.hpp"
34#include "runtime/mutex.hpp"
35#include "utilities/bitMap.inline.hpp"
36#include "utilities/concurrentHashTable.inline.hpp"
37#include "utilities/globalDefinitions.hpp"
38#include "utilities/lockFreeStack.hpp"
39#include "utilities/spinYield.hpp"
40
41#include "gc/shared/gcLogPrecious.hpp"
42#include "gc/shared/gcTraceTime.inline.hpp"
43#include "runtime/java.hpp"
44
45G1CardSet::CardSetPtr G1CardSet::FullCardSet = (G1CardSet::CardSetPtr)-1;
46
47static uint default_log2_card_region_per_region() {
48 uint log2_card_regions_per_heap_region = 0;
49
50 const uint card_container_limit = G1CardSetContainer::LogCardsPerRegionLimit;
51 if (card_container_limit < (uint)HeapRegion::LogCardsPerRegion) {
52 log2_card_regions_per_heap_region = (uint)HeapRegion::LogCardsPerRegion - card_container_limit;
53 }
54
55 return log2_card_regions_per_heap_region;
56}
57
58G1CardSetConfiguration::G1CardSetConfiguration() :
59 G1CardSetConfiguration(HeapRegion::LogCardsPerRegion, /* inline_ptr_bits_per_card */
60 G1RemSetArrayOfCardsEntries, /* max_cards_in_array */
61 (double)G1RemSetCoarsenHowlBitmapToHowlFullPercent / 100, /* cards_in_bitmap_threshold_percent */
62 G1RemSetHowlNumBuckets, /* num_buckets_in_howl */
63 (double)G1RemSetCoarsenHowlToFullPercent / 100, /* cards_in_howl_threshold_percent */
64 (uint)HeapRegion::CardsPerRegion, /* max_cards_in_cardset */
65 default_log2_card_region_per_region()) /* log2_card_region_per_region */
66{
67 assert((_log2_card_regions_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion,do { if (!((_log2_card_regions_per_heap_region + _log2_cards_per_card_region
) == (uint)HeapRegion::LogCardsPerRegion)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 68, "assert(" "(_log2_card_regions_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion"
") failed", "inconsistent heap region virtualization setup")
; ::breakpoint(); } } while (0)
68 "inconsistent heap region virtualization setup")do { if (!((_log2_card_regions_per_heap_region + _log2_cards_per_card_region
) == (uint)HeapRegion::LogCardsPerRegion)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 68, "assert(" "(_log2_card_regions_per_heap_region + _log2_cards_per_card_region) == (uint)HeapRegion::LogCardsPerRegion"
") failed", "inconsistent heap region virtualization setup")
; ::breakpoint(); } } while (0);
69}
70
71G1CardSetConfiguration::G1CardSetConfiguration(uint max_cards_in_array,
72 double cards_in_bitmap_threshold_percent,
73 uint max_buckets_in_howl,
74 double cards_in_howl_threshold_percent,
75 uint max_cards_in_card_set,
76 uint log2_card_region_per_region) :
77 G1CardSetConfiguration(log2i_exact(max_cards_in_card_set), /* inline_ptr_bits_per_card */
78 max_cards_in_array, /* max_cards_in_array */
79 cards_in_bitmap_threshold_percent, /* cards_in_bitmap_threshold_percent */
80 G1CardSetHowl::num_buckets(max_cards_in_card_set, /* num_buckets_in_howl */
81 max_cards_in_array,
82 max_buckets_in_howl),
83 cards_in_howl_threshold_percent, /* cards_in_howl_threshold_percent */
84 max_cards_in_card_set, /* max_cards_in_cardset */
85 log2_card_region_per_region)
86{ }
87
88G1CardSetConfiguration::G1CardSetConfiguration(uint inline_ptr_bits_per_card,
89 uint max_cards_in_array,
90 double cards_in_bitmap_threshold_percent,
91 uint num_buckets_in_howl,
92 double cards_in_howl_threshold_percent,
93 uint max_cards_in_card_set,
94 uint log2_card_regions_per_heap_region) :
95 _inline_ptr_bits_per_card(inline_ptr_bits_per_card),
96 _max_cards_in_array(max_cards_in_array),
97 _num_buckets_in_howl(num_buckets_in_howl),
98 _max_cards_in_card_set(max_cards_in_card_set),
99 _cards_in_howl_threshold(max_cards_in_card_set * cards_in_howl_threshold_percent),
100 _max_cards_in_howl_bitmap(G1CardSetHowl::bitmap_size(_max_cards_in_card_set, _num_buckets_in_howl)),
101 _cards_in_howl_bitmap_threshold(_max_cards_in_howl_bitmap * cards_in_bitmap_threshold_percent),
102 _log2_max_cards_in_howl_bitmap(log2i_exact(_max_cards_in_howl_bitmap)),
103 _bitmap_hash_mask(~(~(0) << _log2_max_cards_in_howl_bitmap)),
104 _log2_card_regions_per_heap_region(log2_card_regions_per_heap_region),
105 _log2_cards_per_card_region(log2i_exact(_max_cards_in_card_set) - _log2_card_regions_per_heap_region) {
106
107 assert(is_power_of_2(_max_cards_in_card_set),do { if (!(is_power_of_2(_max_cards_in_card_set))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 108, "assert(" "is_power_of_2(_max_cards_in_card_set)" ") failed"
, "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set
); ::breakpoint(); } } while (0)
108 "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set)do { if (!(is_power_of_2(_max_cards_in_card_set))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 108, "assert(" "is_power_of_2(_max_cards_in_card_set)" ") failed"
, "max_cards_in_card_set must be a power of 2: %u", _max_cards_in_card_set
); ::breakpoint(); } } while (0);
109
110 init_card_set_alloc_options();
111 log_configuration();
112}
113
114G1CardSetConfiguration::~G1CardSetConfiguration() {
115 FREE_C_HEAP_ARRAY(size_t, _card_set_alloc_options)FreeHeap((char*)(_card_set_alloc_options));
116}
117
118void G1CardSetConfiguration::init_card_set_alloc_options() {
119 _card_set_alloc_options = NEW_C_HEAP_ARRAY(G1CardSetAllocOptions, num_mem_object_types(), mtGC)(G1CardSetAllocOptions*) (AllocateHeap((num_mem_object_types(
)) * sizeof(G1CardSetAllocOptions), mtGC));
120 new (&_card_set_alloc_options[0]) G1CardSetAllocOptions((uint)CardSetHash::get_node_size());
121 new (&_card_set_alloc_options[1]) G1CardSetAllocOptions((uint)G1CardSetArray::size_in_bytes(_max_cards_in_array), 2, 256);
122 new (&_card_set_alloc_options[2]) G1CardSetAllocOptions((uint)G1CardSetBitMap::size_in_bytes(_max_cards_in_howl_bitmap), 2, 256);
123 new (&_card_set_alloc_options[3]) G1CardSetAllocOptions((uint)G1CardSetHowl::size_in_bytes(_num_buckets_in_howl), 2, 256);
124}
125
126void G1CardSetConfiguration::log_configuration() {
127 log_debug_p(gc, remset)GCLogPreciousHandle( LogTargetHandle::create<LogLevel::Debug
, (LogTag::_gc), (LogTag::_remset), (LogTag::__NO_TAG), (LogTag
::__NO_TAG), (LogTag::__NO_TAG), (LogTag::__NO_TAG)>() , "/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 127).write("Card Set container configuration: "
128 "InlinePtr #cards %u size %zu "
129 "Array Of Cards #cards %u size %zu "
130 "Howl #buckets %u coarsen threshold %u "
131 "Howl Bitmap #cards %u size %zu coarsen threshold %u "
132 "Card regions per heap region %u cards per card region %u",
133 max_cards_in_inline_ptr(), sizeof(void*),
134 max_cards_in_array(), G1CardSetArray::size_in_bytes(max_cards_in_array()),
135 num_buckets_in_howl(), cards_in_howl_threshold(),
136 max_cards_in_howl_bitmap(), G1CardSetBitMap::size_in_bytes(max_cards_in_howl_bitmap()), cards_in_howl_bitmap_threshold(),
137 (uint)1 << log2_card_regions_per_heap_region(),
138 (uint)1 << log2_cards_per_card_region());
139}
140
141uint G1CardSetConfiguration::max_cards_in_inline_ptr() const {
142 return max_cards_in_inline_ptr(_inline_ptr_bits_per_card);
143}
144
145uint G1CardSetConfiguration::max_cards_in_inline_ptr(uint bits_per_card) {
146 return G1CardSetInlinePtr::max_cards_in_inline_ptr(bits_per_card);
147}
148
149const G1CardSetAllocOptions* G1CardSetConfiguration::mem_object_alloc_options(uint idx) {
150 return &_card_set_alloc_options[idx];
151}
152
153const char* G1CardSetConfiguration::mem_object_type_name_str(uint index) {
154 const char* names[] = { "Node", "Array", "Bitmap", "Howl" };
155 return names[index];
156}
157
158void G1CardSetCoarsenStats::reset() {
159 STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision))static_assert((sizeof(array_size_impl(_coarsen_from)) == sizeof
(array_size_impl(_coarsen_collision))), "ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)"
);
160 for (uint i = 0; i < ARRAY_SIZE(_coarsen_from)sizeof(array_size_impl(_coarsen_from)); i++) {
161 _coarsen_from[i] = 0;
162 _coarsen_collision[i] = 0;
163 }
164}
165
166void G1CardSetCoarsenStats::subtract_from(G1CardSetCoarsenStats& other) {
167 STATIC_ASSERT(ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision))static_assert((sizeof(array_size_impl(_coarsen_from)) == sizeof
(array_size_impl(_coarsen_collision))), "ARRAY_SIZE(_coarsen_from) == ARRAY_SIZE(_coarsen_collision)"
);
168 for (uint i = 0; i < ARRAY_SIZE(_coarsen_from)sizeof(array_size_impl(_coarsen_from)); i++) {
169 _coarsen_from[i] = other._coarsen_from[i] - _coarsen_from[i];
170 _coarsen_collision[i] = other._coarsen_collision[i] - _coarsen_collision[i];
171 }
172}
173
174void G1CardSetCoarsenStats::record_coarsening(uint tag, bool collision) {
175 assert(tag < ARRAY_SIZE(_coarsen_from), "tag %u out of bounds", tag)do { if (!(tag < sizeof(array_size_impl(_coarsen_from)))) {
(*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 175, "assert(" "tag < sizeof(array_size_impl(_coarsen_from))"
") failed", "tag %u out of bounds", tag); ::breakpoint(); } }
while (0);
176 Atomic::inc(&_coarsen_from[tag], memory_order_relaxed);
177 if (collision) {
178 Atomic::inc(&_coarsen_collision[tag], memory_order_relaxed);
179 }
180}
181
182void G1CardSetCoarsenStats::print_on(outputStream* out) {
183 out->print_cr("Inline->AoC %zu (%zu) "
184 "AoC->Howl %zu (%zu) "
185 "Howl->Full %zu (%zu) "
186 "Inline->AoC %zu (%zu) "
187 "AoC->BitMap %zu (%zu) "
188 "BitMap->Full %zu (%zu) ",
189 _coarsen_from[0], _coarsen_collision[0],
190 _coarsen_from[1], _coarsen_collision[1],
191 // There is no BitMap at the first level so we can't .
192 _coarsen_from[3], _coarsen_collision[3],
193 _coarsen_from[4], _coarsen_collision[4],
194 _coarsen_from[5], _coarsen_collision[5],
195 _coarsen_from[6], _coarsen_collision[6]
196 );
197}
198
199class G1CardSetHashTable : public CHeapObj<mtGCCardSet> {
200 using CardSetPtr = G1CardSet::CardSetPtr;
201
202 // Did we insert at least one card in the table?
203 bool volatile _inserted_card;
204
205 G1CardSetMemoryManager* _mm;
206 CardSetHash _table;
207
208 class G1CardSetHashTableLookUp : public StackObj {
209 uint _region_idx;
210 public:
211 explicit G1CardSetHashTableLookUp(uint region_idx) : _region_idx(region_idx) { }
212
213 uintx get_hash() const { return _region_idx; }
214
215 bool equals(G1CardSetHashTableValue* value, bool* is_dead) {
216 *is_dead = false;
217 return value->_region_idx == _region_idx;
218 }
219 };
220
221 class G1CardSetHashTableFound : public StackObj {
222 G1CardSetHashTableValue* _value;
223 public:
224 G1CardSetHashTableFound() : _value(nullptr) { }
225
226 void operator()(G1CardSetHashTableValue* value) {
227 _value = value;
228 }
229
230 G1CardSetHashTableValue* value() const { return _value; }
231 };
232
233 class G1CardSetHashTableScan : public StackObj {
234 G1CardSet::CardSetPtrClosure* _scan_f;
235 public:
236 explicit G1CardSetHashTableScan(G1CardSet::CardSetPtrClosure* f) : _scan_f(f) { }
237
238 bool operator()(G1CardSetHashTableValue* value) {
239 _scan_f->do_cardsetptr(value->_region_idx, value->_num_occupied, value->_card_set);
240 return true;
241 }
242 };
243
244
245public:
246 static const size_t InitialLogTableSize = 2;
247
248 G1CardSetHashTable(G1CardSetMemoryManager* mm,
249 size_t initial_log_table_size = InitialLogTableSize) :
250 _inserted_card(false),
251 _mm(mm),
252 _table(mm, initial_log_table_size) {
253 }
254
255 ~G1CardSetHashTable() {
256 reset();
257 }
258
259 G1CardSetHashTableValue* get_or_add(uint region_idx, bool* should_grow) {
260 G1CardSetHashTableLookUp lookup(region_idx);
261 G1CardSetHashTableFound found;
262
263 if (_table.get(Thread::current(), lookup, found)) {
264 return found.value();
265 }
266
267 G1CardSetHashTableValue value(region_idx, G1CardSetInlinePtr());
268 bool inserted = _table.insert_get(Thread::current(), lookup, value, found, should_grow);
269
270 if (!_inserted_card && inserted) {
271 // It does not matter to us who is setting the flag so a regular atomic store
272 // is sufficient.
273 Atomic::store(&_inserted_card, true);
274 }
275
276 return found.value();
277 }
278
279 G1CardSetHashTableValue* get(uint region_idx) {
280 G1CardSetHashTableLookUp lookup(region_idx);
281 G1CardSetHashTableFound found;
282
283 _table.get(Thread::current(), lookup, found);
284 return found.value();
285 }
286
287 void iterate_safepoint(G1CardSet::CardSetPtrClosure* cl2) {
288 G1CardSetHashTableScan cl(cl2);
289 _table.do_safepoint_scan(cl);
290 }
291
292 void iterate(G1CardSet::CardSetPtrClosure* cl2) {
293 G1CardSetHashTableScan cl(cl2);
294 _table.do_scan(Thread::current(), cl);
295 }
296
297 void reset() {
298 if (Atomic::load(&_inserted_card)) {
299 _table.unsafe_reset(InitialLogTableSize);
300 Atomic::store(&_inserted_card, false);
301 }
302 }
303
304 void print(outputStream* os) {
305 os->print("TBL " PTR_FORMAT"0x%016" "l" "x" " size %zu mem %zu ", p2i(&_table), _table.get_size_log2(Thread::current()), _table.get_mem_size(Thread::current()));
306 }
307
308 void grow() {
309 size_t new_limit = _table.get_size_log2(Thread::current()) + 1;
310 _table.grow(Thread::current(), new_limit);
311 }
312
313 size_t mem_size() {
314 return sizeof(*this) +
315 _table.get_mem_size(Thread::current()) - sizeof(_table);
316 }
317
318 size_t log_table_size() { return _table.get_size_log2(Thread::current()); }
319};
320
321void* G1CardSetHashTableConfig::allocate_node(void* context, size_t size, Value const& value) {
322 G1CardSetMemoryManager* mm = (G1CardSetMemoryManager*)context;
323 return mm->allocate_node();
324}
325
326void G1CardSetHashTableConfig::free_node(void* context, void* memory, Value const& value) {
327 G1CardSetMemoryManager* mm = (G1CardSetMemoryManager*)context;
328 mm->free_node(memory);
329}
330
331G1CardSetCoarsenStats G1CardSet::_coarsen_stats;
332G1CardSetCoarsenStats G1CardSet::_last_coarsen_stats;
333
334G1CardSet::G1CardSet(G1CardSetConfiguration* config, G1CardSetMemoryManager* mm) :
335 _mm(mm),
336 _config(config),
337 _table(new G1CardSetHashTable(mm)),
338 _num_occupied(0) {
339}
340
341G1CardSet::~G1CardSet() {
342 delete _table;
343 _mm->flush();
344}
345
346uint G1CardSet::card_set_type_to_mem_object_type(uintptr_t type) const {
347 assert(type == G1CardSet::CardSetArrayOfCards ||do { if (!(type == G1CardSet::CardSetArrayOfCards || type == G1CardSet
::CardSetBitMap || type == G1CardSet::CardSetHowl)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 349, "assert(" "type == G1CardSet::CardSetArrayOfCards || type == G1CardSet::CardSetBitMap || type == G1CardSet::CardSetHowl"
") failed", "should not allocate card set type %zu", type); ::
breakpoint(); } } while (0)
348 type == G1CardSet::CardSetBitMap ||do { if (!(type == G1CardSet::CardSetArrayOfCards || type == G1CardSet
::CardSetBitMap || type == G1CardSet::CardSetHowl)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 349, "assert(" "type == G1CardSet::CardSetArrayOfCards || type == G1CardSet::CardSetBitMap || type == G1CardSet::CardSetHowl"
") failed", "should not allocate card set type %zu", type); ::
breakpoint(); } } while (0)
349 type == G1CardSet::CardSetHowl, "should not allocate card set type %zu", type)do { if (!(type == G1CardSet::CardSetArrayOfCards || type == G1CardSet
::CardSetBitMap || type == G1CardSet::CardSetHowl)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 349, "assert(" "type == G1CardSet::CardSetArrayOfCards || type == G1CardSet::CardSetBitMap || type == G1CardSet::CardSetHowl"
") failed", "should not allocate card set type %zu", type); ::
breakpoint(); } } while (0);
350
351 return (uint)type;
352}
353
354uint8_t* G1CardSet::allocate_mem_object(uintptr_t type) {
355 return _mm->allocate(card_set_type_to_mem_object_type(type));
356}
357
358void G1CardSet::free_mem_object(CardSetPtr card_set) {
359 assert(card_set != G1CardSet::FreeCardSet, "should not free Free card set")do { if (!(card_set != G1CardSet::FreeCardSet)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 359, "assert(" "card_set != G1CardSet::FreeCardSet" ") failed"
, "should not free Free card set"); ::breakpoint(); } } while
(0);
360 assert(card_set != G1CardSet::FullCardSet, "should not free Full card set")do { if (!(card_set != G1CardSet::FullCardSet)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 360, "assert(" "card_set != G1CardSet::FullCardSet" ") failed"
, "should not free Full card set"); ::breakpoint(); } } while
(0);
361
362 uintptr_t type = card_set_type(card_set);
363 void* value = strip_card_set_type(card_set);
364
365 assert(type == G1CardSet::CardSetArrayOfCards ||do { if (!(type == G1CardSet::CardSetArrayOfCards || type == G1CardSet
::CardSetBitMap || type == G1CardSet::CardSetHowl)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 367, "assert(" "type == G1CardSet::CardSetArrayOfCards || type == G1CardSet::CardSetBitMap || type == G1CardSet::CardSetHowl"
") failed", "should not free card set type %zu", type); ::breakpoint
(); } } while (0)
366 type == G1CardSet::CardSetBitMap ||do { if (!(type == G1CardSet::CardSetArrayOfCards || type == G1CardSet
::CardSetBitMap || type == G1CardSet::CardSetHowl)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 367, "assert(" "type == G1CardSet::CardSetArrayOfCards || type == G1CardSet::CardSetBitMap || type == G1CardSet::CardSetHowl"
") failed", "should not free card set type %zu", type); ::breakpoint
(); } } while (0)
367 type == G1CardSet::CardSetHowl, "should not free card set type %zu", type)do { if (!(type == G1CardSet::CardSetArrayOfCards || type == G1CardSet
::CardSetBitMap || type == G1CardSet::CardSetHowl)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 367, "assert(" "type == G1CardSet::CardSetArrayOfCards || type == G1CardSet::CardSetBitMap || type == G1CardSet::CardSetHowl"
") failed", "should not free card set type %zu", type); ::breakpoint
(); } } while (0);
368
369#ifdef ASSERT1
370 if (type == G1CardSet::CardSetArrayOfCards ||
371 type == G1CardSet::CardSetBitMap ||
372 type == G1CardSet::CardSetHowl) {
373 G1CardSetContainer* card_set = (G1CardSetContainer*)value;
374 assert((card_set->refcount() == 1), "must be")do { if (!((card_set->refcount() == 1))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 374, "assert(" "(card_set->refcount() == 1)" ") failed",
"must be"); ::breakpoint(); } } while (0);
375 }
376#endif
377
378 _mm->free(card_set_type_to_mem_object_type(type), value);
379}
380
381G1CardSet::CardSetPtr G1CardSet::acquire_card_set(CardSetPtr volatile* card_set_addr) {
382 // Update reference counts under RCU critical section to avoid a
383 // use-after-cleapup bug where we increment a reference count for
384 // an object whose memory has already been cleaned up and reused.
385 GlobalCounter::CriticalSection cs(Thread::current());
386 while (true) {
387 // Get cardsetptr and increment refcount atomically wrt to memory reuse.
388 CardSetPtr card_set = Atomic::load_acquire(card_set_addr);
389 uint cs_type = card_set_type(card_set);
390 if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
391 return card_set;
392 }
393
394 G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
395
396 if (card_set_on_heap->try_increment_refcount()) {
397 assert(card_set_on_heap->refcount() >= 3, "Smallest value is 3")do { if (!(card_set_on_heap->refcount() >= 3)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 397, "assert(" "card_set_on_heap->refcount() >= 3" ") failed"
, "Smallest value is 3"); ::breakpoint(); } } while (0);
398 return card_set;
399 }
400 }
401}
402
403bool G1CardSet::release_card_set(CardSetPtr card_set) {
404 uint cs_type = card_set_type(card_set);
405 if (card_set == FullCardSet || cs_type == CardSetInlinePtr) {
406 return false;
407 }
408
409 G1CardSetContainer* card_set_on_heap = (G1CardSetContainer*)strip_card_set_type(card_set);
410 return card_set_on_heap->decrement_refcount() == 1;
411}
412
413void G1CardSet::release_and_maybe_free_card_set(CardSetPtr card_set) {
414 if (release_card_set(card_set)) {
415 free_mem_object(card_set);
416 }
417}
418
419void G1CardSet::release_and_must_free_card_set(CardSetPtr card_set) {
420 bool should_free = release_card_set(card_set);
421 assert(should_free, "should have been the only one having a reference")do { if (!(should_free)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 421, "assert(" "should_free" ") failed", "should have been the only one having a reference"
); ::breakpoint(); } } while (0);
422 free_mem_object(card_set);
423}
424
425class G1ReleaseCardsets : public StackObj {
426 G1CardSet* _card_set;
427 using CardSetPtr = G1CardSet::CardSetPtr;
428
429 void coarsen_to_full(CardSetPtr* card_set_addr) {
430 while (true) {
431 CardSetPtr cur_card_set = Atomic::load_acquire(card_set_addr);
432 uint cs_type = G1CardSet::card_set_type(cur_card_set);
433 if (cur_card_set == G1CardSet::FullCardSet) {
434 return;
435 }
436
437 CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, G1CardSet::FullCardSet);
438
439 if (old_value == cur_card_set) {
440 _card_set->release_and_maybe_free_card_set(cur_card_set);
441 return;
442 }
443 }
444 }
445
446public:
447 explicit G1ReleaseCardsets(G1CardSet* card_set) : _card_set(card_set) { }
448
449 void operator ()(CardSetPtr* card_set_addr) {
450 coarsen_to_full(card_set_addr);
451 }
452};
453
454G1AddCardResult G1CardSet::add_to_array(CardSetPtr card_set, uint card_in_region) {
455 G1CardSetArray* array = card_set_ptr<G1CardSetArray>(card_set);
456 return array->add(card_in_region);
457}
458
459G1AddCardResult G1CardSet::add_to_howl(CardSetPtr parent_card_set,
460 uint card_region,
461 uint card_in_region,
462 bool increment_total) {
463 G1CardSetHowl* howl = card_set_ptr<G1CardSetHowl>(parent_card_set);
464
465 G1AddCardResult add_result;
466 CardSetPtr to_transfer = nullptr;
467 CardSetPtr card_set;
468
469 uint bucket = _config->howl_bucket_index(card_in_region);
470 volatile CardSetPtr* bucket_entry = howl->get_card_set_addr(bucket);
471
472 while (true) {
473 if (Atomic::load(&howl->_num_entries) >= _config->cards_in_howl_threshold()) {
474 return Overflow;
475 }
476
477 card_set = acquire_card_set(bucket_entry);
478 add_result = add_to_card_set(bucket_entry, card_set, card_region, card_in_region);
479
480 if (add_result != Overflow) {
481 break;
482 }
483 // Card set has overflown. Coarsen or retry.
484 bool coarsened = coarsen_card_set(bucket_entry, card_set, card_in_region, true /* within_howl */);
485 _coarsen_stats.record_coarsening(card_set_type(card_set) + G1CardSetCoarsenStats::CoarsenHowlOffset, !coarsened);
486 if (coarsened) {
487 // We have been the one coarsening this card set (and in the process added that card).
488 add_result = Added;
489 to_transfer = card_set;
490 break;
491 }
492 // Somebody else beat us to coarsening. Retry.
493 release_and_maybe_free_card_set(card_set);
494 }
495
496 if (increment_total && add_result == Added) {
497 Atomic::inc(&howl->_num_entries, memory_order_relaxed);
498 }
499
500 if (to_transfer != nullptr) {
501 transfer_cards_in_howl(parent_card_set, to_transfer, card_region);
502 }
503
504 release_and_maybe_free_card_set(card_set);
505 return add_result;
506}
507
508G1AddCardResult G1CardSet::add_to_bitmap(CardSetPtr card_set, uint card_in_region) {
509 G1CardSetBitMap* bitmap = card_set_ptr<G1CardSetBitMap>(card_set);
510 uint card_offset = _config->howl_bitmap_offset(card_in_region);
511 return bitmap->add(card_offset, _config->cards_in_howl_bitmap_threshold(), _config->max_cards_in_howl_bitmap());
512}
513
514G1AddCardResult G1CardSet::add_to_inline_ptr(CardSetPtr volatile* card_set_addr, CardSetPtr card_set, uint card_in_region) {
515 G1CardSetInlinePtr value(card_set_addr, card_set);
516 return value.add(card_in_region, _config->inline_ptr_bits_per_card(), _config->max_cards_in_inline_ptr());
517}
518
519G1CardSet::CardSetPtr G1CardSet::create_coarsened_array_of_cards(uint card_in_region, bool within_howl) {
520 uint8_t* data = nullptr;
521 CardSetPtr new_card_set;
522 if (within_howl) {
523 uint const size_in_bits = _config->max_cards_in_howl_bitmap();
524 uint card_offset = _config->howl_bitmap_offset(card_in_region);
525 data = allocate_mem_object(CardSetBitMap);
526 new (data) G1CardSetBitMap(card_offset, size_in_bits);
527 new_card_set = make_card_set_ptr(data, CardSetBitMap);
528 } else {
529 data = allocate_mem_object(CardSetHowl);
530 new (data) G1CardSetHowl(card_in_region, _config);
531 new_card_set = make_card_set_ptr(data, CardSetHowl);
532 }
533 return new_card_set;
534}
535
536bool G1CardSet::coarsen_card_set(volatile CardSetPtr* card_set_addr,
537 CardSetPtr cur_card_set,
538 uint card_in_region,
539 bool within_howl) {
540 CardSetPtr new_card_set = nullptr;
541
542 switch (card_set_type(cur_card_set)) {
543 case CardSetArrayOfCards : {
544 new_card_set = create_coarsened_array_of_cards(card_in_region, within_howl);
545 break;
546 }
547 case CardSetBitMap: {
548 new_card_set = FullCardSet;
549 break;
550 }
551 case CardSetInlinePtr: {
552 uint const size = _config->max_cards_in_array();
553 uint8_t* data = allocate_mem_object(CardSetArrayOfCards);
554 new (data) G1CardSetArray(card_in_region, size);
555 new_card_set = make_card_set_ptr(data, CardSetArrayOfCards);
556 break;
557 }
558 case CardSetHowl: {
559 new_card_set = FullCardSet; // anything will do at this point.
560 break;
561 }
562 default:
563 ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here(
"/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 563); ::breakpoint(); } while (0);
564 }
565
566 CardSetPtr old_value = Atomic::cmpxchg(card_set_addr, cur_card_set, new_card_set); // Memory order?
567 if (old_value == cur_card_set) {
568 // Success. Indicate that the cards from the current card set must be transferred
569 // by this caller.
570 // Release the hash table reference to the card. The caller still holds the
571 // reference to this card set, so it can never be released (and we do not need to
572 // check its result).
573 bool should_free = release_card_set(cur_card_set);
574 assert(!should_free, "must have had more than one reference")do { if (!(!should_free)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 574, "assert(" "!should_free" ") failed", "must have had more than one reference"
); ::breakpoint(); } } while (0);
575 // Free containers if cur_card_set is CardSetHowl
576 if (card_set_type(cur_card_set) == CardSetHowl) {
577 G1ReleaseCardsets rel(this);
578 card_set_ptr<G1CardSetHowl>(cur_card_set)->iterate(rel, _config->num_buckets_in_howl());
579 }
580 return true;
581 } else {
582 // Somebody else beat us to coarsening that card set. Exit, but clean up first.
583 if (new_card_set != FullCardSet) {
584 assert(new_card_set != nullptr, "must not be")do { if (!(new_card_set != nullptr)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 584, "assert(" "new_card_set != nullptr" ") failed", "must not be"
); ::breakpoint(); } } while (0);
585 release_and_must_free_card_set(new_card_set);
586 }
587 return false;
588 }
589}
590
591class G1TransferCard : public StackObj {
592 G1CardSet* _card_set;
593 uint _region_idx;
594public:
595 G1TransferCard(G1CardSet* card_set, uint region_idx) : _card_set(card_set), _region_idx(region_idx) { }
596
597 void operator ()(uint card_idx) {
598 _card_set->add_card(_region_idx, card_idx, false);
9
←
Calling 'G1CardSet::add_card'
→
599 }
600};
601
602void G1CardSet::transfer_cards(G1CardSetHashTableValue* table_entry, CardSetPtr source_card_set, uint card_region) {
603 assert(source_card_set != FullCardSet, "Should not need to transfer from full")do { if (!(source_card_set != FullCardSet)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 603, "assert(" "source_card_set != FullCardSet" ") failed",
"Should not need to transfer from full"); ::breakpoint(); } }
while (0);
604 // Need to transfer old entries unless there is a Full card set in place now, i.e.
605 // the old type has been CardSetBitMap. "Full" contains all elements anyway.
606 if (card_set_type(source_card_set) != CardSetHowl) {
607 G1TransferCard iter(this, card_region);
608 iterate_cards_during_transfer(source_card_set, iter);
609 } else {
610 assert(card_set_type(source_card_set) == CardSetHowl, "must be")do { if (!(card_set_type(source_card_set) == CardSetHowl)) { (
*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 610, "assert(" "card_set_type(source_card_set) == CardSetHowl"
") failed", "must be"); ::breakpoint(); } } while (0);
611 // Need to correct for that the Full remembered set occupies more cards than the
612 // AoCS before.
613 Atomic::add(&_num_occupied, _config->max_cards_in_region() - table_entry->_num_occupied, memory_order_relaxed);
614 }
615}
616
617void G1CardSet::transfer_cards_in_howl(CardSetPtr parent_card_set,
618 CardSetPtr source_card_set,
619 uint card_region) {
620 assert(card_set_type(parent_card_set) == CardSetHowl, "must be")do { if (!(card_set_type(parent_card_set) == CardSetHowl)) { (
*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 620, "assert(" "card_set_type(parent_card_set) == CardSetHowl"
") failed", "must be"); ::breakpoint(); } } while (0);
621 assert(source_card_set != FullCardSet, "Should not need to transfer from full")do { if (!(source_card_set != FullCardSet)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 621, "assert(" "source_card_set != FullCardSet" ") failed",
"Should not need to transfer from full"); ::breakpoint(); } }
while (0);
622 // Need to transfer old entries unless there is a Full card set in place now, i.e.
623 // the old type has been CardSetBitMap.
624 if (card_set_type(source_card_set) != CardSetBitMap) {
625 // We only need to transfer from anything below CardSetBitMap.
626 G1TransferCard iter(this, card_region);
627 iterate_cards_during_transfer(source_card_set, iter);
628 } else {
629 uint diff = _config->max_cards_in_howl_bitmap() - card_set_ptr<G1CardSetBitMap>(source_card_set)->num_bits_set();
630
631 // Need to correct for that the Full remembered set occupies more cards than the
632 // bitmap before.
633 // We add 1 card less because the values will be incremented
634 // in G1CardSet::add_card for the current addition or where already incremented in
635 // G1CardSet::add_to_howl after coarsening.
636 diff -= 1;
637
638 G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(parent_card_set);
639 Atomic::add(&howling_array->_num_entries, diff, memory_order_relaxed);
640
641 G1CardSetHashTableValue* table_entry = get_card_set(card_region);
642 assert(table_entry != nullptr, "Table entry not found for transferred cards")do { if (!(table_entry != nullptr)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 642, "assert(" "table_entry != nullptr" ") failed", "Table entry not found for transferred cards"
); ::breakpoint(); } } while (0);
643
644 Atomic::add(&table_entry->_num_occupied, diff, memory_order_relaxed);
645
646 Atomic::add(&_num_occupied, diff, memory_order_relaxed);
647 }
648}
649
650G1AddCardResult G1CardSet::add_to_card_set(volatile CardSetPtr* card_set_addr, CardSetPtr card_set, uint card_region, uint card_in_region, bool increment_total) {
651 assert(card_set_addr != nullptr, "Cannot add to empty cardset")do { if (!(card_set_addr != nullptr)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 651, "assert(" "card_set_addr != nullptr" ") failed", "Cannot add to empty cardset"
); ::breakpoint(); } } while (0);
12
←
Taking false branch
→
13
←
Loop condition is false. Exiting loop
→
652
653 G1AddCardResult add_result;
14
←
'add_result' declared without an initial value
→
654
655 switch (card_set_type(card_set)) {
15
←
Control jumps to the 'default' case at line 676
→
656 case CardSetInlinePtr: {
657 add_result = add_to_inline_ptr(card_set_addr, card_set, card_in_region);
658 break;
659 }
660 case CardSetArrayOfCards : {
661 add_result = add_to_array(card_set, card_in_region);
662 break;
663 }
664 case CardSetBitMap: {
665 add_result = add_to_bitmap(card_set, card_in_region);
666 break;
667 }
668 case CardSetHowl: {
669 assert(CardSetHowl == card_set_type(FullCardSet), "must be")do { if (!(CardSetHowl == card_set_type(FullCardSet))) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 669, "assert(" "CardSetHowl == card_set_type(FullCardSet)" ") failed"
, "must be"); ::breakpoint(); } } while (0);
670 if (card_set == FullCardSet) {
671 return Found;
672 }
673 add_result = add_to_howl(card_set, card_region, card_in_region, increment_total);
674 break;
675 }
676 default:
677 ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here(
"/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 677); ::breakpoint(); } while (0);
16
←
Loop condition is false. Exiting loop
→
678 }
679
680 return add_result;
17
←
Undefined or garbage value returned to caller
681}
682
683G1CardSetHashTableValue* G1CardSet::get_or_add_card_set(uint card_region, bool* should_grow_table) {
684 return _table->get_or_add(card_region, should_grow_table);
685}
686
687G1CardSetHashTableValue* G1CardSet::get_card_set(uint card_region) {
688 return _table->get(card_region);
689}
690
691G1AddCardResult G1CardSet::add_card(uint card_region, uint card_in_region, bool increment_total) {
692 G1AddCardResult add_result;
693 CardSetPtr to_transfer = nullptr;
694 CardSetPtr card_set;
695
696 bool should_grow_table = false;
697 G1CardSetHashTableValue* table_entry = get_or_add_card_set(card_region, &should_grow_table);
698 while (true) {
10
←
Loop condition is true. Entering loop body
→
699 card_set = acquire_card_set(&table_entry->_card_set);
700 add_result = add_to_card_set(&table_entry->_card_set, card_set, card_region, card_in_region, increment_total);
11
←
Calling 'G1CardSet::add_to_card_set'
→
701
702 if (add_result != Overflow) {
703 break;
704 }
705 // Card set has overflown. Coarsen or retry.
706 bool coarsened = coarsen_card_set(&table_entry->_card_set, card_set, card_in_region);
707 _coarsen_stats.record_coarsening(card_set_type(card_set), !coarsened);
708 if (coarsened) {
709 // We have been the one coarsening this card set (and in the process added that card).
710 add_result = Added;
711 to_transfer = card_set;
712 break;
713 }
714 // Somebody else beat us to coarsening. Retry.
715 release_and_maybe_free_card_set(card_set);
716 }
717
718 if (increment_total && add_result == Added) {
719 Atomic::inc(&table_entry->_num_occupied, memory_order_relaxed);
720 Atomic::inc(&_num_occupied, memory_order_relaxed);
721 }
722 if (should_grow_table) {
723 _table->grow();
724 }
725 if (to_transfer != nullptr) {
726 transfer_cards(table_entry, to_transfer, card_region);
727 }
728
729 release_and_maybe_free_card_set(card_set);
730
731 return add_result;
732}
733
734bool G1CardSet::contains_card(uint card_region, uint card_in_region) {
735 assert(card_in_region < _config->max_cards_in_region(),do { if (!(card_in_region < _config->max_cards_in_region
())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 736, "assert(" "card_in_region < _config->max_cards_in_region()"
") failed", "Card %u is beyond max %u", card_in_region, _config
->max_cards_in_region()); ::breakpoint(); } } while (0)
736 "Card %u is beyond max %u", card_in_region, _config->max_cards_in_region())do { if (!(card_in_region < _config->max_cards_in_region
())) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 736, "assert(" "card_in_region < _config->max_cards_in_region()"
") failed", "Card %u is beyond max %u", card_in_region, _config
->max_cards_in_region()); ::breakpoint(); } } while (0);
737
738 // Protect the card set from reclamation.
739 GlobalCounter::CriticalSection cs(Thread::current());
740 G1CardSetHashTableValue* table_entry = get_card_set(card_region);
741 if (table_entry == nullptr) {
742 return false;
743 }
744
745 CardSetPtr card_set = table_entry->_card_set;
746 if (card_set == FullCardSet) {
747 // contains_card() is not a performance critical method so we do not hide that
748 // case in the switch below.
749 return true;
750 }
751
752 switch (card_set_type(card_set)) {
753 case CardSetInlinePtr: {
754 G1CardSetInlinePtr ptr(card_set);
755 return ptr.contains(card_in_region, _config->inline_ptr_bits_per_card());
756 }
757 case CardSetArrayOfCards : return card_set_ptr<G1CardSetArray>(card_set)->contains(card_in_region);
758 case CardSetBitMap: return card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_in_region, _config->max_cards_in_howl_bitmap());
759 case CardSetHowl: {
760 G1CardSetHowl* howling_array = card_set_ptr<G1CardSetHowl>(card_set);
761
762 return howling_array->contains(card_in_region, _config);
763 }
764 }
765 ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here(
"/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 765); ::breakpoint(); } while (0);
766 return false;
767}
768
769void G1CardSet::print_info(outputStream* st, uint card_region, uint card_in_region) {
770 G1CardSetHashTableValue* table_entry = get_card_set(card_region);
771 if (table_entry == nullptr) {
772 st->print("NULL card set");
773 return;
774 }
775
776 CardSetPtr card_set = table_entry->_card_set;
777 if (card_set == FullCardSet) {
778 st->print("FULL card set)");
779 return;
780 }
781 switch (card_set_type(card_set)) {
782 case CardSetInlinePtr: {
783 st->print("InlinePtr not containing %u", card_in_region);
784 break;
785 }
786 case CardSetArrayOfCards : {
787 st->print("AoC not containing %u", card_in_region);
788 break;
789 }
790 case CardSetBitMap: {
791 st->print("BitMap not containing %u", card_in_region);
792 break;
793 }
794 case CardSetHowl: {
795 st->print("CardSetHowl not containing %u", card_in_region);
796 break;
797 }
798 default: st->print("Unknown card set type %u", card_set_type(card_set)); ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here(
"/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 798); ::breakpoint(); } while (0); break;
799 }
800}
801
802template <class CardVisitor>
803void G1CardSet::iterate_cards_during_transfer(CardSetPtr const card_set, CardVisitor& cl) {
804 uint type = card_set_type(card_set);
805 assert(type == CardSetInlinePtr || type == CardSetArrayOfCards,do { if (!(type == CardSetInlinePtr || type == CardSetArrayOfCards
)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 807, "assert(" "type == CardSetInlinePtr || type == CardSetArrayOfCards"
") failed", "invalid card set type %d to transfer from", card_set_type
(card_set)); ::breakpoint(); } } while (0)
1
Assuming 'type' is equal to 'CardSetInlinePtr'
→
2
←
Taking false branch
→
3
←
Loop condition is false. Exiting loop
→
806 "invalid card set type %d to transfer from",do { if (!(type == CardSetInlinePtr || type == CardSetArrayOfCards
)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 807, "assert(" "type == CardSetInlinePtr || type == CardSetArrayOfCards"
") failed", "invalid card set type %d to transfer from", card_set_type
(card_set)); ::breakpoint(); } } while (0)
807 card_set_type(card_set))do { if (!(type == CardSetInlinePtr || type == CardSetArrayOfCards
)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 807, "assert(" "type == CardSetInlinePtr || type == CardSetArrayOfCards"
") failed", "invalid card set type %d to transfer from", card_set_type
(card_set)); ::breakpoint(); } } while (0);
808
809 switch (type) {
4
←
Control jumps to 'case 0:' at line 810
→
810 case CardSetInlinePtr: {
811 G1CardSetInlinePtr ptr(card_set);
812 ptr.iterate(cl, _config->inline_ptr_bits_per_card());
5
←
Calling 'G1CardSetInlinePtr::iterate'
→
813 return;
814 }
815 case CardSetArrayOfCards : {
816 card_set_ptr<G1CardSetArray>(card_set)->iterate(cl);
817 return;
818 }
819 default:
820 ShouldNotReachHere()do { (*g_assert_poison) = 'X';; report_should_not_reach_here(
"/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSet.cpp"
, 820); ::breakpoint(); } while (0);
821 }
822}
823
824void G1CardSet::iterate_containers(CardSetPtrClosure* cl, bool at_safepoint) {
825 if (at_safepoint) {
826 _table->iterate_safepoint(cl);
827 } else {
828 _table->iterate(cl);
829 }
830}
831
832// Applied to all card (ranges) of the containers.
833template <typename Closure>
834class G1ContainerCardsClosure {
835 Closure& _cl;
836 uint _region_idx;
837
838public:
839 G1ContainerCardsClosure(Closure& cl, uint region_idx) : _cl(cl), _region_idx(region_idx) { }
840
841 bool start_iterate(uint tag) { return true; }
842
843 void operator()(uint card_idx) {
844 _cl.do_card(_region_idx, card_idx);
845 }
846
847 void operator()(uint card_idx, uint length) {
848 for (uint i = 0; i < length; i++) {
849 _cl.do_card(_region_idx, card_idx);
850 }
851 }
852};
853
854template <typename Closure, template <typename> class CardOrRanges>
855class G1CardSetContainersClosure : public G1CardSet::CardSetPtrClosure {
856 G1CardSet* _card_set;
857 Closure& _cl;
858
859public:
860
861 G1CardSetContainersClosure(G1CardSet* card_set,
862 Closure& cl) :
863 _card_set(card_set),
864 _cl(cl) { }
865
866 void do_cardsetptr(uint region_idx, size_t num_occupied, G1CardSet::CardSetPtr card_set) override {
867 CardOrRanges<Closure> cl(_cl, region_idx);
868 _card_set->iterate_cards_or_ranges_in_container(card_set, cl);
869 }
870};
871
872void G1CardSet::iterate_cards(CardClosure& cl) {
873 G1CardSetContainersClosure<CardClosure, G1ContainerCardsClosure> cl2(this, cl);
874 iterate_containers(&cl2);
875}
876
877bool G1CardSet::occupancy_less_or_equal_to(size_t limit) const {
878 return occupied() <= limit;
879}
880
881bool G1CardSet::is_empty() const {
882 return _num_occupied == 0;
883}
884
885size_t G1CardSet::occupied() const {
886 return _num_occupied;
887}
888
889size_t G1CardSet::num_containers() {
890 class GetNumberOfContainers : public CardSetPtrClosure {
891 public:
892 size_t _count;
893
894 GetNumberOfContainers() : CardSetPtrClosure(), _count(0) { }
895
896 void do_cardsetptr(uint region_idx, size_t num_occupied, CardSetPtr card_set) override {
897 _count++;
898 }
899 } cl;
900
901 iterate_containers(&cl);
902 return cl._count;
903}
904
905G1CardSetCoarsenStats G1CardSet::coarsen_stats() {
906 return _coarsen_stats;
907}
908
909void G1CardSet::print_coarsen_stats(outputStream* out) {
910 _last_coarsen_stats.subtract_from(_coarsen_stats);
911 out->print("Coarsening (recent): ");
912 _last_coarsen_stats.print_on(out);
913 out->print("Coarsening (all): ");
914 _coarsen_stats.print_on(out);
915}
916
917size_t G1CardSet::mem_size() const {
918 return sizeof(*this) +
919 _table->mem_size() +
920 _mm->mem_size();
921}
922
923size_t G1CardSet::wasted_mem_size() const {
924 return _mm->wasted_mem_size();
925}
926
927size_t G1CardSet::static_mem_size() {
928 return sizeof(FullCardSet) + sizeof(_coarsen_stats);
929}
930
931void G1CardSet::clear() {
932 _table->reset();
933 _num_occupied = 0;
934 _mm->flush();
935}
936
937void G1CardSet::print(outputStream* os) {
938 _table->print(os);
939 _mm->print(os);
940}
←

/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp

1/*
2 * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25#ifndef SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP
26#define SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP
27
28#include "gc/g1/g1CardSetContainers.hpp"
29#include "gc/g1/g1GCPhaseTimes.hpp"
30#include "utilities/globalDefinitions.hpp"
31
32inline G1CardSetInlinePtr::CardSetPtr G1CardSetInlinePtr::merge(CardSetPtr orig_value, uint card_in_region, uint idx, uint bits_per_card) {
33 assert((idx & (SizeFieldMask >> SizeFieldPos)) == idx, "Index %u too large to fit into size field", idx)do { if (!((idx & (SizeFieldMask >> SizeFieldPos)) ==
idx)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 33, "assert(" "(idx & (SizeFieldMask >> SizeFieldPos)) == idx"
") failed", "Index %u too large to fit into size field", idx
); ::breakpoint(); } } while (0);
34 assert(card_in_region < ((uint)1 << bits_per_card), "Card %u too large to fit into card value field", card_in_region)do { if (!(card_in_region < ((uint)1 << bits_per_card
))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 34, "assert(" "card_in_region < ((uint)1 << bits_per_card)"
") failed", "Card %u too large to fit into card value field"
, card_in_region); ::breakpoint(); } } while (0);
35
36 uint8_t card_pos = card_pos_for(idx, bits_per_card);
37 assert(card_pos + bits_per_card < BitsInValue, "Putting card at pos %u with %u bits would extend beyond pointer", card_pos, bits_per_card)do { if (!(card_pos + bits_per_card < BitsInValue)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 37, "assert(" "card_pos + bits_per_card < BitsInValue" ") failed"
, "Putting card at pos %u with %u bits would extend beyond pointer"
, card_pos, bits_per_card); ::breakpoint(); } } while (0);
38
39 // Check that we do not touch any fields we do not own.
40 uintptr_t mask = ((((uintptr_t)1 << bits_per_card) - 1) << card_pos);
41 assert(((uintptr_t)orig_value & mask) == 0, "The bits in the new range should be empty; orig_value " PTR_FORMAT " mask " PTR_FORMAT, p2i(orig_value), mask)do { if (!(((uintptr_t)orig_value & mask) == 0)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 41, "assert(" "((uintptr_t)orig_value & mask) == 0" ") failed"
, "The bits in the new range should be empty; orig_value " "0x%016"
"l" "x" " mask " "0x%016" "l" "x", p2i(orig_value), mask); ::
breakpoint(); } } while (0);
42
43 uintptr_t value = ((uintptr_t)(idx + 1) << SizeFieldPos) | ((uintptr_t)card_in_region << card_pos);
44 uintptr_t res = (((uintptr_t)orig_value & ~SizeFieldMask) | value);
45 return (CardSetPtr)res;
46}
47
48inline G1AddCardResult G1CardSetInlinePtr::add(uint card_idx, uint bits_per_card, uint max_cards_in_inline_ptr) {
49 assert(_value_addr != nullptr, "No value address available, cannot add to set.")do { if (!(_value_addr != nullptr)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 49, "assert(" "_value_addr != nullptr" ") failed", "No value address available, cannot add to set."
); ::breakpoint(); } } while (0);
50
51 uint cur_idx = 0;
52 while (true) {
53 uint num_cards = num_cards_in(_value);
54 if (num_cards > 0) {
55 cur_idx = find(card_idx, bits_per_card, cur_idx, num_cards);
56 }
57 // Check if the card is already stored in the pointer.
58 if (cur_idx < num_cards) {
59 return Found;
60 }
61 // Check if there is actually enough space.
62 if (num_cards >= max_cards_in_inline_ptr) {
63 return Overflow;
64 }
65 CardSetPtr new_value = merge(_value, card_idx, num_cards, bits_per_card);
66 CardSetPtr old_value = Atomic::cmpxchg(_value_addr, _value, new_value, memory_order_relaxed);
67 if (_value == old_value) {
68 return Added;
69 }
70 // Update values and retry.
71 _value = old_value;
72 // The value of the pointer may have changed to something different than
73 // an inline card set. Exit then instead of overwriting.
74 if (G1CardSet::card_set_type(_value) != G1CardSet::CardSetInlinePtr) {
75 return Overflow;
76 }
77 }
78}
79
80inline uint G1CardSetInlinePtr::find(uint card_idx, uint bits_per_card, uint start_at, uint num_cards) {
81 assert(start_at < num_cards, "Precondition!")do { if (!(start_at < num_cards)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 81, "assert(" "start_at < num_cards" ") failed", "Precondition!"
); ::breakpoint(); } } while (0);
82
83 uintptr_t const card_mask = (1 << bits_per_card) - 1;
84 uintptr_t value = ((uintptr_t)_value) >> card_pos_for(start_at, bits_per_card);
85
86 // Check if the card is already stored in the pointer.
87 for (uint cur_idx = start_at; cur_idx < num_cards; cur_idx++) {
88 if ((value & card_mask) == card_idx) {
89 return cur_idx;
90 }
91 value >>= bits_per_card;
92 }
93 return num_cards;
94}
95
96inline bool G1CardSetInlinePtr::contains(uint card_idx, uint bits_per_card) {
97 uint num_cards = num_cards_in(_value);
98 if (num_cards == 0) {
99 return false;
100 }
101 uint cur_idx = find(card_idx, bits_per_card, 0, num_cards);
102 return cur_idx < num_cards;
103}
104
105template <class CardVisitor>
106inline void G1CardSetInlinePtr::iterate(CardVisitor& found, uint bits_per_card) {
107 uint const num_cards = num_cards_in(_value);
108 uintptr_t const card_mask = (1 << bits_per_card) - 1;
109
110 uintptr_t value = ((uintptr_t)_value) >> card_pos_for(0, bits_per_card);
111 for (uint cur_idx = 0; cur_idx < num_cards; cur_idx++) {
6
←
Assuming 'cur_idx' is < 'num_cards'
→
7
←
Loop condition is true. Entering loop body
→
112 found(value & card_mask);
8
←
Calling 'G1TransferCard::operator()'
→
113 value >>= bits_per_card;
114 }
115}
116
117inline bool G1CardSetContainer::try_increment_refcount() {
118 uintptr_t old_value = refcount();
119 while (true) {
120 if (old_value < 3 || (old_value & 0x1) == 0) { // reclaimed, reference counts are odd numbers starting at 3
121 return false; // dead, can't revive.
122 }
123
124 uintptr_t new_value = old_value + 2;
125 uintptr_t ref_count = Atomic::cmpxchg(&_ref_count, old_value, new_value);
126 if (ref_count == old_value) {
127 return true;
128 }
129 old_value = ref_count;
130 }
131}
132
133inline uintptr_t G1CardSetContainer::decrement_refcount() {
134 uintptr_t old_value = refcount();
135 assert((old_value & 0x1) != 0 && old_value >= 3, "precondition")do { if (!((old_value & 0x1) != 0 && old_value >=
3)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 135, "assert(" "(old_value & 0x1) != 0 && old_value >= 3"
") failed", "precondition"); ::breakpoint(); } } while (0);
136 return Atomic::sub(&_ref_count, 2u);
137}
138
139inline G1CardSetArray::G1CardSetArray(uint card_in_region, EntryCountType num_cards) :
140 G1CardSetContainer(),
141 _size(num_cards),
142 _num_entries(1) {
143 assert(_size > 0, "CardSetArray of size 0 not supported.")do { if (!(_size > 0)) { (*g_assert_poison) = 'X';; report_vm_error
("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 143, "assert(" "_size > 0" ") failed", "CardSetArray of size 0 not supported."
); ::breakpoint(); } } while (0);
144 assert(_size < LockBitMask, "Only support CardSetArray of size %u or smaller.", LockBitMask - 1)do { if (!(_size < LockBitMask)) { (*g_assert_poison) = 'X'
;; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 144, "assert(" "_size < LockBitMask" ") failed", "Only support CardSetArray of size %u or smaller."
, LockBitMask - 1); ::breakpoint(); } } while (0);
145 _data[0] = card_in_region;
146}
147
148inline G1CardSetArray::G1CardSetArrayLocker::G1CardSetArrayLocker(EntryCountType volatile* num_entries_addr) :
149 _num_entries_addr(num_entries_addr) {
150 SpinYield s;
151 EntryCountType num_entries = Atomic::load(_num_entries_addr) & EntryMask;
152 while (true) {
153 EntryCountType old_value = Atomic::cmpxchg(_num_entries_addr,
154 num_entries,
155 (EntryCountType)(num_entries | LockBitMask));
156 if (old_value == num_entries) {
157 // Succeeded locking the array.
158 _local_num_entries = num_entries;
159 break;
160 }
161 // Failed. Retry (with the lock bit stripped again).
162 num_entries = old_value & EntryMask;
163 s.wait();
164 }
165}
166
167inline G1AddCardResult G1CardSetArray::add(uint card_idx) {
168 assert(card_idx < (1u << (sizeof(_data[0]) * BitsPerByte)),do { if (!(card_idx < (1u << (sizeof(_data[0]) * BitsPerByte
)))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 169, "assert(" "card_idx < (1u << (sizeof(_data[0]) * BitsPerByte))"
") failed", "Card index %u does not fit allowed card value range."
, card_idx); ::breakpoint(); } } while (0)
169 "Card index %u does not fit allowed card value range.", card_idx)do { if (!(card_idx < (1u << (sizeof(_data[0]) * BitsPerByte
)))) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 169, "assert(" "card_idx < (1u << (sizeof(_data[0]) * BitsPerByte))"
") failed", "Card index %u does not fit allowed card value range."
, card_idx); ::breakpoint(); } } while (0);
170 EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
171 EntryCountType idx = 0;
172 for (; idx < num_entries; idx++) {
173 if (_data[idx] == card_idx) {
174 return Found;
175 }
176 }
177
178 // Since we did not find the card, lock.
179 G1CardSetArrayLocker x(&_num_entries);
180
181 // Reload number of entries from the G1CardSetArrayLocker as it might have changed.
182 // It already read the actual value with the necessary synchronization.
183 num_entries = x.num_entries();
184 // Look if the cards added while waiting for the lock are the same as our card.
185 for (; idx < num_entries; idx++) {
186 if (_data[idx] == card_idx) {
187 return Found;
188 }
189 }
190
191 // Check if there is space left.
192 if (num_entries == _size) {
193 return Overflow;
194 }
195
196 _data[num_entries] = card_idx;
197
198 x.inc_num_entries();
199
200 return Added;
201}
202
203inline bool G1CardSetArray::contains(uint card_idx) {
204 EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
205
206 for (EntryCountType idx = 0; idx < num_entries; idx++) {
207 if (_data[idx] == card_idx) {
208 return true;
209 }
210 }
211 return false;
212}
213
214template <class CardVisitor>
215void G1CardSetArray::iterate(CardVisitor& found) {
216 EntryCountType num_entries = Atomic::load_acquire(&_num_entries) & EntryMask;
217 for (EntryCountType idx = 0; idx < num_entries; idx++) {
218 found(_data[idx]);
219 }
220}
221
222inline G1CardSetBitMap::G1CardSetBitMap(uint card_in_region, uint size_in_bits) :
223 G1CardSetContainer(), _num_bits_set(1) {
224 assert(size_in_bits % (sizeof(_bits[0]) * BitsPerByte) == 0,do { if (!(size_in_bits % (sizeof(_bits[0]) * BitsPerByte) ==
0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 225, "assert(" "size_in_bits % (sizeof(_bits[0]) * BitsPerByte) == 0"
") failed", "Size %u should be aligned to bitmap word size."
, size_in_bits); ::breakpoint(); } } while (0)
225 "Size %u should be aligned to bitmap word size.", size_in_bits)do { if (!(size_in_bits % (sizeof(_bits[0]) * BitsPerByte) ==
0)) { (*g_assert_poison) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 225, "assert(" "size_in_bits % (sizeof(_bits[0]) * BitsPerByte) == 0"
") failed", "Size %u should be aligned to bitmap word size."
, size_in_bits); ::breakpoint(); } } while (0);
226 BitMapView bm(_bits, size_in_bits);
227 bm.clear();
228 bm.set_bit(card_in_region);
229}
230
231inline G1AddCardResult G1CardSetBitMap::add(uint card_idx, size_t threshold, size_t size_in_bits) {
232 BitMapView bm(_bits, size_in_bits);
233 if (_num_bits_set >= threshold) {
234 return bm.at(card_idx) ? Found : Overflow;
235 }
236 if (bm.par_set_bit(card_idx)) {
237 Atomic::inc(&_num_bits_set, memory_order_relaxed);
238 return Added;
239 }
240 return Found;
241}
242
243template <class CardVisitor>
244inline void G1CardSetBitMap::iterate(CardVisitor& found, size_t size_in_bits, uint offset) {
245 BitMapView bm(_bits, size_in_bits);
246 BitMap::idx_t idx = bm.get_next_one_offset(0);
247 while (idx != size_in_bits) {
248 found((offset | (uint)idx));
249 idx = bm.get_next_one_offset(idx + 1);
250 }
251}
252
253inline G1CardSetHowl::G1CardSetHowl(EntryCountType card_in_region, G1CardSetConfiguration* config) :
254 G1CardSetContainer(),
255 _num_entries((config->max_cards_in_array() + 1)) /* Card Transfer will not increment _num_entries */ {
256 EntryCountType num_buckets = config->num_buckets_in_howl();
257 EntryCountType bucket = config->howl_bucket_index(card_in_region);
258 for (uint i = 0; i < num_buckets; ++i) {
259 _buckets[i] = G1CardSetInlinePtr();
260 if (i == bucket) {
261 G1CardSetInlinePtr value(&_buckets[i], _buckets[i]);
262 value.add(card_in_region, config->inline_ptr_bits_per_card(), config->max_cards_in_inline_ptr());
263 }
264 }
265}
266
267inline bool G1CardSetHowl::contains(uint card_idx, G1CardSetConfiguration* config) {
268 EntryCountType bucket = config->howl_bucket_index(card_idx);
269 CardSetPtr* array_entry = get_card_set_addr(bucket);
270 CardSetPtr card_set = Atomic::load_acquire(array_entry);
271
272 switch (G1CardSet::card_set_type(card_set)) {
273 case G1CardSet::CardSetArrayOfCards : {
274 return G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->contains(card_idx);
275 }
276 case G1CardSet::CardSetBitMap: {
277 uint card_offset = config->howl_bitmap_offset(card_idx);
278 return G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->contains(card_offset, config->max_cards_in_howl_bitmap());
279 }
280 case G1CardSet::CardSetInlinePtr: {
281 G1CardSetInlinePtr ptr(card_set);
282 return ptr.contains(card_idx, config->inline_ptr_bits_per_card());
283 }
284 case G1CardSet::CardSetHowl: {// Fullcard set entry
285 assert(card_set == G1CardSet::FullCardSet, "Must be")do { if (!(card_set == G1CardSet::FullCardSet)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 285, "assert(" "card_set == G1CardSet::FullCardSet" ") failed"
, "Must be"); ::breakpoint(); } } while (0);
286 return true;
287 }
288 }
289 return false;
290}
291
292template <class CardOrRangeVisitor>
293inline void G1CardSetHowl::iterate(CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
294 for (uint i = 0; i < config->num_buckets_in_howl(); ++i) {
295 iterate_cardset(_buckets[i], i, found, config);
296 }
297}
298
299template <class CardSetPtrVisitor>
300inline void G1CardSetHowl::iterate(CardSetPtrVisitor& found, uint num_card_sets) {
301 for (uint i = 0; i < num_card_sets; ++i) {
302 found(&_buckets[i]);
303 }
304}
305
306template <class CardOrRangeVisitor>
307inline void G1CardSetHowl::iterate_cardset(CardSetPtr const card_set, uint index, CardOrRangeVisitor& found, G1CardSetConfiguration* config) {
308 switch (G1CardSet::card_set_type(card_set)) {
309 case G1CardSet::CardSetInlinePtr: {
310 if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlInline)) {
311 G1CardSetInlinePtr ptr(card_set);
312 ptr.iterate(found, config->inline_ptr_bits_per_card());
313 }
314 return;
315 }
316 case G1CardSet::CardSetArrayOfCards : {
317 if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlArrayOfCards)) {
318 G1CardSet::card_set_ptr<G1CardSetArray>(card_set)->iterate(found);
319 }
320 return;
321 }
322 case G1CardSet::CardSetBitMap: {
323 if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlBitmap)) {
324 uint offset = index << config->log2_max_cards_in_howl_bitmap();
325 G1CardSet::card_set_ptr<G1CardSetBitMap>(card_set)->iterate(found, config->max_cards_in_howl_bitmap(), offset);
326 }
327 return;
328 }
329 case G1CardSet::CardSetHowl: { // actually FullCardSet
330 assert(card_set == G1CardSet::FullCardSet, "Must be")do { if (!(card_set == G1CardSet::FullCardSet)) { (*g_assert_poison
) = 'X';; report_vm_error("/home/daniel/Projects/java/jdk/src/hotspot/share/gc/g1/g1CardSetContainers.inline.hpp"
, 330, "assert(" "card_set == G1CardSet::FullCardSet" ") failed"
, "Must be"); ::breakpoint(); } } while (0);
331 if (found.start_iterate(G1GCPhaseTimes::MergeRSHowlFull)) {
332 uint offset = index << config->log2_max_cards_in_howl_bitmap();
333 found(offset, config->max_cards_in_howl_bitmap());
334 }
335 return;
336 }
337 }
338}
339
340inline G1CardSetHowl::EntryCountType G1CardSetHowl::num_buckets(size_t size_in_bits, size_t max_cards_in_array, size_t max_num_buckets) {
341 size_t size_bitmap_bytes = BitMap::calc_size_in_words(size_in_bits) * BytesPerWord;
342 // Ensure that in the worst case arrays consume half the memory size
343 // of storing the entire bitmap
344 size_t max_size_arrays_bytes = size_bitmap_bytes / 2;
345 size_t size_array_bytes = max_cards_in_array * sizeof(G1CardSetArray::EntryDataType);
346 size_t num_arrays = max_size_arrays_bytes / size_array_bytes;
347 // We use shifts and masks for indexing the array. So round down to the next
348 // power of two to not use more than expected memory.
349 num_arrays = round_down_power_of_2(MAX2((size_t)1, MIN2(num_arrays, max_num_buckets)));
350 return (EntryCountType)num_arrays;
351}
352
353#endif // SHARE_GC_G1_G1CARDSETCONTAINERS_INLINE_HPP