jabberwocky238/rocksdb
1
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2026-07-07 14:47:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Finish migrating HCC to BitFields API (#14154)

Browse Source 
Summary:
This change builds on https://github.com/facebook/rocksdb/issues/14027 and https://github.com/facebook/rocksdb/issues/13965 to complete migration
of the HyperClockCache implementation to using the hygienic BitFields API.
No semantic change in the implementation details is intended, just
greatly improving readability and safety of the code while maintaining
the same performance.

In more detail,
* Refactor the main metadata atomic for each slot in an HCC table into
SlotMeta using BitFields.
* Extended BitFields APIs with some additional features, and renamed
  BlahTransform classes to BlahTransformer to resolve potential naming
  conflicts with member functions to create them.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/14154

Test Plan:
for correctness, mostly existing tests. but also added tests
for new BitFields features. I especially ran local TSAN whitebox crash
test extensively which caught a couple of refactoring errors.

For performance, I verified with release builds of cache_bench, using
default options, that there was no noticeable/consistent difference
after all these HCC migrations vs. backing them out. That test was with
GCC 11 and -O2, which is a reasonable baseline for expected compiler
optimizations.

Reviewed By: xingbowang

Differential Revision: D87960540

Pulled By: pdillinger

fbshipit-source-id: e0257b7fea8a5c7709daef18911959201ce4e0f3
This commit is contained in:
Peter Dillinger
2025-12-29 17:13:50 -08:00
committed by meta-codesync[bot]
parent 3818cc1aca
commit 1cc1df8dab
5 changed files with 441 additions and 373 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cache/clock_cache.cc
Vendored
+257 -293
View File
@@ -38,13 +38,11 @@ namespace ROCKSDB_NAMESPACE {
namespace clock_cache {
namespace {
inline uint64_t GetRefcount(uint64_t meta) {
return ((meta >> ClockHandle::kAcquireCounterShift) -
(meta >> ClockHandle::kReleaseCounterShift)) &
ClockHandle::kCounterMask;
}
using SlotMeta = ClockHandle::SlotMeta;
using AcquireCounter = SlotMeta::AcquireCounter;
using ReleaseCounter = SlotMeta::ReleaseCounter;
inline uint64_t GetInitialCountdown(Cache::Priority priority) {
inline uint32_t GetInitialCountdown(Cache::Priority priority) {
// Set initial clock data from priority
// TODO: configuration parameters for priority handling and clock cycle
// count?
@@ -65,11 +63,11 @@ inline uint64_t GetInitialCountdown(Cache::Priority priority) {
inline void MarkEmpty(ClockHandle& h) {
#ifndef NDEBUG
// Mark slot as empty, with assertion
uint64_t meta = h.meta.Exchange(0);
assert(meta >> ClockHandle::kStateShift == ClockHandle::kStateConstruction);
auto old_meta = h.meta.Exchange({});
assert(old_meta.IsUnderConstruction());
#else
// Mark slot as empty
h.meta.Store(0);
h.meta.Store({});
#endif
}
@@ -85,18 +83,20 @@ inline void FreeDataMarkEmpty(ClockHandle& h, MemoryAllocator* allocator) {
// Called to undo the effect of referencing an entry for internal purposes,
// so it should not be marked as having been used.
inline void Unref(const ClockHandle& h, uint64_t count = 1) {
inline void Unref(const ClockHandle& h, uint32_t count = 1) {
// Pretend we never took the reference
// WART: there's a tiny chance we release last ref to invisible
// entry here. If that happens, we let eviction take care of it.
uint64_t old_meta = h.meta.FetchSub(ClockHandle::kAcquireIncrement * count);
assert(GetRefcount(old_meta) != 0);
SlotMeta old_meta;
h.meta.Apply(AcquireCounter::MinusTransformPromiseNoUnderflow(count),
&old_meta);
assert(old_meta.GetRefcount() != 0);
(void)old_meta;
}
inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data,
bool* purgeable = nullptr) {
uint64_t meta;
SlotMeta meta;
if (purgeable) {
assert(*purgeable == false);
// In AutoHCC, our eviction process follows the chain structure, so we
@@ -110,46 +110,40 @@ inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data,
meta = h.meta.LoadRelaxed();
}
if (((meta >> ClockHandle::kStateShift) & ClockHandle::kStateShareableBit) ==
0) {
if (!meta.IsShareable()) {
// Only clock update Shareable entries
if (purgeable) {
*purgeable = true;
// AutoHCC only: make sure we only attempt to update non-empty slots
assert((meta >> ClockHandle::kStateShift) &
ClockHandle::kStateOccupiedBit);
assert(!meta.IsEmpty());
}
return false;
}
uint64_t acquire_count =
(meta >> ClockHandle::kAcquireCounterShift) & ClockHandle::kCounterMask;
uint64_t release_count =
(meta >> ClockHandle::kReleaseCounterShift) & ClockHandle::kCounterMask;
uint32_t acquire_count = meta.GetAcquireCounter();
uint32_t release_count = meta.GetReleaseCounter();
if (acquire_count != release_count) {
// Only clock update entries with no outstanding refs
data->seen_pinned_count++;
return false;
}
if ((meta >> ClockHandle::kStateShift == ClockHandle::kStateVisible) &&
acquire_count > 0) {
if (meta.IsVisible() && acquire_count > 0) {
// Decrement clock
uint64_t new_count =
std::min(acquire_count - 1, uint64_t{ClockHandle::kMaxCountdown} - 1);
uint32_t new_count =
std::min(acquire_count - 1, uint32_t{ClockHandle::kMaxCountdown} - 1);
// Compare-exchange in the decremented clock info, but
// not aggressively
uint64_t new_meta =
(uint64_t{ClockHandle::kStateVisible} << ClockHandle::kStateShift) |
(meta & ClockHandle::kHitBitMask) |
(new_count << ClockHandle::kReleaseCounterShift) |
(new_count << ClockHandle::kAcquireCounterShift);
SlotMeta new_meta = meta;
new_meta.SetReleaseCounter(new_count);
new_meta.SetAcquireCounter(new_count);
h.meta.CasStrongRelaxed(meta, new_meta);
return false;
}
// Otherwise, remove entry (either unreferenced invisible or
// unreferenced and expired visible).
if (h.meta.CasStrong(meta, (uint64_t{ClockHandle::kStateConstruction}
<< ClockHandle::kStateShift) |
(meta & ClockHandle::kHitBitMask))) {
SlotMeta construction_meta;
construction_meta.SetUnderConstruction();
construction_meta.SetHit(meta.GetHit());
if (h.meta.CasStrong(meta, construction_meta)) {
// Took ownership.
data->freed_charge += h.GetTotalCharge();
data->freed_count += 1;
@@ -215,39 +209,39 @@ inline bool ClockUpdate(ClockHandle& h, BaseClockTable::EvictionData* data,
// counter to reach "high" state again and bumped back to "medium." (This
// motivates only checking for release counter in high state, not both in high
// state.)
inline void CorrectNearOverflow(uint64_t old_meta,
AcqRelAtomic<uint64_t>& meta) {
inline void CorrectNearOverflow(SlotMeta old_meta,
AcqRelBitFieldsAtomic<SlotMeta>& meta) {
// We clear both top-most counter bits at the same time.
constexpr uint64_t kCounterTopBit = uint64_t{1}
<< (ClockHandle::kCounterNumBits - 1);
constexpr uint64_t kClearBits =
(kCounterTopBit << ClockHandle::kAcquireCounterShift) |
(kCounterTopBit << ClockHandle::kReleaseCounterShift);
// A simple check that allows us to initiate clearing the top bits for
// a large portion of the "high" state space on release counter.
constexpr uint64_t kCheckBits =
(kCounterTopBit | (ClockHandle::kMaxCountdown + 1))
<< ClockHandle::kReleaseCounterShift;
constexpr uint32_t kCounterTopBit = uint32_t{1}
<< (SlotMeta::kCounterNumBits - 1);
// The threshold for correcting "near overflow" is to ensure
// (a) the value has a top bit set that can be cleared
// (b) when we clear the top bit, the eviction state will be preserved
// (everything >= kMaxCountdown is treated equivalently)
// As mentioned above, we only check the release count.
constexpr uint32_t kThreshold = kCounterTopBit + ClockHandle::kMaxCountdown;
if (UNLIKELY(old_meta & kCheckBits)) {
meta.FetchAndRelaxed(~kClearBits);
if (UNLIKELY(old_meta.GetReleaseCounter() > kThreshold)) {
auto clear_transform = AcquireCounter::AndTransform(kCounterTopBit - 1) +
ReleaseCounter::AndTransform(kCounterTopBit - 1);
meta.ApplyRelaxed(clear_transform);
}
}
inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
uint64_t initial_countdown, bool* already_matches) {
uint32_t initial_countdown, bool* already_matches) {
assert(*already_matches == false);
// Optimistically transition the slot from "empty" to
// "under construction" (no effect on other states)
uint64_t old_meta = h.meta.FetchOr(uint64_t{ClockHandle::kStateOccupiedBit}
<< ClockHandle::kStateShift);
uint64_t old_state = old_meta >> ClockHandle::kStateShift;
auto set_occupied = SlotMeta::OccupiedFlag::SetTransform();
SlotMeta old_meta;
h.meta.Apply(set_occupied, &old_meta);
if (old_state == ClockHandle::kStateEmpty) {
if (old_meta.IsEmpty()) {
// We've started inserting into an available slot, and taken
// ownership.
return true;
} else if (old_state != ClockHandle::kStateVisible) {
} else if (!old_meta.IsVisible()) {
// Slot not usable / touchable now
return false;
}
@@ -255,15 +249,17 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
// But first, we need to acquire a ref to read it. In fact, number of
// refs for initial countdown, so that we boost the clock state if
// this is a match.
old_meta =
h.meta.FetchAdd(ClockHandle::kAcquireIncrement * initial_countdown);
auto add_acquire =
AcquireCounter::PlusTransformPromiseNoOverflow(initial_countdown);
h.meta.Apply(add_acquire, &old_meta);
// Like Lookup
if ((old_meta >> ClockHandle::kStateShift) == ClockHandle::kStateVisible) {
if (old_meta.IsVisible()) {
// Acquired a read reference
if (h.hashed_key == proto.hashed_key) {
// Match. Release in a way that boosts the clock state
old_meta =
h.meta.FetchAdd(ClockHandle::kReleaseIncrement * initial_countdown);
auto add_release =
ReleaseCounter::PlusTransformPromiseNoOverflow(initial_countdown);
h.meta.Apply(add_release, &old_meta);
// Correct for possible (but rare) overflow
CorrectNearOverflow(old_meta, h.meta);
// Insert detached instead (only if return handle needed)
@@ -273,8 +269,7 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
// Mismatch.
Unref(h, initial_countdown);
}
} else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) ==
ClockHandle::kStateInvisible)) {
} else if (UNLIKELY(old_meta.IsInvisible())) {
// Pretend we never took the reference
Unref(h, initial_countdown);
} else {
@@ -286,25 +281,23 @@ inline bool BeginSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
}
inline void FinishSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
uint64_t initial_countdown, bool keep_ref) {
uint32_t initial_countdown, bool keep_ref) {
// Save data fields
ClockHandleBasicData* h_alias = &h;
*h_alias = proto;
// Transition from "under construction" state to "visible" state
uint64_t new_meta = uint64_t{ClockHandle::kStateVisible}
<< ClockHandle::kStateShift;
SlotMeta new_meta;
new_meta.SetVisible();
// Maybe with an outstanding reference
new_meta |= initial_countdown << ClockHandle::kAcquireCounterShift;
new_meta |= (initial_countdown - keep_ref)
<< ClockHandle::kReleaseCounterShift;
new_meta.SetAcquireCounter(initial_countdown);
new_meta.SetReleaseCounter(initial_countdown - (keep_ref ? 1 : 0));
#ifndef NDEBUG
// Save the state transition, with assertion
uint64_t old_meta = h.meta.Exchange(new_meta);
assert(old_meta >> ClockHandle::kStateShift ==
ClockHandle::kStateConstruction);
auto old_meta = h.meta.Exchange(new_meta);
assert(old_meta.IsUnderConstruction());
#else
// Save the state transition
h.meta.Store(new_meta);
@@ -312,7 +305,7 @@ inline void FinishSlotInsert(const ClockHandleBasicData& proto, ClockHandle& h,
}
bool TryInsert(const ClockHandleBasicData& proto, ClockHandle& h,
uint64_t initial_countdown, bool keep_ref,
uint32_t initial_countdown, bool keep_ref,
bool* already_matches) {
bool b = BeginSlotInsert(proto, h, initial_countdown, already_matches);
if (b) {
@@ -326,35 +319,32 @@ template <class HandleImpl, class Func>
void ConstApplyToEntriesRange(const Func& func, const HandleImpl* begin,
const HandleImpl* end,
bool apply_if_will_be_deleted) {
uint64_t check_state_mask = ClockHandle::kStateShareableBit;
if (!apply_if_will_be_deleted) {
check_state_mask |= ClockHandle::kStateVisibleBit;
}
for (const HandleImpl* h = begin; h < end; ++h) {
// Note: to avoid using compare_exchange, we have to be extra careful.
uint64_t old_meta = h->meta.LoadRelaxed();
SlotMeta old_meta = h->meta.LoadRelaxed();
// Check if it's an entry visible to lookups
if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) {
// Increment acquire counter. Note: it's possible that the entry has
// completely changed since we loaded old_meta, but incrementing acquire
// count is always safe. (Similar to optimistic Lookup here.)
old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
// Check whether we actually acquired a reference.
if ((old_meta >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit) {
// Apply func if appropriate
if ((old_meta >> ClockHandle::kStateShift) & check_state_mask) {
func(*h);
if (apply_if_will_be_deleted || old_meta.IsVisible()) {
if (old_meta.IsShareable()) {
// Increment acquire counter. Note: it's possible that the entry has
// completely changed since we loaded old_meta, but incrementing acquire
// count is always safe. (Similar to optimistic Lookup here.)
auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
h->meta.Apply(add_acquire, &old_meta);
// Check whether we actually acquired a reference.
if (old_meta.IsShareable()) {
// Apply func if appropriate
if (apply_if_will_be_deleted || old_meta.IsVisible()) {
func(*h);
}
// Pretend we never took the reference
Unref(*h);
// No net change, so don't need to check for overflow
} else {
// For other states, incrementing the acquire counter has no effect
// so we don't need to undo it. Furthermore, we cannot safely undo
// it because we did not acquire a read reference to lock the
// entry in a Shareable state.
}
// Pretend we never took the reference
Unref(*h);
// No net change, so don't need to check for overflow
} else {
// For other states, incrementing the acquire counter has no effect
// so we don't need to undo it. Furthermore, we cannot safely undo
// it because we did not acquire a read reference to lock the
// entry in a Shareable state.
}
}
}
@@ -399,9 +389,9 @@ HandleImpl* BaseClockTable::StandaloneInsert(
h->SetStandalone();
// Single reference (standalone entries only created if returning a refed
// Handle back to user)
uint64_t meta = uint64_t{ClockHandle::kStateInvisible}
<< ClockHandle::kStateShift;
meta |= uint64_t{1} << ClockHandle::kAcquireCounterShift;
SlotMeta meta;
meta.SetInvisible();
meta.SetAcquireCounter(1);
h->meta.Store(meta);
// Keep track of how much of usage is standalone
standalone_usage_.FetchAddRelaxed(proto.GetTotalCharge());
@@ -564,11 +554,10 @@ void BaseClockTable::TrackAndReleaseEvictedEntry(ClockHandle* h) {
if (eviction_callback_) {
// For key reconstructed from hash
UniqueId64x2 unhashed;
took_value_ownership =
eviction_callback_(ClockCacheShard<FixedHyperClockTable>::ReverseHash(
h->GetHash(), &unhashed, hash_seed_),
static_cast<Cache::Handle*>(h),
h->meta.LoadRelaxed() & ClockHandle::kHitBitMask);
took_value_ownership = eviction_callback_(
ClockCacheShard<FixedHyperClockTable>::ReverseHash(
h->GetHash(), &unhashed, hash_seed_),
static_cast<Cache::Handle*>(h), h->meta.LoadRelaxed().GetHit());
}
if (!took_value_ownership) {
h->FreeData(allocator_);
@@ -648,7 +637,7 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
// * Have to insert into a suboptimal location (more probes) so that the
// old entry can be kept around as well.
uint64_t initial_countdown = GetInitialCountdown(priority);
uint32_t initial_countdown = GetInitialCountdown(priority);
assert(initial_countdown > 0);
HandleImpl* e =
@@ -693,34 +682,34 @@ Status BaseClockTable::Insert(const ClockHandleBasicData& proto,
void BaseClockTable::Ref(ClockHandle& h) {
// Increment acquire counter
uint64_t old_meta = h.meta.FetchAdd(ClockHandle::kAcquireIncrement);
SlotMeta old_meta;
h.meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(1), &old_meta);
assert((old_meta >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit);
assert(old_meta.IsShareable());
// Must have already had a reference
assert(GetRefcount(old_meta) > 0);
assert(old_meta.GetRefcount() > 0);
(void)old_meta;
}
#ifndef NDEBUG
void BaseClockTable::TEST_RefN(ClockHandle& h, size_t n) {
void BaseClockTable::TEST_RefN(ClockHandle& h, uint32_t n) {
// Increment acquire counter
uint64_t old_meta = h.meta.FetchAdd(n * ClockHandle::kAcquireIncrement);
SlotMeta old_meta;
h.meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(n), &old_meta);
assert((old_meta >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit);
assert(old_meta.IsShareable());
(void)old_meta;
}
void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, size_t n) {
void BaseClockTable::TEST_ReleaseNMinus1(ClockHandle* h, uint32_t n) {
assert(n > 0);
// Like n-1 Releases, but assumes one more will happen in the caller to take
// care of anything like erasing an unreferenced, invisible entry.
uint64_t old_meta =
h->meta.FetchAdd((n - 1) * ClockHandle::kReleaseIncrement);
assert((old_meta >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit);
SlotMeta old_meta;
h->meta.Apply(ReleaseCounter::PlusTransformPromiseNoOverflow(n - 1),
&old_meta);
assert(old_meta.IsShareable());
(void)old_meta;
}
#endif
@@ -754,23 +743,20 @@ FixedHyperClockTable::~FixedHyperClockTable() {
// in the table.
for (size_t i = 0; i < GetTableSize(); i++) {
HandleImpl& h = array_[i];
switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) {
case ClockHandle::kStateEmpty:
// noop
break;
case ClockHandle::kStateInvisible: // rare but possible
case ClockHandle::kStateVisible:
assert(GetRefcount(h.meta.LoadRelaxed()) == 0);
h.FreeData(allocator_);
SlotMeta meta = h.meta.LoadRelaxed();
if (meta.IsShareable()) {
// NOTE: Reaching here invisible is rare but possible
assert(meta.GetRefcount() == 0);
h.FreeData(allocator_);
#ifndef NDEBUG
Rollback(h.hashed_key, &h);
ReclaimEntryUsage(h.GetTotalCharge());
Rollback(h.hashed_key, &h);
ReclaimEntryUsage(h.GetTotalCharge());
#endif
break;
// otherwise
default:
assert(false);
break;
} else {
// Should be no transient "under construction" states unless a thread
// was killed or we are being destructed while another thread is still
// operating on the structure
assert(meta.IsEmpty());
}
}
@@ -792,7 +778,7 @@ bool FixedHyperClockTable::GrowIfNeeded(size_t new_occupancy, InsertState&) {
}
FixedHyperClockTable::HandleImpl* FixedHyperClockTable::DoInsert(
const ClockHandleBasicData& proto, uint64_t initial_countdown,
const ClockHandleBasicData& proto, uint32_t initial_countdown,
bool keep_ref, InsertState&) {
bool already_matches = false;
HandleImpl* e = FindSlot(
@@ -843,47 +829,46 @@ FixedHyperClockTable::HandleImpl* FixedHyperClockTable::Lookup(
HandleImpl* e = FindSlot(
hashed_key,
[&](HandleImpl* h) {
SlotMeta old_meta;
// Mostly branch-free version (similar performance)
/*
uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement,
std::memory_order_acquire);
bool Shareable = (old_meta >> (ClockHandle::kStateShift + 1)) & 1U;
bool visible = (old_meta >> ClockHandle::kStateShift) & 1U;
bool match = (h->key == key) & visible;
h->meta.FetchSub(static_cast<uint64_t>(Shareable & !match) <<
ClockHandle::kAcquireCounterShift); return
match;
h->meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(1),
&old_meta);
bool shareable = old_meta.IsShareable();
bool visible = old_meta.IsVisible();
bool match = (h->hashed_key == hashed_key) & visible;
h->meta.Apply(AcquireCounter::MinusTransformPromiseNoUnderflow(
uint32_t{shareable} & uint32_t{!match}));
h->meta.Apply(SlotMeta::HitFlag::Or(match));
return match;
*/
// Optimistic lookup should pay off when the table is relatively
// sparse.
constexpr bool kOptimisticLookup = true;
uint64_t old_meta;
if (!kOptimisticLookup) {
old_meta = h->meta.Load();
if ((old_meta >> ClockHandle::kStateShift) !=
ClockHandle::kStateVisible) {
if (!old_meta.IsVisible()) {
return false;
}
}
// (Optimistically) increment acquire counter
old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
h->meta.Apply(AcquireCounter::PlusTransformPromiseNoOverflow(1),
&old_meta);
// Check if it's an entry visible to lookups
if ((old_meta >> ClockHandle::kStateShift) ==
ClockHandle::kStateVisible) {
if (old_meta.IsVisible()) {
// Acquired a read reference
if (h->hashed_key == hashed_key) {
// Match
// Update the hit bit
if (eviction_callback_) {
h->meta.FetchOrRelaxed(uint64_t{1} << ClockHandle::kHitBitShift);
h->meta.ApplyRelaxed(SlotMeta::HitFlag::SetTransform());
}
return true;
} else {
// Mismatch. Pretend we never took the reference
Unref(*h);
}
} else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) ==
ClockHandle::kStateInvisible)) {
} else if (UNLIKELY(old_meta.IsInvisible())) {
// Pretend we never took the reference
Unref(*h);
} else {
@@ -907,53 +892,49 @@ bool FixedHyperClockTable::Release(HandleImpl* h, bool useful,
// is only freed up by EvictFromClock (called by Insert when space is needed)
// and Erase. We do this to avoid an extra atomic read of the variable usage_.
uint64_t old_meta;
SlotMeta old_meta;
if (useful) {
// Increment release counter to indicate was used
old_meta = h->meta.FetchAdd(ClockHandle::kReleaseIncrement);
auto add_release = ReleaseCounter::PlusTransformPromiseNoOverflow(1);
h->meta.Apply(add_release, &old_meta);
} else {
// Decrement acquire counter to pretend it never happened
old_meta = h->meta.FetchSub(ClockHandle::kAcquireIncrement);
auto sub_acquire = AcquireCounter::MinusTransformPromiseNoUnderflow(1);
h->meta.Apply(sub_acquire, &old_meta);
}
assert((old_meta >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit);
assert(old_meta.IsShareable());
// No underflow
assert(((old_meta >> ClockHandle::kAcquireCounterShift) &
ClockHandle::kCounterMask) !=
((old_meta >> ClockHandle::kReleaseCounterShift) &
ClockHandle::kCounterMask));
assert(old_meta.GetAcquireCounter() != old_meta.GetReleaseCounter());
if (erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift ==
ClockHandle::kStateInvisible)) {
if (erase_if_last_ref || UNLIKELY(old_meta.IsInvisible())) {
// FIXME: There's a chance here that another thread could replace this
// entry and we end up erasing the wrong one.
// Update for last FetchAdd op
// Update for last Apply op
if (useful) {
old_meta += ClockHandle::kReleaseIncrement;
old_meta.SetReleaseCounter(old_meta.GetReleaseCounter() + 1);
} else {
old_meta -= ClockHandle::kAcquireIncrement;
old_meta.SetAcquireCounter(old_meta.GetAcquireCounter() - 1);
}
// Take ownership if no refs
SlotMeta construction_meta;
construction_meta.SetUnderConstruction();
do {
if (GetRefcount(old_meta) != 0) {
if (old_meta.GetRefcount() != 0) {
// Not last ref at some point in time during this Release call
// Correct for possible (but rare) overflow
CorrectNearOverflow(old_meta, h->meta);
return false;
}
if ((old_meta & (uint64_t{ClockHandle::kStateShareableBit}
<< ClockHandle::kStateShift)) == 0) {
if (!old_meta.IsShareable()) {
// Someone else took ownership
return false;
}
// Note that there's a small chance that we release, another thread
// replaces this entry with another, reaches zero refs, and then we end
// up erasing that other entry. That's an acceptable risk / imprecision.
} while (
!h->meta.CasWeak(old_meta, uint64_t{ClockHandle::kStateConstruction}
<< ClockHandle::kStateShift));
} while (!h->meta.CasWeak(old_meta, construction_meta));
// Took ownership
size_t total_charge = h->GetTotalCharge();
if (UNLIKELY(h->IsStandalone())) {
@@ -976,7 +957,7 @@ bool FixedHyperClockTable::Release(HandleImpl* h, bool useful,
}
#ifndef NDEBUG
void FixedHyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) {
void FixedHyperClockTable::TEST_ReleaseN(HandleImpl* h, uint32_t n) {
if (n > 0) {
// Do n-1 simple releases first
TEST_ReleaseNMinus1(h, n);
@@ -993,30 +974,29 @@ void FixedHyperClockTable::Erase(const UniqueId64x2& hashed_key) {
[&](HandleImpl* h) {
// Could be multiple entries in rare cases. Erase them all.
// Optimistically increment acquire counter
uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
SlotMeta old_meta, meta;
h->meta.Apply(add_acquire, &old_meta, &meta);
// Check if it's an entry visible to lookups
if ((old_meta >> ClockHandle::kStateShift) ==
ClockHandle::kStateVisible) {
if (meta.IsVisible()) {
// Acquired a read reference
if (h->hashed_key == hashed_key) {
// Match. Set invisible.
old_meta =
h->meta.FetchAnd(~(uint64_t{ClockHandle::kStateVisibleBit}
<< ClockHandle::kStateShift));
// Apply update to local copy
old_meta &= ~(uint64_t{ClockHandle::kStateVisibleBit}
<< ClockHandle::kStateShift);
// Match. Take ownership if no other refs, or set invisible other
// refs exist.
for (;;) {
uint64_t refcount = GetRefcount(old_meta);
uint32_t refcount = meta.GetRefcount();
assert(refcount > 0);
if (refcount > 1) {
// Not last ref at some point in time during this Erase call
// Pretend we never took the reference
// Set invisible
h->meta.Apply(SlotMeta::VisibleFlag::ClearTransform());
// And pretend we never took the reference
Unref(*h);
break;
} else if (h->meta.CasWeak(
old_meta, uint64_t{ClockHandle::kStateConstruction}
<< ClockHandle::kStateShift)) {
}
SlotMeta construction_meta;
construction_meta.SetUnderConstruction();
if (h->meta.CasWeak(meta, construction_meta)) {
// Took ownership
assert(hashed_key == h->hashed_key);
size_t total_charge = h->GetTotalCharge();
@@ -1032,8 +1012,7 @@ void FixedHyperClockTable::Erase(const UniqueId64x2& hashed_key) {
// Mismatch. Pretend we never took the reference
Unref(*h);
}
} else if (UNLIKELY((old_meta >> ClockHandle::kStateShift) ==
ClockHandle::kStateInvisible)) {
} else if (UNLIKELY(old_meta.IsInvisible())) {
// Pretend we never took the reference
Unref(*h);
} else {
@@ -1050,17 +1029,17 @@ void FixedHyperClockTable::EraseUnRefEntries() {
for (size_t i = 0; i <= this->length_bits_mask_; i++) {
HandleImpl& h = array_[i];
uint64_t old_meta = h.meta.LoadRelaxed();
if (old_meta & (uint64_t{ClockHandle::kStateShareableBit}
<< ClockHandle::kStateShift) &&
GetRefcount(old_meta) == 0 &&
h.meta.CasStrong(old_meta, uint64_t{ClockHandle::kStateConstruction}
<< ClockHandle::kStateShift)) {
// Took ownership
size_t total_charge = h.GetTotalCharge();
Rollback(h.hashed_key, &h);
FreeDataMarkEmpty(h, allocator_);
ReclaimEntryUsage(total_charge);
SlotMeta old_meta = h.meta.LoadRelaxed();
if (old_meta.IsShareable() && old_meta.GetRefcount() == 0) {
SlotMeta construction_meta;
construction_meta.SetUnderConstruction();
if (h.meta.CasStrong(old_meta, construction_meta)) {
// Took ownership
size_t total_charge = h.GetTotalCharge();
Rollback(h.hashed_key, &h);
FreeDataMarkEmpty(h, allocator_);
ReclaimEntryUsage(total_charge);
}
}
}
}
@@ -1320,12 +1299,12 @@ bool ClockCacheShard<Table>::Release(HandleImpl* handle, bool useful,
#ifndef NDEBUG
template <class Table>
void ClockCacheShard<Table>::TEST_RefN(HandleImpl* h, size_t n) {
void ClockCacheShard<Table>::TEST_RefN(HandleImpl* h, uint32_t n) {
table_.TEST_RefN(*h, n);
}
template <class Table>
void ClockCacheShard<Table>::TEST_ReleaseN(HandleImpl* h, size_t n) {
void ClockCacheShard<Table>::TEST_ReleaseN(HandleImpl* h, uint32_t n) {
table_.TEST_ReleaseN(h, n);
}
#endif
@@ -1373,8 +1352,8 @@ size_t ClockCacheShard<Table>::GetPinnedUsage() const {
metadata_charge_policy_ == kFullChargeCacheMetadata;
ConstApplyToEntriesRange(
[&table_pinned_usage, charge_metadata](const HandleImpl& h) {
uint64_t meta = h.meta.LoadRelaxed();
uint64_t refcount = GetRefcount(meta);
SlotMeta meta = h.meta.LoadRelaxed();
uint32_t refcount = meta.GetRefcount();
// Holding one ref for ConstApplyToEntriesRange
assert(refcount > 0);
if (refcount > 1) {
@@ -1494,7 +1473,7 @@ void AddShardEvaluation(const FixedHyperClockCache::Shard& shard,
}
bool IsSlotOccupied(const ClockHandle& h) {
return (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) != 0;
return !h.meta.LoadRelaxed().IsEmpty();
}
} // namespace
@@ -1759,12 +1738,12 @@ inline bool MatchAndRef(const UniqueId64x2* hashed_key, const ClockHandle& h,
// Must be at least something to match
assert(hashed_key || shift > 0);
uint64_t old_meta;
SlotMeta old_meta, new_meta;
// (Optimistically) increment acquire counter.
old_meta = h.meta.FetchAdd(ClockHandle::kAcquireIncrement);
auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
h.meta.Apply(add_acquire, &old_meta, &new_meta);
// Check if it's a referencable (sharable) entry
if ((old_meta & (uint64_t{ClockHandle::kStateShareableBit}
<< ClockHandle::kStateShift)) == 0) {
if (!old_meta.IsShareable()) {
// For non-sharable states, incrementing the acquire counter has no effect
// so we don't need to undo it. Furthermore, we cannot safely undo
// it because we did not acquire a read reference to lock the
@@ -1775,10 +1754,9 @@ inline bool MatchAndRef(const UniqueId64x2* hashed_key, const ClockHandle& h,
return false;
}
// Else acquired a read reference
assert(GetRefcount(old_meta + ClockHandle::kAcquireIncrement) > 0);
assert(new_meta.GetRefcount() > 0);
if (hashed_key && h.hashed_key == *hashed_key &&
LIKELY(old_meta & (uint64_t{ClockHandle::kStateVisibleBit}
<< ClockHandle::kStateShift))) {
LIKELY(old_meta.IsVisible())) {
// Match on full key, visible
if (full_match_or_unknown) {
*full_match_or_unknown = true;
@@ -2051,7 +2029,7 @@ AutoHyperClockTable::~AutoHyperClockTable() {
HandleImpl::kUnusedMarker);
assert(array_[i].chain_next_with_shift.LoadRelaxed() ==
HandleImpl::kUnusedMarker);
assert(array_[i].meta.LoadRelaxed() == 0);
assert(array_[i].meta.LoadRelaxed() == SlotMeta{});
}
#endif // MUST_FREE_HEAP_ALLOCATIONS
#ifndef NDEBUG // Extra invariant checking
@@ -2060,30 +2038,27 @@ AutoHyperClockTable::~AutoHyperClockTable() {
#endif // !NDEBUG
for (size_t i = 0; i < used_end; i++) {
HandleImpl& h = array_[i];
switch (h.meta.LoadRelaxed() >> ClockHandle::kStateShift) {
case ClockHandle::kStateEmpty:
// noop
break;
case ClockHandle::kStateInvisible: // rare but possible
case ClockHandle::kStateVisible:
assert(GetRefcount(h.meta.LoadRelaxed()) == 0);
h.FreeData(allocator_);
SlotMeta meta = h.meta.LoadRelaxed();
if (meta.IsShareable()) {
// NOTE: Reaching here invisible is rare but possible
assert(meta.GetRefcount() == 0);
h.FreeData(allocator_);
#ifndef NDEBUG // Extra invariant checking
usage_.FetchSubRelaxed(h.total_charge);
occupancy_.FetchSubRelaxed(1U);
was_populated[i] = true;
if (!h.chain_next_with_shift.LoadRelaxed().IsEnd()) {
assert(!h.chain_next_with_shift.LoadRelaxed().IsLocked());
size_t next = h.chain_next_with_shift.LoadRelaxed().GetNext();
assert(!was_pointed_to[next]);
was_pointed_to[next] = true;
}
usage_.FetchSubRelaxed(h.total_charge);
occupancy_.FetchSubRelaxed(1U);
was_populated[i] = true;
if (!h.chain_next_with_shift.LoadRelaxed().IsEnd()) {
assert(!h.chain_next_with_shift.LoadRelaxed().IsLocked());
size_t next = h.chain_next_with_shift.LoadRelaxed().GetNext();
assert(!was_pointed_to[next]);
was_pointed_to[next] = true;
}
#endif // !NDEBUG
break;
// otherwise
default:
assert(false);
break;
} else {
// Should be no transient "under construction" states unless a thread
// was killed or we are being destructed while another thread is still
// operating on the structure
assert(meta.IsEmpty());
}
#ifndef NDEBUG // Extra invariant checking
if (!h.head_next_with_shift.LoadRelaxed().IsEnd()) {
@@ -2691,20 +2666,17 @@ void AutoHyperClockTable::PurgeImplLocked(OpData* op_data,
op_data->push_back(h);
// Entries for eviction become purgeable
purgeable = true;
assert((h->meta.Load() >> ClockHandle::kStateShift) ==
ClockHandle::kStateConstruction);
assert(h->meta.Load().IsUnderConstruction());
}
} else {
(void)op_data;
(void)data;
purgeable = ((h->meta.Load() >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit) == 0;
purgeable = !h->meta.Load().IsShareable();
}
}
if (purgeable) {
assert((h->meta.Load() >> ClockHandle::kStateShift) ==
ClockHandle::kStateConstruction);
assert(h->meta.Load().IsUnderConstruction());
pending_purge = true;
} else if (pending_purge) {
if (prev_to_keep) {
@@ -2864,7 +2836,7 @@ void AutoHyperClockTable::PurgeImpl(OpData* op_data, size_t home,
}
AutoHyperClockTable::HandleImpl* AutoHyperClockTable::DoInsert(
const ClockHandleBasicData& proto, uint64_t initial_countdown,
const ClockHandleBasicData& proto, uint32_t initial_countdown,
bool take_ref, InsertState& state) {
size_t home;
int orig_home_shift;
@@ -3149,14 +3121,14 @@ AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup(
#endif
if (probably_equal) {
// Increment acquire counter for definitive check
uint64_t old_meta = h->meta.FetchAdd(ClockHandle::kAcquireIncrement);
auto add_acquire = AcquireCounter::PlusTransformPromiseNoOverflow(1);
SlotMeta old_meta, new_meta;
h->meta.Apply(add_acquire, &old_meta, &new_meta);
// Check if it's a referencable (sharable) entry
if (LIKELY(old_meta & (uint64_t{ClockHandle::kStateShareableBit}
<< ClockHandle::kStateShift))) {
assert(GetRefcount(old_meta + ClockHandle::kAcquireIncrement) > 0);
if (LIKELY(old_meta.IsShareable())) {
assert(new_meta.GetRefcount() > 0);
if (LIKELY(h->hashed_key == hashed_key) &&
LIKELY(old_meta & (uint64_t{ClockHandle::kStateVisibleBit}
<< ClockHandle::kStateShift))) {
LIKELY(old_meta.IsVisible())) {
return h;
} else {
Unref(*h);
@@ -3277,7 +3249,7 @@ AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup(
}
// Update the hit bit
if (eviction_callback_) {
h->meta.FetchOrRelaxed(uint64_t{1} << ClockHandle::kHitBitShift);
h->meta.ApplyRelaxed(SlotMeta::HitFlag::SetTransform());
}
// All done.
return h;
@@ -3317,8 +3289,7 @@ AutoHyperClockTable::HandleImpl* AutoHyperClockTable::Lookup(
}
void AutoHyperClockTable::Remove(HandleImpl* h) {
assert((h->meta.Load() >> ClockHandle::kStateShift) ==
ClockHandle::kStateConstruction);
assert(h->meta.Load().IsUnderConstruction());
const HandleImpl& c_h = *h;
PurgeImpl(&c_h.hashed_key);
@@ -3326,26 +3297,23 @@ void AutoHyperClockTable::Remove(HandleImpl* h) {
bool AutoHyperClockTable::TryEraseHandle(HandleImpl* h, bool holding_ref,
bool mark_invisible) {
uint64_t meta;
if (mark_invisible) {
// Set invisible
meta = h->meta.FetchAnd(
~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift));
// To local variable also
meta &=
~(uint64_t{ClockHandle::kStateVisibleBit} << ClockHandle::kStateShift);
} else {
meta = h->meta.Load();
}
SlotMeta meta = h->meta.Load();
assert(!holding_ref || meta.IsShareable());
// Take ownership if no other refs
// Take ownership if no other refs, or set invisible if other refs exist (and
// mark_invisible is set).
SlotMeta construction_meta;
construction_meta.SetUnderConstruction();
do {
if (GetRefcount(meta) != uint64_t{holding_ref}) {
if (meta.GetRefcount() != uint32_t{holding_ref}) {
// Not last ref at some point in time during this call
if (mark_invisible) {
// Set invisible
h->meta.Apply(SlotMeta::VisibleFlag::ClearTransform());
}
return false;
}
if ((meta & (uint64_t{ClockHandle::kStateShareableBit}
<< ClockHandle::kStateShift)) == 0) {
if (!meta.IsShareable()) {
// Someone else took ownership
return false;
}
@@ -3353,8 +3321,7 @@ bool AutoHyperClockTable::TryEraseHandle(HandleImpl* h, bool holding_ref,
// another thread replaces this entry with another, reaches zero refs, and
// then we end up erasing that other entry. That's an acceptable risk /
// imprecision.
} while (!h->meta.CasWeak(meta, uint64_t{ClockHandle::kStateConstruction}
<< ClockHandle::kStateShift));
} while (!h->meta.CasWeak(meta, construction_meta));
// Took ownership
// TODO? Delay freeing?
h->FreeData(allocator_);
@@ -3381,27 +3348,24 @@ bool AutoHyperClockTable::Release(HandleImpl* h, bool useful,
// is needed) and Erase. We do this to avoid an extra atomic read of the
// variable usage_.
uint64_t old_meta;
SlotMeta old_meta;
if (useful) {
// Increment release counter to indicate was used
old_meta = h->meta.FetchAdd(ClockHandle::kReleaseIncrement);
auto add_release = ReleaseCounter::PlusTransformPromiseNoOverflow(1);
h->meta.Apply(add_release, &old_meta);
// Correct for possible (but rare) overflow
CorrectNearOverflow(old_meta, h->meta);
} else {
// Decrement acquire counter to pretend it never happened
old_meta = h->meta.FetchSub(ClockHandle::kAcquireIncrement);
auto sub_acquire = AcquireCounter::MinusTransformPromiseNoUnderflow(1);
h->meta.Apply(sub_acquire, &old_meta);
}
assert((old_meta >> ClockHandle::kStateShift) &
ClockHandle::kStateShareableBit);
assert(old_meta.IsShareable());
// No underflow
assert(((old_meta >> ClockHandle::kAcquireCounterShift) &
ClockHandle::kCounterMask) !=
((old_meta >> ClockHandle::kReleaseCounterShift) &
ClockHandle::kCounterMask));
assert(old_meta.GetAcquireCounter() != old_meta.GetReleaseCounter());
if ((erase_if_last_ref || UNLIKELY(old_meta >> ClockHandle::kStateShift ==
ClockHandle::kStateInvisible))) {
if ((erase_if_last_ref || UNLIKELY(old_meta.IsInvisible()))) {
// FIXME: There's a chance here that another thread could replace this
// entry and we end up erasing the wrong one.
return TryEraseHandle(h, /*holding_ref=*/false, /*mark_invisible=*/false);
@@ -3411,7 +3375,7 @@ bool AutoHyperClockTable::Release(HandleImpl* h, bool useful,
}
#ifndef NDEBUG
void AutoHyperClockTable::TEST_ReleaseN(HandleImpl* h, size_t n) {
void AutoHyperClockTable::TEST_ReleaseN(HandleImpl* h, uint32_t n) {
if (n > 0) {
// Do n-1 simple releases first
TEST_ReleaseNMinus1(h, n);
@@ -3441,20 +3405,20 @@ void AutoHyperClockTable::EraseUnRefEntries() {
for (size_t i = 0; i < usable_size; i++) {
HandleImpl& h = array_[i];
uint64_t old_meta = h.meta.LoadRelaxed();
if (old_meta & (uint64_t{ClockHandle::kStateShareableBit}
<< ClockHandle::kStateShift) &&
GetRefcount(old_meta) == 0 &&
h.meta.CasStrong(old_meta, uint64_t{ClockHandle::kStateConstruction}
<< ClockHandle::kStateShift)) {
// Took ownership
h.FreeData(allocator_);
usage_.FetchSubRelaxed(h.total_charge);
// NOTE: could be more efficient with a dedicated variant of
// PurgeImpl, but this is not a common operation
Remove(&h);
MarkEmpty(h);
occupancy_.FetchSub(1U);
SlotMeta old_meta = h.meta.LoadRelaxed();
if (old_meta.IsShareable() && old_meta.GetRefcount() == 0) {
SlotMeta construction_meta;
construction_meta.SetUnderConstruction();
if (h.meta.CasStrong(old_meta, construction_meta)) {
// Took ownership
h.FreeData(allocator_);
usage_.FetchSubRelaxed(h.total_charge);
// NOTE: could be more efficient with a dedicated variant of
// PurgeImpl, but this is not a common operation
Remove(&h);
MarkEmpty(h);
occupancy_.FetchSub(1U);
}
}
}
}
cache/clock_cache.h
Vendored
+87 -38
View File
@@ -317,40 +317,89 @@ struct ClockHandle : public ClockHandleBasicData {
// | acquire counter | release counter | hit bit | state marker |
// -----------------------------------------------------------------------
// For reading or updating counters in meta word.
static constexpr uint8_t kCounterNumBits = 30;
static constexpr uint64_t kCounterMask = (uint64_t{1} << kCounterNumBits) - 1;
struct SlotMeta : public BitFields<uint64_t, SlotMeta> {
// For reading or updating counters in meta word.
static constexpr uint8_t kCounterNumBits = 30;
// Number of times the a reference has been acquired (or attempted)
// since last reset by eviction processing
using AcquireCounter =
UnsignedBitField<SlotMeta, kCounterNumBits, NoPrevBitField>;
// Number of times the a reference has been released (or attempted)
// since last reset by eviction processing
using ReleaseCounter =
UnsignedBitField<SlotMeta, kCounterNumBits, AcquireCounter>;
// Metadata bit in support of secondary cache
using HitFlag = BoolBitField<SlotMeta, ReleaseCounter>;
// Occupied means any state other than empty
using OccupiedFlag = BoolBitField<SlotMeta, HitFlag>;
// Shareable means the entry is reference counted (visible or invisible)
// (only set if also occupied)
using ShareableFlag = BoolBitField<SlotMeta, OccupiedFlag>;
// Visible is only set if also shareable (invisible can't be found by
// Lookup)
using VisibleFlag = BoolBitField<SlotMeta, ShareableFlag>;
static constexpr uint8_t kAcquireCounterShift = 0;
static constexpr uint64_t kAcquireIncrement = uint64_t{1}
<< kAcquireCounterShift;
static constexpr uint8_t kReleaseCounterShift = kCounterNumBits;
static constexpr uint64_t kReleaseIncrement = uint64_t{1}
<< kReleaseCounterShift;
// Convenience functions
uint32_t GetAcquireCounter() const { return Get<AcquireCounter>(); }
void SetAcquireCounter(uint32_t val) { Set<AcquireCounter>(val); }
uint32_t GetReleaseCounter() const { return Get<ReleaseCounter>(); }
void SetReleaseCounter(uint32_t val) { Set<ReleaseCounter>(val); }
uint32_t GetRefcount() const {
return Get<AcquireCounter>() - Get<ReleaseCounter>();
}
bool GetHit() const { return Get<HitFlag>(); }
void SetHit(bool val) { Set<HitFlag>(val); }
// For setting the hit bit
static constexpr uint8_t kHitBitShift = 2U * kCounterNumBits;
static constexpr uint64_t kHitBitMask = uint64_t{1} << kHitBitShift;
// Some distinct states for the various state flags
bool IsEmpty() const {
bool rv = !Get<OccupiedFlag>();
if (rv) {
assert(!Get<ShareableFlag>());
assert(!Get<VisibleFlag>());
}
return rv;
}
// For reading or updating the state marker in meta word
static constexpr uint8_t kStateShift = kHitBitShift + 1;
bool IsUnderConstruction() const {
bool rv = Get<OccupiedFlag>() && !Get<ShareableFlag>();
if (rv) {
assert(!Get<VisibleFlag>());
}
return rv;
}
void SetUnderConstruction() {
Set<OccupiedFlag>(true);
Set<ShareableFlag>(false);
Set<VisibleFlag>(false);
}
// Bits contribution to state marker.
// Occupied means any state other than empty
static constexpr uint8_t kStateOccupiedBit = 0b100;
// Shareable means the entry is reference counted (visible or invisible)
// (only set if also occupied)
static constexpr uint8_t kStateShareableBit = 0b010;
// Visible is only set if also shareable
static constexpr uint8_t kStateVisibleBit = 0b001;
bool IsShareable() const { return Get<ShareableFlag>(); }
bool IsInvisible() const {
bool rv = Get<ShareableFlag>() && !Get<VisibleFlag>();
if (rv) {
assert(Get<OccupiedFlag>());
}
return rv;
}
void SetInvisible() {
Set<OccupiedFlag>(true);
Set<ShareableFlag>(true);
Set<VisibleFlag>(false);
}
// Complete state markers (not shifted into full word)
static constexpr uint8_t kStateEmpty = 0b000;
static constexpr uint8_t kStateConstruction = kStateOccupiedBit;
static constexpr uint8_t kStateInvisible =
kStateOccupiedBit | kStateShareableBit;
static constexpr uint8_t kStateVisible =
kStateOccupiedBit | kStateShareableBit | kStateVisibleBit;
bool IsVisible() const {
bool rv = Get<ShareableFlag>() && Get<VisibleFlag>();
if (rv) {
assert(Get<OccupiedFlag>());
}
return rv;
}
void SetVisible() {
Set<OccupiedFlag>(true);
Set<ShareableFlag>(true);
Set<VisibleFlag>(true);
}
};
// Constants for initializing the countdown clock. (Countdown clock is only
// in effect with zero refs, acquire counter == release counter, and in that
@@ -364,7 +413,7 @@ struct ClockHandle : public ClockHandleBasicData {
// TODO: make these coundown values tuning parameters for eviction?
// See above. Mutable for read reference counting.
mutable AcqRelAtomic<uint64_t> meta{};
mutable AcqRelBitFieldsAtomic<SlotMeta> meta{};
}; // struct ClockHandle
class BaseClockTable {
@@ -431,9 +480,9 @@ class BaseClockTable {
bool IsEvictionEffortExceeded(const BaseClockTable::EvictionData& data) const;
#ifndef NDEBUG
// Acquire N references
void TEST_RefN(ClockHandle& handle, size_t n);
void TEST_RefN(ClockHandle& handle, uint32_t n);
// Helper for TEST_ReleaseN
void TEST_ReleaseNMinus1(ClockHandle* handle, size_t n);
void TEST_ReleaseNMinus1(ClockHandle* handle, uint32_t n);
#endif
private: // fns
@@ -586,7 +635,7 @@ class FixedHyperClockTable : public BaseClockTable {
bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
HandleImpl* DoInsert(const ClockHandleBasicData& proto,
uint64_t initial_countdown, bool take_ref,
uint32_t initial_countdown, bool take_ref,
InsertState& state);
// Runs the clock eviction algorithm trying to reclaim at least
@@ -614,7 +663,7 @@ class FixedHyperClockTable : public BaseClockTable {
}
// Release N references
void TEST_ReleaseN(HandleImpl* handle, size_t n);
void TEST_ReleaseN(HandleImpl* handle, uint32_t n);
#endif
// The load factor p is a real number in (0, 1) such that at all
@@ -897,7 +946,7 @@ class AutoHyperClockTable : public BaseClockTable {
bool GrowIfNeeded(size_t new_occupancy, InsertState& state);
HandleImpl* DoInsert(const ClockHandleBasicData& proto,
uint64_t initial_countdown, bool take_ref,
uint32_t initial_countdown, bool take_ref,
InsertState& state);
// Runs the clock eviction algorithm trying to reclaim at least
@@ -925,7 +974,7 @@ class AutoHyperClockTable : public BaseClockTable {
}
// Release N references
void TEST_ReleaseN(HandleImpl* handle, size_t n);
void TEST_ReleaseN(HandleImpl* handle, uint32_t n);
#endif
// Maximum ratio of number of occupied slots to number of usable slots. The
@@ -1130,8 +1179,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) ClockCacheShard final : public CacheShardBase {
return table_.TEST_MutableOccupancyLimit();
}
// Acquire/release N references
void TEST_RefN(HandleImpl* handle, size_t n);
void TEST_ReleaseN(HandleImpl* handle, size_t n);
void TEST_RefN(HandleImpl* handle, uint32_t n);
void TEST_ReleaseN(HandleImpl* handle, uint32_t n);
#endif
private: // data
test_util/sync_point.h
+5 -4
View File
@@ -6,10 +6,9 @@
#include <assert.h>
#include <atomic>
#include <functional>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
#include "rocksdb/rocksdb_namespace.h"
@@ -190,6 +189,8 @@ namespace ROCKSDB_NAMESPACE {
// Intentionally not based on std::exception to reduce places where this
// would be caught
struct TestableAssertionFailure {};
// Tracks whether to throw on testable_assert failure instead of aborting.
// This is an atomic counter for re-entrancy / thread-safety.
extern std::atomic<int> g_throw_on_testable_assertion_failure;
} // namespace ROCKSDB_NAMESPACE
#define testable_assert(cond) \
@@ -202,7 +203,7 @@ extern std::atomic<int> g_throw_on_testable_assertion_failure;
} else { \
assert(cond); \
} \
} while (0)
} while (0) // require ; in caller
#define ASSERT_TESTABLE_FAILURE(expr) \
do { \
ROCKSDB_NAMESPACE::g_throw_on_testable_assertion_failure.fetch_add( \
@@ -210,5 +211,5 @@ extern std::atomic<int> g_throw_on_testable_assertion_failure;
ASSERT_THROW(expr, ROCKSDB_NAMESPACE::TestableAssertionFailure); \
ROCKSDB_NAMESPACE::g_throw_on_testable_assertion_failure.fetch_sub( \
1, std::memory_order_relaxed); \
} while (0)
} while (0) // require ; in caller
#endif
util/bit_fields.h
+58 -38
View File
@@ -70,7 +70,7 @@ struct BitFields {
// Return a copy with the given field modified
template <typename BitFieldT>
Derived With(typename BitFieldT::V value) const {
constexpr Derived With(typename BitFieldT::V value) const {
static_assert(std::is_same_v<typename BitFieldT::Parent, Derived>);
Derived rv = static_cast<const Derived&>(*this);
BitFieldT::SetIn(rv, value);
@@ -125,24 +125,26 @@ struct BitFields {
// For building atomic updates affecting one or more fields, assuming all the
// updates are bitwise-or.
template <typename BitFieldsT>
struct OrTransform {
struct OrTransformer {
using U = typename BitFieldsT::U;
U to_or = 0;
// + for general combine
OrTransform<BitFieldsT> operator+(OrTransform<BitFieldsT> other) const {
return OrTransform<BitFieldsT>{to_or | other.to_or};
OrTransformer<BitFieldsT> operator+(
const OrTransformer<BitFieldsT>& other) const {
return OrTransformer<BitFieldsT>{to_or | other.to_or};
}
};
// For building atomic updates affecting one or more fields, assuming all the
// updates are bitwise-and.
template <typename BitFieldsT>
struct AndTransform {
struct AndTransformer {
using U = typename BitFieldsT::U;
U to_and = 0;
// + for general combine
AndTransform<BitFieldsT> operator+(AndTransform<BitFieldsT> other) const {
return AndTransform<BitFieldsT>{to_and & other.to_and};
AndTransformer<BitFieldsT> operator+(
const AndTransformer<BitFieldsT>& other) const {
return AndTransformer<BitFieldsT>{to_and & other.to_and};
}
};
@@ -152,7 +154,7 @@ struct AndTransform {
// corresponding preconditions. (NOTE that when representing a subtraction, we
// rely on overflow of the unsigned representation.)
template <typename BitFieldsT>
struct AddTransform {
struct AddTransformer {
using U = typename BitFieldsT::U;
U to_add = 0;
#ifndef NDEBUG
@@ -162,7 +164,7 @@ struct AddTransform {
};
std::vector<Precondition> preconditions;
#endif // NDEBUG
void AssertPreconditions([[maybe_unused]] U from) {
void AssertPreconditions([[maybe_unused]] U from) const {
#ifndef NDEBUG
for (auto p : preconditions) {
U tmp = (from & p.mask) + p.piece;
@@ -174,8 +176,9 @@ struct AddTransform {
#endif // NDEBUG
}
// + for general combine
AddTransform<BitFieldsT> operator+(AddTransform<BitFieldsT> other) const {
AddTransform<BitFieldsT> rv{to_add + other.to_add};
AddTransformer<BitFieldsT> operator+(
const AddTransformer<BitFieldsT>& other) const {
AddTransformer<BitFieldsT> rv{to_add + other.to_add};
#ifndef NDEBUG
rv.preconditions = preconditions;
rv.preconditions.insert(rv.preconditions.end(), other.preconditions.begin(),
@@ -214,14 +217,18 @@ struct BoolBitField {
return (bf.underlying & (U{1} << kBitOffset)) != 0;
}
static void SetIn(ParentBase& bf, bool value) {
// NOTE: avoiding conditional branches is usually best for speed on modern
// processors
bf.underlying =
(bf.underlying & ~(U{1} << kBitOffset)) | (U{value} << kBitOffset);
}
static OrTransform<BitFieldsT> SetTransform() {
return OrTransform<BitFieldsT>{U{1} << kBitOffset};
static OrTransformer<BitFieldsT> SetTransform() { return Or(true); }
static OrTransformer<BitFieldsT> Or(bool b) {
return OrTransformer<BitFieldsT>{U{b} << kBitOffset};
}
static AndTransform<BitFieldsT> ClearTransform() {
return AndTransform<BitFieldsT>{~(U{1} << kBitOffset)};
static AndTransformer<BitFieldsT> ClearTransform() { return And(false); }
static AndTransformer<BitFieldsT> And(bool b) {
return AndTransformer<BitFieldsT>{~(U{!b} << kBitOffset)};
}
};
@@ -258,18 +265,31 @@ struct UnsignedBitField {
bf.underlying |= static_cast<U>(value & kMask) << kBitOffset;
}
// Create a transfor for clearing this field to zero.
static AndTransform<BitFieldsT> ClearTransform() {
return AndTransform<BitFieldsT>{~(static_cast<U>(kMask) << kBitOffset)};
// Create a transform for clearing this field to zero.
static AndTransformer<BitFieldsT> ClearTransform() {
return AndTransformer<BitFieldsT>{~(static_cast<U>(kMask) << kBitOffset)};
}
// Create a transform for bitwise-and
static AndTransformer<BitFieldsT> AndTransform(V value) {
assert((value & ~kMask) == 0);
return AndTransformer<BitFieldsT>{
~(static_cast<U>(value ^ kMask) << kBitOffset)};
}
// Create a transform for bitwise-or
static OrTransformer<BitFieldsT> OrTransform(V value) {
assert((value & ~kMask) == 0);
return OrTransformer<BitFieldsT>{static_cast<U>(value) << kBitOffset};
}
// Create a transform for adding a particular value, but with the precondition
// that adding the value will not overflow the field. This applies for fields
// that do not include the top bit of the underlying representation. Can be
// combined with other additive transforms for other fields.
static AddTransform<BitFieldsT> PlusTransformPromiseNoOverflow(V value) {
static AddTransformer<BitFieldsT> PlusTransformPromiseNoOverflow(V value) {
static_assert(!kIncludesTopBit);
AddTransform<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
AddTransformer<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
#ifndef NDEBUG
rv.preconditions.push_back(
{static_cast<U>(kMask) << kBitOffset, rv.to_add});
@@ -281,9 +301,9 @@ struct UnsignedBitField {
// in that field. This applies for fields that include the top bit of the
// underlying representation. Can be combined with other additive transforms
// for other fields.
static AddTransform<BitFieldsT> PlusTransformIgnoreOverflow(V value) {
static AddTransformer<BitFieldsT> PlusTransformIgnoreOverflow(V value) {
static_assert(kIncludesTopBit);
AddTransform<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
AddTransformer<BitFieldsT> rv{static_cast<U>(value) << kBitOffset};
return rv;
}
@@ -292,9 +312,9 @@ struct UnsignedBitField {
// applies for fields that do not include the top bit of the underlying
// representation. Can be combined with other additive transforms for other
// fields.
static AddTransform<BitFieldsT> MinusTransformPromiseNoUnderflow(V value) {
static AddTransformer<BitFieldsT> MinusTransformPromiseNoUnderflow(V value) {
static_assert(!kIncludesTopBit);
AddTransform<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
AddTransformer<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
#ifndef NDEBUG
rv.preconditions.push_back(
{static_cast<U>(kMask) << kBitOffset, rv.to_add});
@@ -306,9 +326,9 @@ struct UnsignedBitField {
// underflow in that field. This applies for fields that include the top bit
// of the underlying representation. Can be combined with other additive
// transforms for other fields.
static AddTransform<BitFieldsT> MinusTransformIgnoreUnderflow(V value) {
static AddTransformer<BitFieldsT> MinusTransformIgnoreUnderflow(V value) {
static_assert(kIncludesTopBit);
AddTransform<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
AddTransformer<BitFieldsT> rv{U{0} - (static_cast<U>(value) << kBitOffset)};
return rv;
}
};
@@ -347,22 +367,22 @@ class RelaxedBitFieldsAtomic {
return BitFieldsT{
v_.exchange(desired.underlying, std::memory_order_relaxed)};
}
void ApplyRelaxed(OrTransform<BitFieldsT> transform,
void ApplyRelaxed(const OrTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
ApplyImpl<std::memory_order_relaxed>(transform, before, after);
}
void ApplyRelaxed(AndTransform<BitFieldsT> transform,
void ApplyRelaxed(const AndTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
ApplyImpl<std::memory_order_relaxed>(transform, before, after);
}
void ApplyRelaxed(AddTransform<BitFieldsT> transform,
void ApplyRelaxed(const AddTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
ApplyImpl<std::memory_order_relaxed>(transform, before, after);
}
protected: // fns
template <std::memory_order kOrder>
void ApplyImpl(OrTransform<BitFieldsT> transform,
void ApplyImpl(const OrTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
U before_val = v_.fetch_or(transform.to_or, kOrder);
if (before) {
@@ -373,7 +393,7 @@ class RelaxedBitFieldsAtomic {
}
}
template <std::memory_order kOrder>
void ApplyImpl(AndTransform<BitFieldsT> transform,
void ApplyImpl(const AndTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
U before_val = v_.fetch_and(transform.to_and, kOrder);
if (before) {
@@ -384,7 +404,7 @@ class RelaxedBitFieldsAtomic {
}
}
template <std::memory_order kOrder>
void ApplyImpl(AddTransform<BitFieldsT> transform,
void ApplyImpl(const AddTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
U before_val = v_.fetch_add(transform.to_add, kOrder);
transform.AssertPreconditions(before_val);
@@ -428,18 +448,18 @@ class AcqRelBitFieldsAtomic : public RelaxedBitFieldsAtomic<BitFieldsT> {
return BitFieldsT{
Base::v_.exchange(desired.underlying, std::memory_order_acq_rel)};
}
void Apply(OrTransform<BitFieldsT> transform, BitFieldsT* before = nullptr,
BitFieldsT* after = nullptr) {
void Apply(const OrTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
Base::template ApplyImpl<std::memory_order_acq_rel>(transform, before,
after);
}
void Apply(AndTransform<BitFieldsT> transform, BitFieldsT* before = nullptr,
BitFieldsT* after = nullptr) {
void Apply(const AndTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
Base::template ApplyImpl<std::memory_order_acq_rel>(transform, before,
after);
}
void Apply(AddTransform<BitFieldsT> transform, BitFieldsT* before = nullptr,
BitFieldsT* after = nullptr) {
void Apply(const AddTransformer<BitFieldsT>& transform,
BitFieldsT* before = nullptr, BitFieldsT* after = nullptr) {
Base::template ApplyImpl<std::memory_order_acq_rel>(transform, before,
after);
}
util/slice_test.cc
+34
View File
@@ -622,6 +622,29 @@ TEST(BitFieldsTest, BitFields) {
ASSERT_EQ(after.Get<Field3>(), true);
ASSERT_EQ(state.Get<Field4>(), 3U);
auto transform2a = Field2::And(true) + Field3::And(false);
acqrel.Apply(transform2a, &before, &after);
ASSERT_EQ(after.Get<Field2>(), true);
ASSERT_EQ(after.Get<Field3>(), false);
auto transform2b = Field2::And(false) + Field3::And(true);
acqrel.Apply(transform2b, &before, &after);
ASSERT_EQ(after.Get<Field2>(), false);
ASSERT_EQ(after.Get<Field3>(), false);
auto transform2c = Field2::Or(true) + Field3::Or(false);
acqrel.Apply(transform2c, &before, &after);
ASSERT_EQ(after.Get<Field2>(), true);
ASSERT_EQ(after.Get<Field3>(), false);
auto transform2d = Field2::Or(false) + Field3::Or(true);
acqrel.Apply(transform2d, &before, &after);
ASSERT_EQ(after.Get<Field2>(), true);
ASSERT_EQ(after.Get<Field3>(), true);
ASSERT_EQ(state.Get<Field1>(), 45U);
ASSERT_EQ(state.Get<Field4>(), 3U);
auto transform3 = Field1::PlusTransformPromiseNoOverflow(10000U) +
Field4::MinusTransformPromiseNoUnderflow(3U);
acqrel.Apply(transform3, &before, &after);
@@ -636,6 +659,17 @@ TEST(BitFieldsTest, BitFields) {
ASSERT_EQ(after.Get<Field1>(), 9046U);
ASSERT_EQ(after.Get<Field4>(), 31U);
auto transform4a =
Field1::AndTransform(8192U + 4096U) + Field4::AndTransform(15U);
acqrel.Apply(transform4a, &before, &after);
ASSERT_EQ(after.Get<Field1>(), 8192U);
ASSERT_EQ(after.Get<Field4>(), 15U);
auto transform4b = Field1::OrTransform(127U) + Field4::OrTransform(16U);
acqrel.Apply(transform4b, &before, &after);
ASSERT_EQ(after.Get<Field1>(), 8192U + 127U);
ASSERT_EQ(after.Get<Field4>(), 31U);
// Unmodified
ASSERT_EQ(after.Get<Field2>(), true);
ASSERT_EQ(after.Get<Field3>(), true);