From de7b8e8f7d34dccce4995a31f3851ccc0ca0f5d4 Mon Sep 17 00:00:00 2001 From: Xingbo Wang Date: Wed, 1 Jul 2026 10:34:16 -0700 Subject: [PATCH] Add RocksDB flush reason and atomic flush request metrics (#14898) Summary: This PR adds flush observability for debugging write stalls and flush bottlenecks. Changes include: - Add flush reason tickers for physical flush jobs: - rocksdb.flush.reason.write_buffer_full - rocksdb.flush.reason.write_buffer_manager - rocksdb.flush.reason.memtable_max_range_deletions - Add flush memtable size histograms: - rocksdb.flush.memtable.memory.bytes - rocksdb.flush.memtable.total.data.size.bytes - rocksdb.flush.write_buffer_full.memtable.memory.bytes - rocksdb.flush.write_buffer_manager.memtable.memory.bytes - Add FlushReason::kMemtableMaxRangeDeletions and propagate it from memtables when memtable_max_range_deletions triggers a flush. - Preserve atomic flush as a single request-level reason. For atomic flush, the reason is derived only from CFs that scheduled the flush, not every CF selected into the atomic flush cut. - Add atomic flush request-level counters: - rocksdb.atomic_flush.request.reason.write_buffer_full - rocksdb.atomic_flush.request.reason.write_buffer_manager - rocksdb.atomic_flush.request.reason.memtable_max_range_deletions - rocksdb.atomic_flush.request.reason.other - Keep existing per-CF flush job reason counters unchanged. Under atomic flush, rocksdb.flush.reason.* still counts physical CF flush jobs, while rocksdb.atomic_flush.request.reason.* counts one logical atomic flush request. - Wire the new flush reason, ticker, and histogram types through Java and JNI bindings. Pull Request resolved: https://github.com/facebook/rocksdb/pull/14898 Test Plan: CI Reviewed By: anand1976 Differential Revision: D110214337 Pulled By: xingbowang fbshipit-source-id: 4a445c4175d2e09ff5edbe66fde81fa00bb9e8da --- db/db_etc2_test.cc | 25 +++++++ db/db_flush_test.cc | 67 +++++++++++++++++ db/db_impl/db_impl_compaction_flush.cc | 27 +++++++ db/db_impl/db_impl_write.cc | 25 ++++++- db/db_range_del_test.cc | 73 +++++++++++++++++++ db/flush_job.cc | 21 +++++- db/memtable.cc | 9 +++ db/memtable.h | 4 + include/rocksdb/listener.h | 1 + include/rocksdb/statistics.h | 29 ++++++++ java/rocksjni/portal.h | 56 ++++++++++++++ .../main/java/org/rocksdb/FlushReason.java | 3 +- .../main/java/org/rocksdb/HistogramType.java | 20 +++++ .../src/main/java/org/rocksdb/TickerType.java | 40 ++++++++++ monitoring/statistics.cc | 20 +++++ 15 files changed, 412 insertions(+), 8 deletions(-) diff --git a/db/db_etc2_test.cc b/db/db_etc2_test.cc index d7411e8a54..14127b4225 100644 --- a/db/db_etc2_test.cc +++ b/db/db_etc2_test.cc @@ -330,6 +330,8 @@ class DBTestSharedWriteBufferAcrossCFs TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) { Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); options.arena_block_size = 4096; auto flush_listener = std::make_shared(); options.listeners.push_back(flush_listener); @@ -497,6 +499,29 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) { ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_.get(), "nikitich"), static_cast(2)); } + const uint64_t wbm_flushes = + TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_MANAGER); + ASSERT_GT(wbm_flushes, 0); + EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL)); + + HistogramData all_memtable_memory; + options.statistics->histogramData(FLUSH_MEMTABLE_MEMORY_BYTES, + &all_memtable_memory); + EXPECT_GE(all_memtable_memory.count, wbm_flushes); + EXPECT_GT(all_memtable_memory.sum, 0); + + HistogramData all_memtable_data_size; + options.statistics->histogramData(FLUSH_MEMTABLE_TOTAL_DATA_SIZE, + &all_memtable_data_size); + EXPECT_GE(all_memtable_data_size.count, wbm_flushes); + EXPECT_GT(all_memtable_data_size.sum, 0); + + HistogramData wbm_memtable_memory; + options.statistics->histogramData( + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, &wbm_memtable_memory); + EXPECT_EQ(wbm_flushes, wbm_memtable_memory.count); + EXPECT_GT(wbm_memtable_memory.sum, 0); + if (cost_cache_) { ASSERT_GE(cache->GetUsage(), 256 * 1024); Close(); diff --git a/db/db_flush_test.cc b/db/db_flush_test.cc index cc46236c3f..3fe3d01266 100644 --- a/db/db_flush_test.cc +++ b/db/db_flush_test.cc @@ -473,6 +473,61 @@ TEST_F(DBFlushTest, StatisticsGarbageBasic) { Close(); } +TEST_F(DBFlushTest, FlushReasonStatsWriteBufferFull) { + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.create_if_missing = true; + options.compression = kNoCompression; + options.disable_auto_compactions = true; + options.avoid_flush_during_shutdown = true; + options.write_buffer_size = 16 * 1024; + options.max_write_buffer_number = 8; + + auto flush_listener = std::make_shared(); + flush_listener->expected_flush_reason = FlushReason::kWriteBufferFull; + options.listeners.push_back(flush_listener); + + ASSERT_OK(TryReopen(options)); + + WriteOptions write_options; + write_options.disableWAL = true; + for (int i = 0; i < 20; ++i) { + ASSERT_OK(Put(Key(i), DummyString(10 * 1024), write_options)); + } + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + + const uint64_t write_buffer_full_flushes = + TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL); + ASSERT_GT(write_buffer_full_flushes, 0); + EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_MANAGER)); + + HistogramData all_memtable_memory; + options.statistics->histogramData(FLUSH_MEMTABLE_MEMORY_BYTES, + &all_memtable_memory); + EXPECT_GE(all_memtable_memory.count, write_buffer_full_flushes); + EXPECT_GT(all_memtable_memory.sum, 0); + + HistogramData all_memtable_data_size; + options.statistics->histogramData(FLUSH_MEMTABLE_TOTAL_DATA_SIZE, + &all_memtable_data_size); + EXPECT_GE(all_memtable_data_size.count, write_buffer_full_flushes); + EXPECT_GT(all_memtable_data_size.sum, 0); + + HistogramData write_buffer_full_memtable_memory; + options.statistics->histogramData( + FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES, + &write_buffer_full_memtable_memory); + EXPECT_EQ(write_buffer_full_flushes, write_buffer_full_memtable_memory.count); + EXPECT_GT(write_buffer_full_memtable_memory.sum, 0); + + HistogramData wbm_memtable_memory; + options.statistics->histogramData( + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, &wbm_memtable_memory); + EXPECT_EQ(0, wbm_memtable_memory.count); +} + TEST_F(DBFlushTest, StatisticsGarbageInsertAndDeletes) { Options options = CurrentOptions(); options.statistics = CreateDBStatistics(); @@ -2747,6 +2802,8 @@ TEST_P(DBAtomicFlushTest, ManualAtomicFlush) { Options options = CurrentOptions(); options.create_if_missing = true; options.atomic_flush = GetParam(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); options.write_buffer_size = (static_cast(64) << 20); auto flush_listener = std::make_shared(); flush_listener->expected_flush_reason = FlushReason::kManualFlush; @@ -2773,6 +2830,16 @@ TEST_P(DBAtomicFlushTest, ManualAtomicFlush) { } ASSERT_OK(Flush(cf_ids)); + EXPECT_EQ(options.atomic_flush ? 1 : 0, + TestGetTickerCount(options, ATOMIC_FLUSH_REQUEST_REASON_OTHER)); + EXPECT_EQ(0, TestGetTickerCount( + options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL)); + EXPECT_EQ(0, TestGetTickerCount( + options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER)); + EXPECT_EQ(0, TestGetTickerCount( + options, + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS)); + for (size_t i = 0; i != num_cfs; ++i) { auto cfh = static_cast(handles_[i]); ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed()); diff --git a/db/db_impl/db_impl_compaction_flush.cc b/db/db_impl/db_impl_compaction_flush.cc index ac3316c297..361bb3952d 100644 --- a/db/db_impl/db_impl_compaction_flush.cc +++ b/db/db_impl/db_impl_compaction_flush.cc @@ -35,6 +35,32 @@ namespace ROCKSDB_NAMESPACE { +namespace { + +void RecordAtomicFlushRequestReason(Statistics* stats, + FlushReason flush_reason) { + // Keep this aligned with the existing rocksdb.flush.reason.* counters: they + // cover automatic flush triggers used for write-stall debugging. Other + // FlushReason values are counted by the catch-all other ticker. + switch (flush_reason) { + case FlushReason::kWriteBufferFull: + RecordTick(stats, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL); + break; + case FlushReason::kWriteBufferManager: + RecordTick(stats, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER); + break; + case FlushReason::kMemtableMaxRangeDeletions: + RecordTick(stats, + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS); + break; + default: + RecordTick(stats, ATOMIC_FLUSH_REQUEST_REASON_OTHER); + break; + } +} + +} // namespace + bool DBImpl::EnoughRoomForCompaction( ColumnFamilyData* cfd, const std::vector& inputs, bool* sfm_reserved_compact_space, LogBuffer* log_buffer) { @@ -3553,6 +3579,7 @@ bool DBImpl::EnqueuePendingFlush(const FlushRequest& flush_req) { } ++unscheduled_flushes_; flush_queue_.push_back(flush_req); + RecordAtomicFlushRequestReason(stats_, flush_req.flush_reason); enqueued = true; } return enqueued; diff --git a/db/db_impl/db_impl_write.cc b/db/db_impl/db_impl_write.cc index 96a483c51b..a812952d7c 100644 --- a/db/db_impl/db_impl_write.cc +++ b/db/db_impl/db_impl_write.cc @@ -3029,12 +3029,23 @@ Status DBImpl::TrimMemtableHistory(WriteContext* context) { Status DBImpl::ScheduleFlushes(WriteContext* context) { autovector cfds; + FlushReason atomic_flush_reason = FlushReason::kWriteBufferFull; if (immutable_db_options_.atomic_flush) { + // Atomic flush has one request-level reason. Derive it only from CFs that + // scheduled the flush, not from every CF selected into the atomic cut. + ColumnFamilyData* trigger_cfd; + while ((trigger_cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) { + if (!trigger_cfd->mem()->IsEmpty() && + trigger_cfd->mem()->GetFlushReason() == + FlushReason::kMemtableMaxRangeDeletions) { + atomic_flush_reason = FlushReason::kMemtableMaxRangeDeletions; + } + trigger_cfd->UnrefAndTryDelete(); + } SelectColumnFamiliesForAtomicFlush(&cfds); for (auto cfd : cfds) { cfd->Ref(); } - flush_scheduler_.Clear(); } else { ColumnFamilyData* tmp_cfd; while ((tmp_cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) { @@ -3050,10 +3061,15 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) { TEST_SYNC_POINT_CALLBACK("DBImpl::ScheduleFlushes:PreSwitchMemtable", nullptr); + autovector flush_reasons; + flush_reasons.reserve(cfds.size()); for (auto& cfd : cfds) { + FlushReason flush_reason = FlushReason::kWriteBufferFull; if (status.ok() && !cfd->mem()->IsEmpty()) { + flush_reason = cfd->mem()->GetFlushReason(); status = SwitchMemtable(cfd, context); } + flush_reasons.push_back(flush_reason); if (cfd->UnrefAndTryDelete()) { cfd = nullptr; } @@ -3068,12 +3084,13 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) { AssignAtomicFlushSeq(cfds); FlushRequest flush_req; flush_req.atomic_flush = true; - GenerateFlushRequest(cfds, FlushReason::kWriteBufferFull, &flush_req); + GenerateFlushRequest(cfds, atomic_flush_reason, &flush_req); EnqueuePendingFlush(flush_req); } else { - for (auto* cfd : cfds) { + assert(flush_reasons.size() == cfds.size()); + for (size_t i = 0; i < cfds.size(); ++i) { FlushRequest flush_req; - GenerateFlushRequest({cfd}, FlushReason::kWriteBufferFull, &flush_req); + GenerateFlushRequest({cfds[i]}, flush_reasons[i], &flush_req); EnqueuePendingFlush(flush_req); } } diff --git a/db/db_range_del_test.cc b/db/db_range_del_test.cc index f0996ce34c..5bb5c2bd00 100644 --- a/db/db_range_del_test.cc +++ b/db/db_range_del_test.cc @@ -3559,6 +3559,79 @@ TEST_F(DBRangeDelTest, MemtableMaxRangeDeletions) { ASSERT_EQ(3, NumTableFilesAtLevel(0)); } +TEST_F(DBRangeDelTest, FlushReasonStatsMemtableMaxRangeDeletions) { + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.disable_auto_compactions = true; + options.memtable_max_range_deletions = 1; + + auto flush_listener = std::make_shared(); + flush_listener->expected_flush_reason = + FlushReason::kMemtableMaxRangeDeletions; + options.listeners.push_back(flush_listener); + + DestroyAndReopen(options); + + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(10))); + // The next write observes the pending flush request. + ASSERT_OK(Put(Key(0), "value")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + + const uint64_t range_delete_flushes = + TestGetTickerCount(options, FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS); + ASSERT_GT(range_delete_flushes, 0); + EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL)); + EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_MANAGER)); + EXPECT_EQ(0, TestGetTickerCount( + options, + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS)); + EXPECT_EQ(0, TestGetTickerCount(options, ATOMIC_FLUSH_REQUEST_REASON_OTHER)); + EXPECT_EQ(range_delete_flushes, + static_cast(flush_listener->count.load())); +} + +TEST_F(DBRangeDelTest, AtomicFlushReasonStatsMemtableMaxRangeDeletions) { + Options options = CurrentOptions(); + options.statistics = CreateDBStatistics(); + options.statistics->set_stats_level(StatsLevel::kAll); + options.atomic_flush = true; + options.disable_auto_compactions = true; + options.memtable_max_range_deletions = 1; + + auto flush_listener = std::make_shared(); + flush_listener->expected_flush_reason = + FlushReason::kMemtableMaxRangeDeletions; + options.listeners.push_back(flush_listener); + + CreateAndReopenWithCF({"pikachu"}, options); + + ASSERT_OK(Put(1, Key(0), "value")); + ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0), + Key(10))); + // The next write observes the pending flush request. + ASSERT_OK(Put(Key(0), "value")); + ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable()); + ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork()); + + const uint64_t range_delete_flushes = + TestGetTickerCount(options, FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS); + ASSERT_GE(range_delete_flushes, 2); + EXPECT_EQ(1, TestGetTickerCount( + options, + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS)); + EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL)); + EXPECT_EQ(0, TestGetTickerCount( + options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL)); + EXPECT_EQ(0, TestGetTickerCount( + options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER)); + EXPECT_EQ(0, TestGetTickerCount(options, ATOMIC_FLUSH_REQUEST_REASON_OTHER)); + EXPECT_EQ(range_delete_flushes, + static_cast(flush_listener->count.load())); +} + TEST_F(DBRangeDelTest, RangeDelReseekAfterFileReadError) { // This is to test a bug that is fixed in // https://github.com/facebook/rocksdb/pull/11786. diff --git a/db/flush_job.cc b/db/flush_job.cc index cb0d00089c..5e87466363 100644 --- a/db/flush_job.cc +++ b/db/flush_job.cc @@ -33,6 +33,7 @@ #include "logging/logging.h" #include "monitoring/iostats_context_imp.h" #include "monitoring/perf_context_imp.h" +#include "monitoring/statistics_impl.h" #include "monitoring/thread_status_util.h" #include "port/port.h" #include "rocksdb/db.h" @@ -82,6 +83,8 @@ const char* GetFlushReasonString(FlushReason flush_reason) { return "WAL Full"; case FlushReason::kCatchUpAfterErrorRecovery: return "Catch Up After Error Recovery"; + case FlushReason::kMemtableMaxRangeDeletions: + return "Memtable Max Range Deletions"; default: return "Invalid"; } @@ -959,9 +962,21 @@ Status FlushJob::WriteLevel0Table() { total_num_range_deletes += m->NumRangeDeletion(); } - // TODO(cbi): when memtable is flushed due to number of range deletions - // hitting limit memtable_max_range_deletions, flush_reason_ is still - // "Write Buffer Full", should make update flush_reason_ accordingly. + RecordInHistogram(stats_, FLUSH_MEMTABLE_MEMORY_BYTES, total_memory_usage); + RecordInHistogram(stats_, FLUSH_MEMTABLE_TOTAL_DATA_SIZE, total_data_size); + if (flush_reason_ == FlushReason::kWriteBufferFull) { + RecordTick(stats_, FLUSH_REASON_WRITE_BUFFER_FULL); + RecordInHistogram(stats_, FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES, + total_memory_usage); + } else if (flush_reason_ == FlushReason::kWriteBufferManager) { + RecordTick(stats_, FLUSH_REASON_WRITE_BUFFER_MANAGER); + RecordInHistogram(stats_, + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, + total_memory_usage); + } else if (flush_reason_ == FlushReason::kMemtableMaxRangeDeletions) { + RecordTick(stats_, FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS); + } + event_logger_->Log() << "job" << job_context_->job_id << "event" << "flush_started" << "num_memtables" << mems_.size() << "total_num_input_entries" << total_num_input_entries diff --git a/db/memtable.cc b/db/memtable.cc index 259515cdc3..6bde044fec 100644 --- a/db/memtable.cc +++ b/db/memtable.cc @@ -390,6 +390,15 @@ bool MemTable::ShouldFlushNow() { return arena_.AllocatedAndUnused() < kArenaBlockSize / 4; } +FlushReason MemTable::GetFlushReason() const { + if (memtable_max_range_deletions_ > 0 && + num_range_deletes_.LoadRelaxed() >= + static_cast(memtable_max_range_deletions_)) { + return FlushReason::kMemtableMaxRangeDeletions; + } + return FlushReason::kWriteBufferFull; +} + void MemTable::UpdateFlushState() { auto state = flush_state_.load(std::memory_order_relaxed); if (state == FLUSH_NOT_REQUESTED && ShouldFlushNow()) { diff --git a/db/memtable.h b/db/memtable.h index 2d6909bac5..c1da602391 100644 --- a/db/memtable.h +++ b/db/memtable.h @@ -858,6 +858,10 @@ class MemTable final : public ReadOnlyMemTable { // Returns a heuristic flush decision bool ShouldFlushNow(); + // Returns the reason for a flush scheduled because this memtable decided it + // should stop accepting more writes. + FlushReason GetFlushReason() const; + // Updates `fragmented_range_tombstone_list_` that will be used to serve reads // when this memtable becomes an immutable memtable (in some // MemtableListVersion::memlist_). Should be called when this memtable is diff --git a/include/rocksdb/listener.h b/include/rocksdb/listener.h index 93bec2df2b..8569da8d3f 100644 --- a/include/rocksdb/listener.h +++ b/include/rocksdb/listener.h @@ -185,6 +185,7 @@ enum class FlushReason : int { kWalFull = 0xd, // SwitchMemtable will not be called for this flush reason. kCatchUpAfterErrorRecovery = 0xe, + kMemtableMaxRangeDeletions = 0xf, // When adding flush reason, make sure to also add it to FlushReason in Java. }; diff --git a/include/rocksdb/statistics.h b/include/rocksdb/statistics.h index 8b1e52f6d3..c1c56c0760 100644 --- a/include/rocksdb/statistics.h +++ b/include/rocksdb/statistics.h @@ -629,6 +629,26 @@ enum Tickers : uint32_t { // # of times MANIFEST content validation detected corruption on DB close MANIFEST_VALIDATION_FAILURE_COUNT, + // Number of flushes triggered because the memtable reached write_buffer_size. + FLUSH_REASON_WRITE_BUFFER_FULL, + // Number of flushes triggered by WriteBufferManager memory pressure. + FLUSH_REASON_WRITE_BUFFER_MANAGER, + // Number of flushes triggered because the memtable reached + // memtable_max_range_deletions. + FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS, + // Number of atomic flush requests triggered because a memtable reached + // write_buffer_size. + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL, + // Number of atomic flush requests triggered by WriteBufferManager memory + // pressure. + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER, + // Number of atomic flush requests triggered because a memtable reached + // memtable_max_range_deletions. + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS, + // Number of atomic flush requests triggered for reasons that do not have a + // dedicated atomic flush request reason ticker. + ATOMIC_FLUSH_REQUEST_REASON_OTHER, + TICKER_ENUM_MAX }; @@ -796,6 +816,15 @@ enum Histograms : uint32_t { // Time write requests spent waiting for rate limiter refills. RATE_LIMITER_WAIT_MICROS_WRITE, + // Distribution of total memtable memory usage at flush start. + FLUSH_MEMTABLE_MEMORY_BYTES, + // Distribution of total memtable data size at flush start. + FLUSH_MEMTABLE_TOTAL_DATA_SIZE, + // Distribution of total memtable memory usage for write-buffer-full flushes. + FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES, + // Distribution of total memtable memory usage for WBM-triggered flushes. + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, + HISTOGRAM_ENUM_MAX }; diff --git a/java/rocksjni/portal.h b/java/rocksjni/portal.h index b400e51b79..7eb2263424 100644 --- a/java/rocksjni/portal.h +++ b/java/rocksjni/portal.h @@ -5339,6 +5339,24 @@ class TickerTypeJni { return -0x6A; case ROCKSDB_NAMESPACE::Tickers::MANIFEST_VALIDATION_FAILURE_COUNT: return -0x6B; + case ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_FULL: + return -0x79; + case ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_MANAGER: + return -0x7A; + case ROCKSDB_NAMESPACE::Tickers:: + FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS: + return -0x7B; + case ROCKSDB_NAMESPACE::Tickers:: + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL: + return -0x7C; + case ROCKSDB_NAMESPACE::Tickers:: + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER: + return -0x7D; + case ROCKSDB_NAMESPACE::Tickers:: + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS: + return -0x7E; + case ROCKSDB_NAMESPACE::Tickers::ATOMIC_FLUSH_REQUEST_REASON_OTHER: + return -0x7F; case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX: // -0x54 is the max value at this time. Since these values are exposed // directly to Java clients, we'll keep the value the same till the next @@ -5868,6 +5886,24 @@ class TickerTypeJni { return ROCKSDB_NAMESPACE::Tickers::READ_PATH_RANGE_TOMBSTONES_DISCARDED; case -0x6B: return ROCKSDB_NAMESPACE::Tickers::MANIFEST_VALIDATION_FAILURE_COUNT; + case -0x79: + return ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_FULL; + case -0x7A: + return ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_MANAGER; + case -0x7B: + return ROCKSDB_NAMESPACE::Tickers:: + FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS; + case -0x7C: + return ROCKSDB_NAMESPACE::Tickers:: + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL; + case -0x7D: + return ROCKSDB_NAMESPACE::Tickers:: + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER; + case -0x7E: + return ROCKSDB_NAMESPACE::Tickers:: + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS; + case -0x7F: + return ROCKSDB_NAMESPACE::Tickers::ATOMIC_FLUSH_REQUEST_REASON_OTHER; case -0x54: // -0x54 is the max value at this time. Since these values are exposed // directly to Java clients, we'll keep the value the same till the next @@ -6038,6 +6074,16 @@ class HistogramTypeJni { return 0x45; case ROCKSDB_NAMESPACE::Histograms::RATE_LIMITER_WAIT_MICROS_WRITE: return 0x46; + case ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_MEMORY_BYTES: + return 0x47; + case ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_TOTAL_DATA_SIZE: + return 0x48; + case ROCKSDB_NAMESPACE::Histograms:: + FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES: + return 0x49; + case ROCKSDB_NAMESPACE::Histograms:: + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES: + return 0x4A; case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX: // 0x3E is reserved for backwards compatibility on current minor // version. @@ -6199,6 +6245,16 @@ class HistogramTypeJni { return ROCKSDB_NAMESPACE::Histograms::RATE_LIMITER_WAIT_MICROS_READ; case 0x46: return ROCKSDB_NAMESPACE::Histograms::RATE_LIMITER_WAIT_MICROS_WRITE; + case 0x47: + return ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_MEMORY_BYTES; + case 0x48: + return ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_TOTAL_DATA_SIZE; + case 0x49: + return ROCKSDB_NAMESPACE::Histograms:: + FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES; + case 0x4A: + return ROCKSDB_NAMESPACE::Histograms:: + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES; case 0x3E: // 0x3E is reserved for backwards compatibility on current minor // version. diff --git a/java/src/main/java/org/rocksdb/FlushReason.java b/java/src/main/java/org/rocksdb/FlushReason.java index 21abbb3521..87f0c8ce32 100644 --- a/java/src/main/java/org/rocksdb/FlushReason.java +++ b/java/src/main/java/org/rocksdb/FlushReason.java @@ -20,7 +20,8 @@ public enum FlushReason { ERROR_RECOVERY((byte) 0x0b), ERROR_RECOVERY_RETRY_FLUSH((byte) 0x0c), WAL_FULL((byte) 0x0d), - CATCH_UP_AFTER_ERROR_RECOVERY((byte) 0x0e); + CATCH_UP_AFTER_ERROR_RECOVERY((byte) 0x0e), + MEMTABLE_MAX_RANGE_DELETIONS((byte) 0x0f); private final byte value; diff --git a/java/src/main/java/org/rocksdb/HistogramType.java b/java/src/main/java/org/rocksdb/HistogramType.java index 500c28f490..e4b3de06cd 100644 --- a/java/src/main/java/org/rocksdb/HistogramType.java +++ b/java/src/main/java/org/rocksdb/HistogramType.java @@ -251,6 +251,26 @@ public enum HistogramType { */ RATE_LIMITER_WAIT_MICROS_WRITE((byte) 0x46), + /** + * Total memtable memory usage at flush start. + */ + FLUSH_MEMTABLE_MEMORY_BYTES((byte) 0x47), + + /** + * Total memtable data size at flush start. + */ + FLUSH_MEMTABLE_TOTAL_DATA_SIZE((byte) 0x48), + + /** + * Total memtable memory usage for write-buffer-full flushes. + */ + FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES((byte) 0x49), + + /** + * Total memtable memory usage for WBM-triggered flushes. + */ + FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES((byte) 0x4A), + // 0x3E is reserved for backwards compatibility on current minor version. HISTOGRAM_ENUM_MAX((byte) 0x3E); diff --git a/java/src/main/java/org/rocksdb/TickerType.java b/java/src/main/java/org/rocksdb/TickerType.java index 56aeb8ae2f..2a040bf20e 100644 --- a/java/src/main/java/org/rocksdb/TickerType.java +++ b/java/src/main/java/org/rocksdb/TickerType.java @@ -1037,6 +1037,46 @@ public enum TickerType { */ MANIFEST_VALIDATION_FAILURE_COUNT((byte) -0x6B), + /** + * # of flushes triggered because the memtable reached write_buffer_size. + */ + FLUSH_REASON_WRITE_BUFFER_FULL((byte) -0x79), + + /** + * # of flushes triggered by WriteBufferManager memory pressure. + */ + FLUSH_REASON_WRITE_BUFFER_MANAGER((byte) -0x7A), + + /** + * # of flushes triggered because the memtable reached + * memtable_max_range_deletions. + */ + FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS((byte) -0x7B), + + /** + * # of atomic flush requests triggered because a memtable reached + * write_buffer_size. + */ + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL((byte) -0x7C), + + /** + * # of atomic flush requests triggered by WriteBufferManager memory + * pressure. + */ + ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER((byte) -0x7D), + + /** + * # of atomic flush requests triggered because a memtable reached + * memtable_max_range_deletions. + */ + ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS((byte) -0x7E), + + /** + * # of atomic flush requests triggered for reasons that do not have a + * dedicated atomic flush request reason ticker. + */ + ATOMIC_FLUSH_REQUEST_REASON_OTHER((byte) -0x7F), + TICKER_ENUM_MAX((byte) -0x54); private final byte value; diff --git a/monitoring/statistics.cc b/monitoring/statistics.cc index 476b468021..f82a8db601 100644 --- a/monitoring/statistics.cc +++ b/monitoring/statistics.cc @@ -321,6 +321,19 @@ const std::vector> TickersNameMap = { {FILE_OPEN_METADATA_PASSED, "rocksdb.file.open.metadata.passed"}, {MANIFEST_VALIDATION_FAILURE_COUNT, "rocksdb.manifest.validation.failure.count"}, + {FLUSH_REASON_WRITE_BUFFER_FULL, "rocksdb.flush.reason.write_buffer_full"}, + {FLUSH_REASON_WRITE_BUFFER_MANAGER, + "rocksdb.flush.reason.write_buffer_manager"}, + {FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS, + "rocksdb.flush.reason.memtable_max_range_deletions"}, + {ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL, + "rocksdb.atomic_flush.request.reason.write_buffer_full"}, + {ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER, + "rocksdb.atomic_flush.request.reason.write_buffer_manager"}, + {ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS, + "rocksdb.atomic_flush.request.reason.memtable_max_range_deletions"}, + {ATOMIC_FLUSH_REQUEST_REASON_OTHER, + "rocksdb.atomic_flush.request.reason.other"}, }; const std::vector> HistogramsNameMap = { @@ -403,6 +416,13 @@ const std::vector> HistogramsNameMap = { {INGEST_EXTERNAL_FILE_RUN_TIME, "rocksdb.ingest.external.file.run.micros"}, {RATE_LIMITER_WAIT_MICROS_READ, "rocksdb.rate.limiter.wait.micros.read"}, {RATE_LIMITER_WAIT_MICROS_WRITE, "rocksdb.rate.limiter.wait.micros.write"}, + {FLUSH_MEMTABLE_MEMORY_BYTES, "rocksdb.flush.memtable.memory.bytes"}, + {FLUSH_MEMTABLE_TOTAL_DATA_SIZE, + "rocksdb.flush.memtable.total.data.size.bytes"}, + {FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES, + "rocksdb.flush.write_buffer_full.memtable.memory.bytes"}, + {FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, + "rocksdb.flush.write_buffer_manager.memtable.memory.bytes"}, }; std::shared_ptr CreateDBStatistics() {