Add RocksDB flush reason and atomic flush request metrics (#14898)

Summary:
This PR adds flush observability for debugging write stalls and flush bottlenecks.

  Changes include:

  - Add flush reason tickers for physical flush jobs:
      - rocksdb.flush.reason.write_buffer_full
      - rocksdb.flush.reason.write_buffer_manager
      - rocksdb.flush.reason.memtable_max_range_deletions

  - Add flush memtable size histograms:
      - rocksdb.flush.memtable.memory.bytes
      - rocksdb.flush.memtable.total.data.size.bytes
      - rocksdb.flush.write_buffer_full.memtable.memory.bytes
      - rocksdb.flush.write_buffer_manager.memtable.memory.bytes

  - Add FlushReason::kMemtableMaxRangeDeletions and propagate it from memtables when memtable_max_range_deletions triggers a flush.
  - Preserve atomic flush as a single request-level reason. For atomic flush, the reason is derived only from CFs that scheduled the flush, not every CF selected into the atomic flush cut.
  - Add atomic flush request-level counters:
      - rocksdb.atomic_flush.request.reason.write_buffer_full
      - rocksdb.atomic_flush.request.reason.write_buffer_manager
      - rocksdb.atomic_flush.request.reason.memtable_max_range_deletions
      - rocksdb.atomic_flush.request.reason.other

  - Keep existing per-CF flush job reason counters unchanged. Under atomic flush, rocksdb.flush.reason.* still counts physical CF flush jobs, while rocksdb.atomic_flush.request.reason.* counts one logical atomic flush request.
  - Wire the new flush reason, ticker, and histogram types through Java and JNI bindings.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/14898

Test Plan: CI

Reviewed By: anand1976

Differential Revision: D110214337

Pulled By: xingbowang

fbshipit-source-id: 4a445c4175d2e09ff5edbe66fde81fa00bb9e8da

db/db_etc2_test.cc

@@ -330,6 +330,8 @@ class DBTestSharedWriteBufferAcrossCFs
 
 TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
   Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
   options.arena_block_size = 4096;
   auto flush_listener = std::make_shared<FlushCounterListener>();
   options.listeners.push_back(flush_listener);
@@ -497,6 +499,29 @@ TEST_P(DBTestSharedWriteBufferAcrossCFs, SharedWriteBufferAcrossCFs) {
     ASSERT_EQ(GetNumberOfSstFilesForColumnFamily(db_.get(), "nikitich"),
               static_cast<uint64_t>(2));
   }
+  const uint64_t wbm_flushes =
+      TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_MANAGER);
+  ASSERT_GT(wbm_flushes, 0);
+  EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL));
+
+  HistogramData all_memtable_memory;
+  options.statistics->histogramData(FLUSH_MEMTABLE_MEMORY_BYTES,
+                                    &all_memtable_memory);
+  EXPECT_GE(all_memtable_memory.count, wbm_flushes);
+  EXPECT_GT(all_memtable_memory.sum, 0);
+
+  HistogramData all_memtable_data_size;
+  options.statistics->histogramData(FLUSH_MEMTABLE_TOTAL_DATA_SIZE,
+                                    &all_memtable_data_size);
+  EXPECT_GE(all_memtable_data_size.count, wbm_flushes);
+  EXPECT_GT(all_memtable_data_size.sum, 0);
+
+  HistogramData wbm_memtable_memory;
+  options.statistics->histogramData(
+      FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, &wbm_memtable_memory);
+  EXPECT_EQ(wbm_flushes, wbm_memtable_memory.count);
+  EXPECT_GT(wbm_memtable_memory.sum, 0);
+
   if (cost_cache_) {
     ASSERT_GE(cache->GetUsage(), 256 * 1024);
     Close();

db/db_flush_test.cc

@@ -473,6 +473,61 @@ TEST_F(DBFlushTest, StatisticsGarbageBasic) {
   Close();
 }
 
+TEST_F(DBFlushTest, FlushReasonStatsWriteBufferFull) {
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
+  options.create_if_missing = true;
+  options.compression = kNoCompression;
+  options.disable_auto_compactions = true;
+  options.avoid_flush_during_shutdown = true;
+  options.write_buffer_size = 16 * 1024;
+  options.max_write_buffer_number = 8;
+
+  auto flush_listener = std::make_shared<FlushCounterListener>();
+  flush_listener->expected_flush_reason = FlushReason::kWriteBufferFull;
+  options.listeners.push_back(flush_listener);
+
+  ASSERT_OK(TryReopen(options));
+
+  WriteOptions write_options;
+  write_options.disableWAL = true;
+  for (int i = 0; i < 20; ++i) {
+    ASSERT_OK(Put(Key(i), DummyString(10 * 1024), write_options));
+  }
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+
+  const uint64_t write_buffer_full_flushes =
+      TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL);
+  ASSERT_GT(write_buffer_full_flushes, 0);
+  EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_MANAGER));
+
+  HistogramData all_memtable_memory;
+  options.statistics->histogramData(FLUSH_MEMTABLE_MEMORY_BYTES,
+                                    &all_memtable_memory);
+  EXPECT_GE(all_memtable_memory.count, write_buffer_full_flushes);
+  EXPECT_GT(all_memtable_memory.sum, 0);
+
+  HistogramData all_memtable_data_size;
+  options.statistics->histogramData(FLUSH_MEMTABLE_TOTAL_DATA_SIZE,
+                                    &all_memtable_data_size);
+  EXPECT_GE(all_memtable_data_size.count, write_buffer_full_flushes);
+  EXPECT_GT(all_memtable_data_size.sum, 0);
+
+  HistogramData write_buffer_full_memtable_memory;
+  options.statistics->histogramData(
+      FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES,
+      &write_buffer_full_memtable_memory);
+  EXPECT_EQ(write_buffer_full_flushes, write_buffer_full_memtable_memory.count);
+  EXPECT_GT(write_buffer_full_memtable_memory.sum, 0);
+
+  HistogramData wbm_memtable_memory;
+  options.statistics->histogramData(
+      FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES, &wbm_memtable_memory);
+  EXPECT_EQ(0, wbm_memtable_memory.count);
+}
+
 TEST_F(DBFlushTest, StatisticsGarbageInsertAndDeletes) {
   Options options = CurrentOptions();
   options.statistics = CreateDBStatistics();
@@ -2747,6 +2802,8 @@ TEST_P(DBAtomicFlushTest, ManualAtomicFlush) {
   Options options = CurrentOptions();
   options.create_if_missing = true;
   options.atomic_flush = GetParam();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
   options.write_buffer_size = (static_cast<size_t>(64) << 20);
   auto flush_listener = std::make_shared<FlushCounterListener>();
   flush_listener->expected_flush_reason = FlushReason::kManualFlush;
@@ -2773,6 +2830,16 @@ TEST_P(DBAtomicFlushTest, ManualAtomicFlush) {
   }
   ASSERT_OK(Flush(cf_ids));
 
+  EXPECT_EQ(options.atomic_flush ? 1 : 0,
+            TestGetTickerCount(options, ATOMIC_FLUSH_REQUEST_REASON_OTHER));
+  EXPECT_EQ(0, TestGetTickerCount(
+                   options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL));
+  EXPECT_EQ(0, TestGetTickerCount(
+                   options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER));
+  EXPECT_EQ(0, TestGetTickerCount(
+                   options,
+                   ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS));
+
   for (size_t i = 0; i != num_cfs; ++i) {
     auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
     ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());

db/db_impl/db_impl_compaction_flush.cc

@@ -35,6 +35,32 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+namespace {
+
+void RecordAtomicFlushRequestReason(Statistics* stats,
+                                    FlushReason flush_reason) {
+  // Keep this aligned with the existing rocksdb.flush.reason.* counters: they
+  // cover automatic flush triggers used for write-stall debugging. Other
+  // FlushReason values are counted by the catch-all other ticker.
+  switch (flush_reason) {
+    case FlushReason::kWriteBufferFull:
+      RecordTick(stats, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL);
+      break;
+    case FlushReason::kWriteBufferManager:
+      RecordTick(stats, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER);
+      break;
+    case FlushReason::kMemtableMaxRangeDeletions:
+      RecordTick(stats,
+                 ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS);
+      break;
+    default:
+      RecordTick(stats, ATOMIC_FLUSH_REQUEST_REASON_OTHER);
+      break;
+  }
+}
+
+}  // namespace
+
 bool DBImpl::EnoughRoomForCompaction(
     ColumnFamilyData* cfd, const std::vector<CompactionInputFiles>& inputs,
     bool* sfm_reserved_compact_space, LogBuffer* log_buffer) {
@@ -3553,6 +3579,7 @@ bool DBImpl::EnqueuePendingFlush(const FlushRequest& flush_req) {
     }
     ++unscheduled_flushes_;
     flush_queue_.push_back(flush_req);
+    RecordAtomicFlushRequestReason(stats_, flush_req.flush_reason);
     enqueued = true;
   }
   return enqueued;

db/db_impl/db_impl_write.cc

@@ -3029,12 +3029,23 @@ Status DBImpl::TrimMemtableHistory(WriteContext* context) {
 
 Status DBImpl::ScheduleFlushes(WriteContext* context) {
   autovector<ColumnFamilyData*> cfds;
+  FlushReason atomic_flush_reason = FlushReason::kWriteBufferFull;
   if (immutable_db_options_.atomic_flush) {
+    // Atomic flush has one request-level reason. Derive it only from CFs that
+    // scheduled the flush, not from every CF selected into the atomic cut.
+    ColumnFamilyData* trigger_cfd;
+    while ((trigger_cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
+      if (!trigger_cfd->mem()->IsEmpty() &&
+          trigger_cfd->mem()->GetFlushReason() ==
+              FlushReason::kMemtableMaxRangeDeletions) {
+        atomic_flush_reason = FlushReason::kMemtableMaxRangeDeletions;
+      }
+      trigger_cfd->UnrefAndTryDelete();
+    }
     SelectColumnFamiliesForAtomicFlush(&cfds);
     for (auto cfd : cfds) {
       cfd->Ref();
     }
-    flush_scheduler_.Clear();
   } else {
     ColumnFamilyData* tmp_cfd;
     while ((tmp_cfd = flush_scheduler_.TakeNextColumnFamily()) != nullptr) {
@@ -3050,10 +3061,15 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) {
 
   TEST_SYNC_POINT_CALLBACK("DBImpl::ScheduleFlushes:PreSwitchMemtable",
                            nullptr);
+  autovector<FlushReason> flush_reasons;
+  flush_reasons.reserve(cfds.size());
   for (auto& cfd : cfds) {
+    FlushReason flush_reason = FlushReason::kWriteBufferFull;
     if (status.ok() && !cfd->mem()->IsEmpty()) {
+      flush_reason = cfd->mem()->GetFlushReason();
       status = SwitchMemtable(cfd, context);
     }
+    flush_reasons.push_back(flush_reason);
     if (cfd->UnrefAndTryDelete()) {
       cfd = nullptr;
     }
@@ -3068,12 +3084,13 @@ Status DBImpl::ScheduleFlushes(WriteContext* context) {
       AssignAtomicFlushSeq(cfds);
       FlushRequest flush_req;
       flush_req.atomic_flush = true;
-      GenerateFlushRequest(cfds, FlushReason::kWriteBufferFull, &flush_req);
+      GenerateFlushRequest(cfds, atomic_flush_reason, &flush_req);
       EnqueuePendingFlush(flush_req);
     } else {
-      for (auto* cfd : cfds) {
+      assert(flush_reasons.size() == cfds.size());
+      for (size_t i = 0; i < cfds.size(); ++i) {
         FlushRequest flush_req;
-        GenerateFlushRequest({cfd}, FlushReason::kWriteBufferFull, &flush_req);
+        GenerateFlushRequest({cfds[i]}, flush_reasons[i], &flush_req);
         EnqueuePendingFlush(flush_req);
       }
     }

db/db_range_del_test.cc

@@ -3559,6 +3559,79 @@ TEST_F(DBRangeDelTest, MemtableMaxRangeDeletions) {
   ASSERT_EQ(3, NumTableFilesAtLevel(0));
 }
 
+TEST_F(DBRangeDelTest, FlushReasonStatsMemtableMaxRangeDeletions) {
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
+  options.disable_auto_compactions = true;
+  options.memtable_max_range_deletions = 1;
+
+  auto flush_listener = std::make_shared<FlushCounterListener>();
+  flush_listener->expected_flush_reason =
+      FlushReason::kMemtableMaxRangeDeletions;
+  options.listeners.push_back(flush_listener);
+
+  DestroyAndReopen(options);
+
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
+                             Key(10)));
+  // The next write observes the pending flush request.
+  ASSERT_OK(Put(Key(0), "value"));
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+
+  const uint64_t range_delete_flushes =
+      TestGetTickerCount(options, FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS);
+  ASSERT_GT(range_delete_flushes, 0);
+  EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL));
+  EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_MANAGER));
+  EXPECT_EQ(0, TestGetTickerCount(
+                   options,
+                   ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS));
+  EXPECT_EQ(0, TestGetTickerCount(options, ATOMIC_FLUSH_REQUEST_REASON_OTHER));
+  EXPECT_EQ(range_delete_flushes,
+            static_cast<uint64_t>(flush_listener->count.load()));
+}
+
+TEST_F(DBRangeDelTest, AtomicFlushReasonStatsMemtableMaxRangeDeletions) {
+  Options options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  options.statistics->set_stats_level(StatsLevel::kAll);
+  options.atomic_flush = true;
+  options.disable_auto_compactions = true;
+  options.memtable_max_range_deletions = 1;
+
+  auto flush_listener = std::make_shared<FlushCounterListener>();
+  flush_listener->expected_flush_reason =
+      FlushReason::kMemtableMaxRangeDeletions;
+  options.listeners.push_back(flush_listener);
+
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  ASSERT_OK(Put(1, Key(0), "value"));
+  ASSERT_OK(db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), Key(0),
+                             Key(10)));
+  // The next write observes the pending flush request.
+  ASSERT_OK(Put(Key(0), "value"));
+  ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable());
+  ASSERT_OK(dbfull()->TEST_WaitForBackgroundWork());
+
+  const uint64_t range_delete_flushes =
+      TestGetTickerCount(options, FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS);
+  ASSERT_GE(range_delete_flushes, 2);
+  EXPECT_EQ(1, TestGetTickerCount(
+                   options,
+                   ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS));
+  EXPECT_EQ(0, TestGetTickerCount(options, FLUSH_REASON_WRITE_BUFFER_FULL));
+  EXPECT_EQ(0, TestGetTickerCount(
+                   options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL));
+  EXPECT_EQ(0, TestGetTickerCount(
+                   options, ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER));
+  EXPECT_EQ(0, TestGetTickerCount(options, ATOMIC_FLUSH_REQUEST_REASON_OTHER));
+  EXPECT_EQ(range_delete_flushes,
+            static_cast<uint64_t>(flush_listener->count.load()));
+}
+
 TEST_F(DBRangeDelTest, RangeDelReseekAfterFileReadError) {
   // This is to test a bug that is fixed in
   // https://github.com/facebook/rocksdb/pull/11786.

db/flush_job.cc

@@ -33,6 +33,7 @@
 #include "logging/logging.h"
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/perf_context_imp.h"
+#include "monitoring/statistics_impl.h"
 #include "monitoring/thread_status_util.h"
 #include "port/port.h"
 #include "rocksdb/db.h"
@@ -82,6 +83,8 @@ const char* GetFlushReasonString(FlushReason flush_reason) {
       return "WAL Full";
     case FlushReason::kCatchUpAfterErrorRecovery:
       return "Catch Up After Error Recovery";
+    case FlushReason::kMemtableMaxRangeDeletions:
+      return "Memtable Max Range Deletions";
     default:
       return "Invalid";
   }
@@ -959,9 +962,21 @@ Status FlushJob::WriteLevel0Table() {
       total_num_range_deletes += m->NumRangeDeletion();
     }
 
-    // TODO(cbi): when memtable is flushed due to number of range deletions
-    //  hitting limit memtable_max_range_deletions, flush_reason_ is still
-    //  "Write Buffer Full", should make update flush_reason_ accordingly.
+    RecordInHistogram(stats_, FLUSH_MEMTABLE_MEMORY_BYTES, total_memory_usage);
+    RecordInHistogram(stats_, FLUSH_MEMTABLE_TOTAL_DATA_SIZE, total_data_size);
+    if (flush_reason_ == FlushReason::kWriteBufferFull) {
+      RecordTick(stats_, FLUSH_REASON_WRITE_BUFFER_FULL);
+      RecordInHistogram(stats_, FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES,
+                        total_memory_usage);
+    } else if (flush_reason_ == FlushReason::kWriteBufferManager) {
+      RecordTick(stats_, FLUSH_REASON_WRITE_BUFFER_MANAGER);
+      RecordInHistogram(stats_,
+                        FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES,
+                        total_memory_usage);
+    } else if (flush_reason_ == FlushReason::kMemtableMaxRangeDeletions) {
+      RecordTick(stats_, FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS);
+    }
+
     event_logger_->Log() << "job" << job_context_->job_id << "event"
                          << "flush_started" << "num_memtables" << mems_.size()
                          << "total_num_input_entries" << total_num_input_entries

db/memtable.cc

@@ -390,6 +390,15 @@ bool MemTable::ShouldFlushNow() {
   return arena_.AllocatedAndUnused() < kArenaBlockSize / 4;
 }
 
+FlushReason MemTable::GetFlushReason() const {
+  if (memtable_max_range_deletions_ > 0 &&
+      num_range_deletes_.LoadRelaxed() >=
+          static_cast<uint64_t>(memtable_max_range_deletions_)) {
+    return FlushReason::kMemtableMaxRangeDeletions;
+  }
+  return FlushReason::kWriteBufferFull;
+}
+
 void MemTable::UpdateFlushState() {
   auto state = flush_state_.load(std::memory_order_relaxed);
   if (state == FLUSH_NOT_REQUESTED && ShouldFlushNow()) {

db/memtable.h

@@ -858,6 +858,10 @@ class MemTable final : public ReadOnlyMemTable {
   // Returns a heuristic flush decision
   bool ShouldFlushNow();
 
+  // Returns the reason for a flush scheduled because this memtable decided it
+  // should stop accepting more writes.
+  FlushReason GetFlushReason() const;
+
   // Updates `fragmented_range_tombstone_list_` that will be used to serve reads
   // when this memtable becomes an immutable memtable (in some
   // MemtableListVersion::memlist_). Should be called when this memtable is

include/rocksdb/listener.h

@@ -185,6 +185,7 @@ enum class FlushReason : int {
   kWalFull = 0xd,
   // SwitchMemtable will not be called for this flush reason.
   kCatchUpAfterErrorRecovery = 0xe,
+  kMemtableMaxRangeDeletions = 0xf,
 
   // When adding flush reason, make sure to also add it to FlushReason in Java.
 };

include/rocksdb/statistics.h

@@ -629,6 +629,26 @@ enum Tickers : uint32_t {
   // # of times MANIFEST content validation detected corruption on DB close
   MANIFEST_VALIDATION_FAILURE_COUNT,
 
+  // Number of flushes triggered because the memtable reached write_buffer_size.
+  FLUSH_REASON_WRITE_BUFFER_FULL,
+  // Number of flushes triggered by WriteBufferManager memory pressure.
+  FLUSH_REASON_WRITE_BUFFER_MANAGER,
+  // Number of flushes triggered because the memtable reached
+  // memtable_max_range_deletions.
+  FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS,
+  // Number of atomic flush requests triggered because a memtable reached
+  // write_buffer_size.
+  ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL,
+  // Number of atomic flush requests triggered by WriteBufferManager memory
+  // pressure.
+  ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER,
+  // Number of atomic flush requests triggered because a memtable reached
+  // memtable_max_range_deletions.
+  ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS,
+  // Number of atomic flush requests triggered for reasons that do not have a
+  // dedicated atomic flush request reason ticker.
+  ATOMIC_FLUSH_REQUEST_REASON_OTHER,
+
   TICKER_ENUM_MAX
 };
 
@@ -796,6 +816,15 @@ enum Histograms : uint32_t {
   // Time write requests spent waiting for rate limiter refills.
   RATE_LIMITER_WAIT_MICROS_WRITE,
 
+  // Distribution of total memtable memory usage at flush start.
+  FLUSH_MEMTABLE_MEMORY_BYTES,
+  // Distribution of total memtable data size at flush start.
+  FLUSH_MEMTABLE_TOTAL_DATA_SIZE,
+  // Distribution of total memtable memory usage for write-buffer-full flushes.
+  FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES,
+  // Distribution of total memtable memory usage for WBM-triggered flushes.
+  FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES,
+
   HISTOGRAM_ENUM_MAX
 };
 

java/rocksjni/portal.h

@@ -5339,6 +5339,24 @@ class TickerTypeJni {
         return -0x6A;
       case ROCKSDB_NAMESPACE::Tickers::MANIFEST_VALIDATION_FAILURE_COUNT:
         return -0x6B;
+      case ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_FULL:
+        return -0x79;
+      case ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_MANAGER:
+        return -0x7A;
+      case ROCKSDB_NAMESPACE::Tickers::
+          FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS:
+        return -0x7B;
+      case ROCKSDB_NAMESPACE::Tickers::
+          ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL:
+        return -0x7C;
+      case ROCKSDB_NAMESPACE::Tickers::
+          ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER:
+        return -0x7D;
+      case ROCKSDB_NAMESPACE::Tickers::
+          ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS:
+        return -0x7E;
+      case ROCKSDB_NAMESPACE::Tickers::ATOMIC_FLUSH_REQUEST_REASON_OTHER:
+        return -0x7F;
       case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
         // -0x54 is the max value at this time. Since these values are exposed
         // directly to Java clients, we'll keep the value the same till the next
@@ -5868,6 +5886,24 @@ class TickerTypeJni {
         return ROCKSDB_NAMESPACE::Tickers::READ_PATH_RANGE_TOMBSTONES_DISCARDED;
       case -0x6B:
         return ROCKSDB_NAMESPACE::Tickers::MANIFEST_VALIDATION_FAILURE_COUNT;
+      case -0x79:
+        return ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_FULL;
+      case -0x7A:
+        return ROCKSDB_NAMESPACE::Tickers::FLUSH_REASON_WRITE_BUFFER_MANAGER;
+      case -0x7B:
+        return ROCKSDB_NAMESPACE::Tickers::
+            FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS;
+      case -0x7C:
+        return ROCKSDB_NAMESPACE::Tickers::
+            ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL;
+      case -0x7D:
+        return ROCKSDB_NAMESPACE::Tickers::
+            ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER;
+      case -0x7E:
+        return ROCKSDB_NAMESPACE::Tickers::
+            ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS;
+      case -0x7F:
+        return ROCKSDB_NAMESPACE::Tickers::ATOMIC_FLUSH_REQUEST_REASON_OTHER;
       case -0x54:
         // -0x54 is the max value at this time. Since these values are exposed
         // directly to Java clients, we'll keep the value the same till the next
@@ -6038,6 +6074,16 @@ class HistogramTypeJni {
         return 0x45;
       case ROCKSDB_NAMESPACE::Histograms::RATE_LIMITER_WAIT_MICROS_WRITE:
         return 0x46;
+      case ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_MEMORY_BYTES:
+        return 0x47;
+      case ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_TOTAL_DATA_SIZE:
+        return 0x48;
+      case ROCKSDB_NAMESPACE::Histograms::
+          FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES:
+        return 0x49;
+      case ROCKSDB_NAMESPACE::Histograms::
+          FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES:
+        return 0x4A;
       case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX:
         // 0x3E is reserved for backwards compatibility on current minor
         // version.
@@ -6199,6 +6245,16 @@ class HistogramTypeJni {
         return ROCKSDB_NAMESPACE::Histograms::RATE_LIMITER_WAIT_MICROS_READ;
       case 0x46:
         return ROCKSDB_NAMESPACE::Histograms::RATE_LIMITER_WAIT_MICROS_WRITE;
+      case 0x47:
+        return ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_MEMORY_BYTES;
+      case 0x48:
+        return ROCKSDB_NAMESPACE::Histograms::FLUSH_MEMTABLE_TOTAL_DATA_SIZE;
+      case 0x49:
+        return ROCKSDB_NAMESPACE::Histograms::
+            FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES;
+      case 0x4A:
+        return ROCKSDB_NAMESPACE::Histograms::
+            FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES;
       case 0x3E:
         // 0x3E is reserved for backwards compatibility on current minor
         // version.

java/src/main/java/org/rocksdb/FlushReason.java

@@ -20,7 +20,8 @@ public enum FlushReason {
   ERROR_RECOVERY((byte) 0x0b),
   ERROR_RECOVERY_RETRY_FLUSH((byte) 0x0c),
   WAL_FULL((byte) 0x0d),
-  CATCH_UP_AFTER_ERROR_RECOVERY((byte) 0x0e);
+  CATCH_UP_AFTER_ERROR_RECOVERY((byte) 0x0e),
+  MEMTABLE_MAX_RANGE_DELETIONS((byte) 0x0f);
 
   private final byte value;
 

java/src/main/java/org/rocksdb/HistogramType.java

@@ -251,6 +251,26 @@ public enum HistogramType {
    */
   RATE_LIMITER_WAIT_MICROS_WRITE((byte) 0x46),
 
+  /**
+   * Total memtable memory usage at flush start.
+   */
+  FLUSH_MEMTABLE_MEMORY_BYTES((byte) 0x47),
+
+  /**
+   * Total memtable data size at flush start.
+   */
+  FLUSH_MEMTABLE_TOTAL_DATA_SIZE((byte) 0x48),
+
+  /**
+   * Total memtable memory usage for write-buffer-full flushes.
+   */
+  FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES((byte) 0x49),
+
+  /**
+   * Total memtable memory usage for WBM-triggered flushes.
+   */
+  FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES((byte) 0x4A),
+
   // 0x3E is reserved for backwards compatibility on current minor version.
   HISTOGRAM_ENUM_MAX((byte) 0x3E);
 

java/src/main/java/org/rocksdb/TickerType.java

@@ -1037,6 +1037,46 @@ public enum TickerType {
      */
     MANIFEST_VALIDATION_FAILURE_COUNT((byte) -0x6B),
 
+    /**
+     * # of flushes triggered because the memtable reached write_buffer_size.
+     */
+    FLUSH_REASON_WRITE_BUFFER_FULL((byte) -0x79),
+
+    /**
+     * # of flushes triggered by WriteBufferManager memory pressure.
+     */
+    FLUSH_REASON_WRITE_BUFFER_MANAGER((byte) -0x7A),
+
+    /**
+     * # of flushes triggered because the memtable reached
+     * memtable_max_range_deletions.
+     */
+    FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS((byte) -0x7B),
+
+    /**
+     * # of atomic flush requests triggered because a memtable reached
+     * write_buffer_size.
+     */
+    ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL((byte) -0x7C),
+
+    /**
+     * # of atomic flush requests triggered by WriteBufferManager memory
+     * pressure.
+     */
+    ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER((byte) -0x7D),
+
+    /**
+     * # of atomic flush requests triggered because a memtable reached
+     * memtable_max_range_deletions.
+     */
+    ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS((byte) -0x7E),
+
+    /**
+     * # of atomic flush requests triggered for reasons that do not have a
+     * dedicated atomic flush request reason ticker.
+     */
+    ATOMIC_FLUSH_REQUEST_REASON_OTHER((byte) -0x7F),
+
     TICKER_ENUM_MAX((byte) -0x54);
 
     private final byte value;

monitoring/statistics.cc

@@ -321,6 +321,19 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {FILE_OPEN_METADATA_PASSED, "rocksdb.file.open.metadata.passed"},
     {MANIFEST_VALIDATION_FAILURE_COUNT,
      "rocksdb.manifest.validation.failure.count"},
+    {FLUSH_REASON_WRITE_BUFFER_FULL, "rocksdb.flush.reason.write_buffer_full"},
+    {FLUSH_REASON_WRITE_BUFFER_MANAGER,
+     "rocksdb.flush.reason.write_buffer_manager"},
+    {FLUSH_REASON_MEMTABLE_MAX_RANGE_DELETIONS,
+     "rocksdb.flush.reason.memtable_max_range_deletions"},
+    {ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_FULL,
+     "rocksdb.atomic_flush.request.reason.write_buffer_full"},
+    {ATOMIC_FLUSH_REQUEST_REASON_WRITE_BUFFER_MANAGER,
+     "rocksdb.atomic_flush.request.reason.write_buffer_manager"},
+    {ATOMIC_FLUSH_REQUEST_REASON_MEMTABLE_MAX_RANGE_DELETIONS,
+     "rocksdb.atomic_flush.request.reason.memtable_max_range_deletions"},
+    {ATOMIC_FLUSH_REQUEST_REASON_OTHER,
+     "rocksdb.atomic_flush.request.reason.other"},
 };
 
 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
@@ -403,6 +416,13 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
     {INGEST_EXTERNAL_FILE_RUN_TIME, "rocksdb.ingest.external.file.run.micros"},
     {RATE_LIMITER_WAIT_MICROS_READ, "rocksdb.rate.limiter.wait.micros.read"},
     {RATE_LIMITER_WAIT_MICROS_WRITE, "rocksdb.rate.limiter.wait.micros.write"},
+    {FLUSH_MEMTABLE_MEMORY_BYTES, "rocksdb.flush.memtable.memory.bytes"},
+    {FLUSH_MEMTABLE_TOTAL_DATA_SIZE,
+     "rocksdb.flush.memtable.total.data.size.bytes"},
+    {FLUSH_WRITE_BUFFER_FULL_MEMTABLE_MEMORY_BYTES,
+     "rocksdb.flush.write_buffer_full.memtable.memory.bytes"},
+    {FLUSH_WRITE_BUFFER_MANAGER_MEMTABLE_MEMORY_BYTES,
+     "rocksdb.flush.write_buffer_manager.memtable.memory.bytes"},
 };
 
 std::shared_ptr<Statistics> CreateDBStatistics() {