Add MultiScan statistics (#14248)

Summary:
Pull Request resolved: https://github.com/facebook/rocksdb/pull/14248

### Overview

This diff introduces the addition of multi-scan statistics to RocksDB, enhancing the database's ability to monitor and analyze performance during multi-scan operations.

### Key Changes

#### Implemented Multi-Scan Statistics

The following statistics were implemented to provide deeper insights into multi-scan operations:

- **MULTISCAN_PREPARE_MICROS**: Measures the time (in microseconds) spent preparing for multi-scan operations.
- **MULTISCAN_BLOCKS_PER_PREPARE**: Tracks the number of blocks processed per multi-scan prepare operation.
- **Wasted Prefetch Blocks Count**: Counts the number of prefetched blocks that were not used (i.e., wasted) if the iterator is abandoned before accessing them.
- **MULTISCAN_TOTAL_BLOCKS_SCANNED**: Tracks the total number of blocks scanned during all multi-scan operations.
- **MULTISCAN_TOTAL_KEYS_SCANNED**: Measures the total number of keys scanned across all multi-scan operations.
- **MULTISCAN_TOTAL_MICROS**: Captures the total time (in microseconds) spent in multi-scan operations.
- **MULTISCAN_PREFETCHED_BLOCKS**: Counts the number of blocks that were prefetched during multi-scan operations.
- **MULTISCAN_USED_PREFETCH_BLOCKS**: Tracks the number of prefetched blocks that were actually used during multi-scan operations.

### Impact

This diff provides more fine-grained statistics for multi-scan operations, allowing developers and users to better understand and optimize the performance of their RocksDB instances.

Reviewed By: krhancoc

Differential Revision: D91053297

fbshipit-source-id: 7158741b9f026c0b5ce8ba1264dbd137e7fe985d

db/db_iterator_test.cc

@@ -5027,6 +5027,152 @@ TEST_P(DBMultiScanIteratorTest, AsyncPrefetchWithExternalFileIngestion) {
   ASSERT_EQ(total_keys, 400);
   iter.reset();
 }
+
+TEST_P(DBMultiScanIteratorTest, StatisticsTest) {
+  // Test that multi scan statistics are properly recorded
+  auto options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  // Use small block size to ensure multiple blocks
+  BlockBasedTableOptions table_options;
+  table_options.block_size = 256;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+
+  // Create data across multiple blocks
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(3) << std::setfill('0') << i;
+    // Use larger values to ensure multiple blocks
+    ASSERT_OK(Put("k" + ss.str(), std::string(100, 'v')));
+  }
+  ASSERT_OK(Flush());
+
+  // Reset stats before multi scan
+  ASSERT_OK(options.statistics->Reset());
+
+  // Set up two scan ranges
+  std::vector<std::string> key_ranges({"k010", "k030", "k060", "k080"});
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert(key_ranges[0], key_ranges[1]);
+  scan_options.insert(key_ranges[2], key_ranges[3]);
+
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+
+  // Iterate through all ranges
+  int count = 0;
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        (void)it;
+        count++;
+      }
+    }
+  } catch (MultiScanException& ex) {
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  }
+  ASSERT_EQ(count, 40);  // 20 keys per range
+  iter.reset();
+
+  // Check statistics
+  // MULTISCAN_PREPARE_CALLS should be at least 1
+  ASSERT_GE(TestGetTickerCount(options, MULTISCAN_PREPARE_CALLS), 1);
+
+  // MULTISCAN_PREPARE_ERRORS should be 0
+  ASSERT_EQ(TestGetTickerCount(options, MULTISCAN_PREPARE_ERRORS), 0);
+
+  // MULTISCAN_SEEK_ERRORS should be 0
+  ASSERT_EQ(TestGetTickerCount(options, MULTISCAN_SEEK_ERRORS), 0);
+
+  // Blocks should be prefetched or from cache
+  uint64_t blocks_prefetched =
+      TestGetTickerCount(options, MULTISCAN_BLOCKS_PREFETCHED);
+  uint64_t blocks_from_cache =
+      TestGetTickerCount(options, MULTISCAN_BLOCKS_FROM_CACHE);
+  ASSERT_GT(blocks_prefetched + blocks_from_cache, 0);
+
+  // If blocks were prefetched, prefetch bytes and IO requests should be > 0
+  if (blocks_prefetched > 0) {
+    ASSERT_GT(TestGetTickerCount(options, MULTISCAN_PREFETCH_BYTES), 0);
+    uint64_t io_requests = TestGetTickerCount(options, MULTISCAN_IO_REQUESTS);
+    ASSERT_GT(io_requests, 0);
+    ASSERT_LE(io_requests, blocks_prefetched);
+  }
+
+  // Wasted blocks should be 0 since we iterated through everything
+  ASSERT_EQ(TestGetTickerCount(options, MULTISCAN_PREFETCH_BLOCKS_WASTED), 0);
+}
+
+TEST_P(DBMultiScanIteratorTest, StatisticsWastedBlocksTest) {
+  // Test that wasted blocks are tracked when iteration is abandoned early
+  auto options = CurrentOptions();
+  options.statistics = CreateDBStatistics();
+  // Use small block size to ensure multiple blocks
+  BlockBasedTableOptions table_options;
+  table_options.block_size = 256;
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  DestroyAndReopen(options);
+
+  // Create data across multiple blocks
+  for (int i = 0; i < 100; ++i) {
+    std::stringstream ss;
+    ss << std::setw(3) << std::setfill('0') << i;
+    ASSERT_OK(Put("k" + ss.str(), std::string(100, 'v')));
+  }
+  ASSERT_OK(Flush());
+
+  // Reset stats before multi scan
+  ASSERT_OK(options.statistics->Reset());
+
+  // Set up a large scan range
+  ReadOptions ro;
+  ro.fill_cache = GetParam();
+  MultiScanArgs scan_options(BytewiseComparator());
+  scan_options.insert("k000", "k099");
+
+  ColumnFamilyHandle* cfh = dbfull()->DefaultColumnFamily();
+  std::unique_ptr<MultiScan> iter =
+      dbfull()->NewMultiScan(ro, cfh, scan_options);
+
+  // Only iterate through a few keys, then abandon
+  int count = 0;
+  try {
+    for (auto range : *iter) {
+      for (auto it : range) {
+        (void)it;
+        count++;
+        if (count >= 5) {
+          break;  // Abandon iteration early
+        }
+      }
+      if (count >= 5) {
+        break;
+      }
+    }
+  } catch (MultiScanException& ex) {
+    ASSERT_NOK(ex.status());
+    std::cerr << "Iterator returned status " << ex.what();
+    abort();
+  }
+  ASSERT_EQ(count, 5);
+
+  // Destroy iterator to trigger wasted blocks counting
+  iter.reset();
+
+  uint64_t blocks_prefetched =
+      TestGetTickerCount(options, MULTISCAN_BLOCKS_PREFETCHED);
+
+  // If blocks were prefetched, some should be wasted since we abandoned early
+  if (blocks_prefetched > 1) {
+    // We only read a few keys, so there should be wasted blocks
+    ASSERT_GT(TestGetTickerCount(options, MULTISCAN_PREFETCH_BLOCKS_WASTED), 0);
+  }
+}
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

include/rocksdb/statistics.h

@@ -552,6 +552,27 @@ enum Tickers : uint32_t {
   // Failure to load the UDI during SST table open
   SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
 
+  // MultiScan statistics
+  // # of Prepare() calls
+  MULTISCAN_PREPARE_CALLS,
+  // # of Prepare() calls that failed
+  MULTISCAN_PREPARE_ERRORS,
+  // # of data blocks prefetched from storage during MultiScan
+  MULTISCAN_BLOCKS_PREFETCHED,
+  // # of blocks found already in cache during MultiScan Prepare
+  MULTISCAN_BLOCKS_FROM_CACHE,
+  // Total bytes prefetched during MultiScan
+  MULTISCAN_PREFETCH_BYTES,
+  // # of prefetched blocks that were never accessed
+  MULTISCAN_PREFETCH_BLOCKS_WASTED,
+  // # of actual I/O requests issued during MultiScan
+  MULTISCAN_IO_REQUESTS,
+  // # of non-adjacent blocks coalesced into single I/O (within
+  // io_coalesce_threshold)
+  MULTISCAN_IO_COALESCED_NONADJACENT,
+  // # of seeks that failed validation (out of order, etc.)
+  MULTISCAN_SEEK_ERRORS,
+
   TICKER_ENUM_MAX
 };
 
@@ -695,6 +716,11 @@ enum Histograms : uint32_t {
   // MultiScan Prefill iterator Prepare cost
   MULTISCAN_PREPARE_ITERATORS,
 
+  // Total Prepare() latency for MultiScan
+  MULTISCAN_PREPARE_MICROS,
+  // Distribution of blocks prefetched per MultiScan Prepare()
+  MULTISCAN_BLOCKS_PER_PREPARE,
+
   HISTOGRAM_ENUM_MAX
 };
 

java/rocksjni/portal.h

@@ -5289,6 +5289,24 @@ class TickerTypeJni {
         return -0x5D;
       case ROCKSDB_NAMESPACE::Tickers::SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT:
         return -0x5E;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_CALLS:
+        return -0x60;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_ERRORS:
+        return -0x61;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_PREFETCHED:
+        return -0x62;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_FROM_CACHE:
+        return -0x63;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BYTES:
+        return -0x64;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BLOCKS_WASTED:
+        return -0x65;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_REQUESTS:
+        return -0x66;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_COALESCED_NONADJACENT:
+        return -0x67;
+      case ROCKSDB_NAMESPACE::Tickers::MULTISCAN_SEEK_ERRORS:
+        return -0x68;
       case ROCKSDB_NAMESPACE::Tickers::TICKER_ENUM_MAX:
         // -0x54 is the max value at this time. Since these values are exposed
         // directly to Java clients, we'll keep the value the same till the next
@@ -5768,6 +5786,24 @@ class TickerTypeJni {
       case -0x5E:
         return ROCKSDB_NAMESPACE::Tickers::
             SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT;
+      case -0x60:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_CALLS;
+      case -0x61:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREPARE_ERRORS;
+      case -0x62:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_PREFETCHED;
+      case -0x63:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_BLOCKS_FROM_CACHE;
+      case -0x64:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BYTES;
+      case -0x65:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_PREFETCH_BLOCKS_WASTED;
+      case -0x66:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_REQUESTS;
+      case -0x67:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_IO_COALESCED_NONADJACENT;
+      case -0x68:
+        return ROCKSDB_NAMESPACE::Tickers::MULTISCAN_SEEK_ERRORS;
       case -0x54:
         // -0x54 is the max value at this time. Since these values are exposed
         // directly to Java clients, we'll keep the value the same till the next
@@ -5924,6 +5960,10 @@ class HistogramTypeJni {
         return 0x3D;
       case ROCKSDB_NAMESPACE::Histograms::COMPACTION_PREFETCH_BYTES:
         return 0x3F;
+      case ROCKSDB_NAMESPACE::Histograms::MULTISCAN_PREPARE_MICROS:
+        return 0x40;
+      case ROCKSDB_NAMESPACE::Histograms::MULTISCAN_BLOCKS_PER_PREPARE:
+        return 0x41;
       case ROCKSDB_NAMESPACE::Histograms::HISTOGRAM_ENUM_MAX:
         // 0x3E is reserved for backwards compatibility on current minor
         // version.
@@ -6071,6 +6111,10 @@ class HistogramTypeJni {
             TABLE_OPEN_PREFETCH_TAIL_READ_BYTES;
       case 0x3F:
         return ROCKSDB_NAMESPACE::Histograms::COMPACTION_PREFETCH_BYTES;
+      case 0x40:
+        return ROCKSDB_NAMESPACE::Histograms::MULTISCAN_PREPARE_MICROS;
+      case 0x41:
+        return ROCKSDB_NAMESPACE::Histograms::MULTISCAN_BLOCKS_PER_PREPARE;
       case 0x3E:
         // 0x3E is reserved for backwards compatibility on current minor
         // version.

java/src/main/java/org/rocksdb/HistogramType.java

@@ -212,6 +212,20 @@ public enum HistogramType {
 
   COMPACTION_PREFETCH_BYTES((byte) 0x3F),
 
+  /**
+   * MultiScan histogram statistics
+   */
+
+  /**
+   * Time spent in Iterator::Prepare() for multi-scan (microseconds)
+   */
+  MULTISCAN_PREPARE_MICROS((byte) 0x40),
+
+  /**
+   * Number of blocks per multi-scan Prepare() call
+   */
+  MULTISCAN_BLOCKS_PER_PREPARE((byte) 0x41),
+
   // 0x3E is reserved for backwards compatibility on current minor version.
   HISTOGRAM_ENUM_MAX((byte) 0x3E);
 

java/src/main/java/org/rocksdb/TickerType.java

@@ -906,6 +906,55 @@ public enum TickerType {
      */
     REMOTE_COMPACT_RESUMED_BYTES((byte) -0x5F),
 
+    /**
+     * MultiScan statistics
+     */
+
+    /**
+     * # of calls to Iterator::Prepare() for multi-scan
+     */
+    MULTISCAN_PREPARE_CALLS((byte) -0x60),
+
+    /**
+     * # of errors during Iterator::Prepare() for multi-scan
+     */
+    MULTISCAN_PREPARE_ERRORS((byte) -0x61),
+
+    /**
+     * # of data blocks prefetched during multi-scan Prepare()
+     */
+    MULTISCAN_BLOCKS_PREFETCHED((byte) -0x62),
+
+    /**
+     * # of data blocks found in cache during multi-scan Prepare()
+     */
+    MULTISCAN_BLOCKS_FROM_CACHE((byte) -0x63),
+
+    /**
+     * Total bytes prefetched during multi-scan Prepare()
+     */
+    MULTISCAN_PREFETCH_BYTES((byte) -0x64),
+
+    /**
+     * # of prefetched blocks that were never accessed (wasted)
+     */
+    MULTISCAN_PREFETCH_BLOCKS_WASTED((byte) -0x65),
+
+    /**
+     * # of I/O requests issued during multi-scan Prepare()
+     */
+    MULTISCAN_IO_REQUESTS((byte) -0x66),
+
+    /**
+     * # of non-adjacent blocks coalesced into single I/O request
+     */
+    MULTISCAN_IO_COALESCED_NONADJACENT((byte) -0x67),
+
+    /**
+     * # of seek errors during multi-scan iteration
+     */
+    MULTISCAN_SEEK_ERRORS((byte) -0x68),
+
     TICKER_ENUM_MAX((byte) -0x54);
 
     private final byte value;

monitoring/statistics.cc

@@ -280,6 +280,17 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
     {NUMBER_WBWI_INGEST, "rocksdb.number.wbwi.ingest"},
     {SST_USER_DEFINED_INDEX_LOAD_FAIL_COUNT,
      "rocksdb.sst.user.defined.index.load.fail.count"},
+    {MULTISCAN_PREPARE_CALLS, "rocksdb.multiscan.prepare.calls"},
+    {MULTISCAN_PREPARE_ERRORS, "rocksdb.multiscan.prepare.errors"},
+    {MULTISCAN_BLOCKS_PREFETCHED, "rocksdb.multiscan.blocks.prefetched"},
+    {MULTISCAN_BLOCKS_FROM_CACHE, "rocksdb.multiscan.blocks.from.cache"},
+    {MULTISCAN_PREFETCH_BYTES, "rocksdb.multiscan.prefetch.bytes"},
+    {MULTISCAN_PREFETCH_BLOCKS_WASTED,
+     "rocksdb.multiscan.prefetch.blocks.wasted"},
+    {MULTISCAN_IO_REQUESTS, "rocksdb.multiscan.io.requests"},
+    {MULTISCAN_IO_COALESCED_NONADJACENT,
+     "rocksdb.multiscan.io.coalesced.nonadjacent"},
+    {MULTISCAN_SEEK_ERRORS, "rocksdb.multiscan.seek.errors"},
 };
 
 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
@@ -354,6 +365,8 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
     {NUM_OP_PER_TRANSACTION, "rocksdb.num.op.per.transaction"},
     {MULTISCAN_PREPARE_ITERATORS,
      "rocksdb.multiscan.op.prepare.iterators.micros"},
+    {MULTISCAN_PREPARE_MICROS, "rocksdb.multiscan.prepare.micros"},
+    {MULTISCAN_BLOCKS_PER_PREPARE, "rocksdb.multiscan.blocks.per.prepare"},
 };
 
 std::shared_ptr<Statistics> CreateDBStatistics() {

monitoring/stats_history_test.cc

@@ -185,7 +185,7 @@ TEST_F(StatsHistoryTest, GetStatsHistoryInMemory) {
 
 TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
   constexpr int kPeriodSec = 1;
-  constexpr int kEstimatedOneSliceSize = 16000;
+  constexpr int kEstimatedOneSliceSize = 22000;
 
   Options options;
   options.create_if_missing = true;
@@ -277,7 +277,7 @@ TEST_F(StatsHistoryTest, InMemoryStatsHistoryPurging) {
   // If `slice_count == 0` when new statistics are added, consider increasing
   // `kEstimatedOneSliceSize`
   ASSERT_EQ(slice_count, 1);
-  ASSERT_TRUE(stats_history_size_reopen < 16000 &&
+  ASSERT_TRUE(stats_history_size_reopen < kEstimatedOneSliceSize &&
               stats_history_size_reopen > 0);
   ASSERT_TRUE(stats_count_reopen < stats_count && stats_count_reopen > 0);
   Close();

table/block_based/block_based_table_iterator.cc

@@ -920,6 +920,21 @@ void BlockBasedTableIterator::BlockCacheLookupForReadAheadSize(
 }
 
 BlockBasedTableIterator::MultiScanState::~MultiScanState() {
+  // Count remaining non-empty blocks as wasted (iterator abandoned before
+  // accessing them). Start from cur_data_block_idx since blocks before that
+  // have already been processed and counted if skipped.
+  for (size_t i = cur_data_block_idx; i < pinned_data_blocks.size(); ++i) {
+    if (!pinned_data_blocks[i].IsEmpty()) {
+      ++wasted_blocks_count;
+    }
+  }
+
+  // Record wasted blocks stat
+  if (wasted_blocks_count > 0 && statistics != nullptr) {
+    RecordTick(statistics, MULTISCAN_PREFETCH_BLOCKS_WASTED,
+               wasted_blocks_count);
+  }
+
   // Abort any pending async IO operations to prevent callback being called
   // after async read states are destructed.
   if (!async_states.empty()) {
@@ -978,13 +993,19 @@ BlockBasedTableIterator::MultiScanState::~MultiScanState() {
 // moving forward.
 void BlockBasedTableIterator::Prepare(const MultiScanArgs* multiscan_opts) {
   assert(!multi_scan_);
+  RecordTick(table_->GetStatistics(), MULTISCAN_PREPARE_CALLS);
+  StopWatch sw(table_->get_rep()->ioptions.clock, table_->GetStatistics(),
+               MULTISCAN_PREPARE_MICROS);
+
   if (!index_iter_->status().ok()) {
     multi_scan_status_ = index_iter_->status();
+    RecordTick(table_->GetStatistics(), MULTISCAN_PREPARE_ERRORS);
     return;
   }
   if (multi_scan_) {
     multi_scan_.reset();
     multi_scan_status_ = Status::InvalidArgument("Prepare already called");
+    RecordTick(table_->GetStatistics(), MULTISCAN_PREPARE_ERRORS);
     return;
   }
 
@@ -998,6 +1019,7 @@ void BlockBasedTableIterator::Prepare(const MultiScanArgs* multiscan_opts) {
       CollectBlockHandles(scan_opts, &scan_block_handles,
                           &block_index_ranges_per_scan, &data_block_separators);
   if (!multi_scan_status_.ok()) {
+    RecordTick(table_->GetStatistics(), MULTISCAN_PREPARE_ERRORS);
     return;
   }
 
@@ -1010,23 +1032,44 @@ void BlockBasedTableIterator::Prepare(const MultiScanArgs* multiscan_opts) {
       scan_block_handles, multiscan_opts, &block_indices_to_read,
       &pinned_data_blocks_guard, &prefetched_max_idx);
   if (!multi_scan_status_.ok()) {
+    RecordTick(table_->GetStatistics(), MULTISCAN_PREPARE_ERRORS);
     return;
   }
 
+  // Record cache hit/miss stats
+  size_t blocks_from_cache =
+      scan_block_handles.size() - block_indices_to_read.size();
+  RecordTick(table_->GetStatistics(), MULTISCAN_BLOCKS_FROM_CACHE,
+             blocks_from_cache);
+  RecordTick(table_->GetStatistics(), MULTISCAN_BLOCKS_PREFETCHED,
+             block_indices_to_read.size());
+
   std::vector<AsyncReadState> async_states;
   // Maps from block index into async read request (index into async_states[])
   UnorderedMap<size_t, size_t> block_idx_to_readreq_idx;
   if (!block_indices_to_read.empty()) {
     std::vector<FSReadRequest> read_reqs;
     std::vector<std::vector<size_t>> coalesced_block_indices;
+    size_t nonadjacent_coalesced = 0;
+    uint64_t total_prefetch_bytes = 0;
     PrepareIORequests(block_indices_to_read, scan_block_handles, multiscan_opts,
                       &read_reqs, &block_idx_to_readreq_idx,
-                      &coalesced_block_indices);
+                      &coalesced_block_indices, &nonadjacent_coalesced,
+                      &total_prefetch_bytes);
+
+    // Record I/O stats
+    RecordTick(table_->GetStatistics(), MULTISCAN_IO_REQUESTS,
+               read_reqs.size());
+    RecordTick(table_->GetStatistics(), MULTISCAN_PREFETCH_BYTES,
+               total_prefetch_bytes);
+    RecordTick(table_->GetStatistics(), MULTISCAN_IO_COALESCED_NONADJACENT,
+               nonadjacent_coalesced);
 
     multi_scan_status_ =
         ExecuteIO(scan_block_handles, multiscan_opts, coalesced_block_indices,
                   &read_reqs, &async_states, &pinned_data_blocks_guard);
     if (!multi_scan_status_.ok()) {
+      RecordTick(table_->GetStatistics(), MULTISCAN_PREPARE_ERRORS);
       return;
     }
   }
@@ -1038,7 +1081,11 @@ void BlockBasedTableIterator::Prepare(const MultiScanArgs* multiscan_opts) {
       std::move(pinned_data_blocks_guard), std::move(data_block_separators),
       std::move(block_index_ranges_per_scan),
       std::move(block_idx_to_readreq_idx), std::move(async_states),
-      prefetched_max_idx);
+      prefetched_max_idx, table_->GetStatistics());
+
+  // Record histogram for blocks per prepare
+  RecordInHistogram(table_->GetStatistics(), MULTISCAN_BLOCKS_PER_PREPARE,
+                    scan_block_handles.size());
 
   is_index_at_curr_block_ = false;
   block_iter_points_to_real_block_ = false;
@@ -1056,6 +1103,7 @@ void BlockBasedTableIterator::SeekMultiScan(const Slice* seek_target) {
   if (!seek_target) {
     // start key must be set for multi-scan
     multi_scan_status_ = Status::InvalidArgument("No seek key for MultiScan");
+    RecordTick(table_->GetStatistics(), MULTISCAN_SEEK_ERRORS);
     return;
   }
 
@@ -1161,6 +1209,7 @@ void BlockBasedTableIterator::SeekMultiScan(const Slice* seek_target) {
         multi_scan_status_ = Status::InvalidArgument(
             "Seek target is before the previous prepared range at index " +
             std::to_string(multi_scan_->next_scan_idx));
+        RecordTick(table_->GetStatistics(), MULTISCAN_SEEK_ERRORS);
         return;
       }
       // It should only be possible to seek a key between the start of current
@@ -1248,6 +1297,7 @@ void BlockBasedTableIterator::MultiScanUnexpectedSeekTarget(
        unpin_block_idx < cur_scan_start_idx; unpin_block_idx++) {
     if (!multi_scan_->pinned_data_blocks[unpin_block_idx].IsEmpty()) {
       multi_scan_->pinned_data_blocks[unpin_block_idx].Reset();
+      ++multi_scan_->wasted_blocks_count;
     }
   }
 
@@ -1263,6 +1313,7 @@ void BlockBasedTableIterator::MultiScanUnexpectedSeekTarget(
     // Unpin the blocks that are passed
     if (!multi_scan_->pinned_data_blocks[block_idx].IsEmpty()) {
       multi_scan_->pinned_data_blocks[block_idx].Reset();
+      ++multi_scan_->wasted_blocks_count;
     }
     block_idx++;
   }
@@ -1303,6 +1354,7 @@ void BlockBasedTableIterator::MultiScanSeekTargetFromBlock(
     if (!multi_scan_->pinned_data_blocks[multi_scan_->cur_data_block_idx]
              .IsEmpty()) {
       multi_scan_->pinned_data_blocks[multi_scan_->cur_data_block_idx].Reset();
+      ++multi_scan_->wasted_blocks_count;
     }
     multi_scan_->cur_data_block_idx++;
   }
@@ -1578,9 +1630,12 @@ void BlockBasedTableIterator::PrepareIORequests(
     const std::vector<BlockHandle>& scan_block_handles,
     const MultiScanArgs* multiscan_opts, std::vector<FSReadRequest>* read_reqs,
     UnorderedMap<size_t, size_t>* block_idx_to_readreq_idx,
-    std::vector<std::vector<size_t>>* coalesced_block_indices) {
+    std::vector<std::vector<size_t>>* coalesced_block_indices,
+    size_t* nonadjacent_coalesced_count, uint64_t* total_prefetch_bytes) {
   assert(coalesced_block_indices->empty());
   coalesced_block_indices->resize(1);
+  *nonadjacent_coalesced_count = 0;
+  *total_prefetch_bytes = 0;
 
   for (const auto& block_idx : block_indices_to_read) {
     if (!coalesced_block_indices->back().empty()) {
@@ -1596,6 +1651,9 @@ void BlockBasedTableIterator::PrepareIORequests(
           last_block_end + multiscan_opts->io_coalesce_threshold) {
         // new IO
         coalesced_block_indices->emplace_back();
+      } else if (current_start > last_block_end) {
+        // Non-adjacent but within threshold, so coalesced
+        ++(*nonadjacent_coalesced_count);
       }
     }
     coalesced_block_indices->back().emplace_back(block_idx);
@@ -1648,6 +1706,7 @@ void BlockBasedTableIterator::PrepareIORequests(
     read_reqs->emplace_back();
     read_reqs->back().offset = start_offset;
     read_reqs->back().len = end_offset - start_offset;
+    *total_prefetch_bytes += read_reqs->back().len;
 
     if (multiscan_opts->use_async_io) {
       for (const auto& block_idx : block_indices) {

table/block_based/block_based_table_iterator.h

@@ -491,13 +491,18 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
     UnorderedMap<size_t, size_t> block_idx_to_readreq_idx;
     size_t prefetch_max_idx;
 
+    // For tracking wasted prefetch blocks
+    Statistics* statistics;
+    size_t wasted_blocks_count;
+
     MultiScanState(
         const std::shared_ptr<FileSystem>& _fs, const MultiScanArgs* _scan_opts,
         std::vector<CachableEntry<Block>>&& _pinned_data_blocks,
         std::vector<std::string>&& _data_block_separators,
         std::vector<std::tuple<size_t, size_t>>&& _block_index_ranges_per_scan,
         UnorderedMap<size_t, size_t>&& _block_idx_to_readreq_idx,
-        std::vector<AsyncReadState>&& _async_states, size_t _prefetch_max_idx)
+        std::vector<AsyncReadState>&& _async_states, size_t _prefetch_max_idx,
+        Statistics* _statistics)
         : fs(_fs),
           scan_opts(_scan_opts),
           pinned_data_blocks(std::move(_pinned_data_blocks)),
@@ -507,7 +512,9 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
           cur_data_block_idx(0),
           async_states(std::move(_async_states)),
           block_idx_to_readreq_idx(std::move(_block_idx_to_readreq_idx)),
-          prefetch_max_idx(_prefetch_max_idx) {}
+          prefetch_max_idx(_prefetch_max_idx),
+          statistics(_statistics),
+          wasted_blocks_count(0) {}
 
     ~MultiScanState();
   };
@@ -728,7 +735,8 @@ class BlockBasedTableIterator : public InternalIteratorBase<Slice> {
       const MultiScanArgs* multiscan_opts,
       std::vector<FSReadRequest>* read_reqs,
       UnorderedMap<size_t, size_t>* block_idx_to_readreq_idx,
-      std::vector<std::vector<size_t>>* coalesced_block_indices);
+      std::vector<std::vector<size_t>>* coalesced_block_indices,
+      size_t* nonadjacent_coalesced_count, uint64_t* total_prefetch_bytes);
 
   Status ExecuteIO(
       const std::vector<BlockHandle>& scan_block_handles,