jabberwocky238/rocksdb
1
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2026-07-07 14:47:40 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add per-block-type block read byte perf counters (#14473)

Browse Source 
Summary:
Pull Request resolved: https://github.com/facebook/rocksdb/pull/14473

Add separate PerfContext byte counters for data, index, filter, compression-dictionary, and metadata block reads while preserving block_read_byte as the aggregate. Wire the new counters through the block fetch and multi-read data block paths, expose them via the C perfcontext API, and extend table tests to verify byte attribution and that the classified counters sum back to block_read_byte.

Reviewed By: xingbowang

Differential Revision: D97333746

fbshipit-source-id: 3411844d7fa9c76c9ff28af477b3a72a5d6e5d9b
This commit is contained in:
Yoshinori Matsunobu
2026-03-19 14:54:24 -07:00
committed by meta-codesync[bot]
parent b6f498b2c9
commit b23fc77aca
7 changed files with 88 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
db/c.cc
+10
View File
@@ -5802,6 +5802,16 @@ uint64_t rocksdb_perfcontext_metric(rocksdb_perfcontext_t* context,
return rep->internal_range_del_reseek_count;
case rocksdb_block_read_cpu_time:
return rep->block_read_cpu_time;
case rocksdb_data_block_read_byte:
return rep->data_block_read_byte;
case rocksdb_index_block_read_byte:
return rep->index_block_read_byte;
case rocksdb_filter_block_read_byte:
return rep->filter_block_read_byte;
case rocksdb_compression_dict_block_read_byte:
return rep->compression_dict_block_read_byte;
case rocksdb_metadata_block_read_byte:
return rep->metadata_block_read_byte;
default:
break;
}
include/rocksdb/c.h
+6 -1
View File
@@ -2181,7 +2181,12 @@ enum {
rocksdb_internal_range_del_reseek_count,
rocksdb_block_read_cpu_time,
rocksdb_internal_merge_point_lookup_count,
rocksdb_total_metric_count = 80
rocksdb_data_block_read_byte,
rocksdb_index_block_read_byte,
rocksdb_filter_block_read_byte,
rocksdb_compression_dict_block_read_byte,
rocksdb_metadata_block_read_byte,
rocksdb_total_metric_count = 85
};
extern ROCKSDB_LIBRARY_API void rocksdb_set_perf_level(int);
include/rocksdb/perf_context.h
+8
View File
@@ -290,6 +290,14 @@ struct PerfContextBase {
uint64_t file_ingestion_nanos;
// Time IngestExternalFile blocked live writes.
uint64_t file_ingestion_blocking_live_writes_nanos;
// Bytes read from storage for each block category. These add up to
// block_read_byte.
uint64_t data_block_read_byte;
uint64_t index_block_read_byte;
uint64_t filter_block_read_byte;
uint64_t compression_dict_block_read_byte;
uint64_t metadata_block_read_byte;
};
struct PerfContext : public PerfContextBase {
monitoring/perf_context.cc
+6 -1
View File
@@ -155,7 +155,12 @@ struct PerfContextByLevelInt {
defCmd(decrypt_data_nanos) \
defCmd(number_async_seek) \
defCmd(file_ingestion_nanos) \
defCmd(file_ingestion_blocking_live_writes_nanos)
defCmd(file_ingestion_blocking_live_writes_nanos)\
defCmd(data_block_read_byte) \
defCmd(index_block_read_byte) \
defCmd(filter_block_read_byte) \
defCmd(compression_dict_block_read_byte) \
defCmd(metadata_block_read_byte)
// clang-format on
struct PerfContextInt {
table/block_based/block_based_table_reader_sync_and_async.h
+1
View File
@@ -117,6 +117,7 @@ DEFINE_SYNC_AND_ASYNC(void, BlockBasedTable::RetrieveMultipleBlocks)
PERF_COUNTER_ADD(block_read_count, 1);
PERF_COUNTER_ADD(block_read_byte, BlockSizeWithTrailer(handle));
PERF_COUNTER_ADD(data_block_read_byte, BlockSizeWithTrailer(handle));
}
// Handle the last block and process the pending last request
if (prev_len != 0) {
table/block_fetcher.cc
+28
View File
@@ -29,6 +29,33 @@
namespace ROCKSDB_NAMESPACE {
namespace {
inline void RecordBlockReadBytePerfCounter(BlockType block_type,
uint64_t block_size_with_trailer) {
switch (block_type) {
case BlockType::kData:
PERF_COUNTER_ADD(data_block_read_byte, block_size_with_trailer);
break;
case BlockType::kFilter:
case BlockType::kFilterPartitionIndex:
PERF_COUNTER_ADD(filter_block_read_byte, block_size_with_trailer);
break;
case BlockType::kCompressionDictionary:
PERF_COUNTER_ADD(compression_dict_block_read_byte,
block_size_with_trailer);
break;
case BlockType::kIndex:
PERF_COUNTER_ADD(index_block_read_byte, block_size_with_trailer);
break;
default:
PERF_COUNTER_ADD(metadata_block_read_byte, block_size_with_trailer);
break;
}
}
} // namespace
inline void BlockFetcher::ProcessTrailerIfPresent() {
if (footer_.GetBlockTrailerSize() > 0) {
assert(footer_.GetBlockTrailerSize() == BlockBasedTable::kBlockTrailerSize);
@@ -324,6 +351,7 @@ void BlockFetcher::ReadBlock(bool retry) {
}
PERF_COUNTER_ADD(block_read_byte, block_size_with_trailer_);
RecordBlockReadBytePerfCounter(block_type_, block_size_with_trailer_);
IGNORE_STATUS_IF_ERROR(io_status_);
if (io_status_.ok()) {
if (use_fs_scratch_ && !read_req.status.ok()) {
table/table_test.cc
+29
View File
@@ -4331,14 +4331,26 @@ TEST_P(BlockBasedTableTest, BlockReadCountTest) {
get_perf_context()->Reset();
ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context,
moptions.prefix_extractor.get()));
const uint64_t total_classified_bytes =
get_perf_context()->data_block_read_byte +
get_perf_context()->index_block_read_byte +
get_perf_context()->filter_block_read_byte +
get_perf_context()->compression_dict_block_read_byte +
get_perf_context()->metadata_block_read_byte;
ASSERT_EQ(get_perf_context()->block_read_byte, total_classified_bytes);
if (index_and_filter_in_cache) {
// data, index and filter block
ASSERT_EQ(get_perf_context()->block_read_count, 3);
ASSERT_EQ(get_perf_context()->index_block_read_count, 1);
ASSERT_EQ(get_perf_context()->filter_block_read_count, 1);
ASSERT_GT(get_perf_context()->data_block_read_byte, 0);
ASSERT_GT(get_perf_context()->index_block_read_byte, 0);
ASSERT_GT(get_perf_context()->filter_block_read_byte, 0);
} else {
// just the data block
ASSERT_EQ(get_perf_context()->block_read_count, 1);
ASSERT_EQ(get_perf_context()->block_read_byte,
get_perf_context()->data_block_read_byte);
}
ASSERT_EQ(get_context.State(), GetContext::kFound);
ASSERT_STREQ(value.data(), "hello");
@@ -4357,6 +4369,13 @@ TEST_P(BlockBasedTableTest, BlockReadCountTest) {
get_perf_context()->Reset();
ASSERT_OK(reader->Get(ReadOptions(), encoded_key, &get_context,
moptions.prefix_extractor.get()));
const uint64_t total_classified_bytes =
get_perf_context()->data_block_read_byte +
get_perf_context()->index_block_read_byte +
get_perf_context()->filter_block_read_byte +
get_perf_context()->compression_dict_block_read_byte +
get_perf_context()->metadata_block_read_byte;
ASSERT_EQ(get_perf_context()->block_read_byte, total_classified_bytes);
ASSERT_EQ(get_context.State(), GetContext::kNotFound);
}
@@ -4370,6 +4389,8 @@ TEST_P(BlockBasedTableTest, BlockReadCountTest) {
// with full-filter, we read filter first and then we stop
ASSERT_EQ(get_perf_context()->block_read_count, 1);
ASSERT_EQ(get_perf_context()->filter_block_read_count, 1);
ASSERT_EQ(get_perf_context()->block_read_byte,
get_perf_context()->filter_block_read_byte);
}
} else {
// filter is already in memory and it figures out that the key doesn't
@@ -5836,8 +5857,12 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
auto metaindex_handle = footer.metaindex_handle();
BlockContents metaindex_contents;
get_perf_context()->Reset();
BlockFetchHelper(metaindex_handle, BlockType::kMetaIndex,
&metaindex_contents);
ASSERT_GT(get_perf_context()->metadata_block_read_byte, 0);
ASSERT_EQ(get_perf_context()->block_read_byte,
get_perf_context()->metadata_block_read_byte);
Block metaindex_block(std::move(metaindex_contents));
std::unique_ptr<InternalIterator> meta_iter(metaindex_block.NewDataIterator(
@@ -5849,8 +5874,12 @@ TEST_P(BlockBasedTableTest, PropertiesBlockRestartPointTest) {
&properties_handle));
ASSERT_FALSE(properties_handle.IsNull());
BlockContents properties_contents;
get_perf_context()->Reset();
BlockFetchHelper(properties_handle, BlockType::kProperties,
&properties_contents);
ASSERT_GT(get_perf_context()->metadata_block_read_byte, 0);
ASSERT_EQ(get_perf_context()->block_read_byte,
get_perf_context()->metadata_block_read_byte);
Block properties_block(std::move(properties_contents));
ASSERT_EQ(properties_block.NumRestarts(), 1u);