Deprecate random access max buffer size references - take #2 (#13288)

Summary:
This time properly marking db option as `kDeprecated` in `db_options.cc`. Original PR: https://github.com/facebook/rocksdb/pull/13278.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/13288

Reviewed By: pdillinger

Differential Revision: D68024379

Pulled By: mszeszko-meta

fbshipit-source-id: 8e1f08b048ccf5971d899811edaf0b0ef16581ef
This commit is contained in:
Maciej Szeszko
2025-01-10 15:32:38 -08:00
committed by Facebook GitHub Bot
parent 3040868e5f
commit 541761eaaa
24 changed files with 3 additions and 252 deletions
Vendored
-2
View File
@@ -1087,8 +1087,6 @@ void AssignEnvOptions(EnvOptions* env_options, const DBOptions& options) {
env_options->set_fd_cloexec = options.is_fd_close_on_exec;
env_options->bytes_per_sync = options.bytes_per_sync;
env_options->compaction_readahead_size = options.compaction_readahead_size;
env_options->random_access_max_buffer_size =
options.random_access_max_buffer_size;
env_options->rate_limiter = options.rate_limiter.get();
env_options->writable_file_max_buffer_size =
options.writable_file_max_buffer_size;
-1
View File
@@ -48,7 +48,6 @@
table_cache_numshardbits=4
max_file_opening_threads=1
writable_file_max_buffer_size=1048576
random_access_max_buffer_size=1048576
use_fsync=false
max_total_wal_size=0
max_open_files=-1
-3
View File
@@ -133,9 +133,6 @@ struct EnvOptions {
// See DBOptions doc
size_t compaction_readahead_size = 0;
// See DBOptions doc
size_t random_access_max_buffer_size = 0;
// See DBOptions doc
size_t writable_file_max_buffer_size = 1024 * 1024;
-17
View File
@@ -1065,23 +1065,6 @@ struct DBOptions {
// Dynamically changeable through SetDBOptions() API.
size_t compaction_readahead_size = 2 * 1024 * 1024;
// This is a maximum buffer size that is used by WinMmapReadableFile in
// unbuffered disk I/O mode. We need to maintain an aligned buffer for
// reads. We allow the buffer to grow until the specified value and then
// for bigger requests allocate one shot buffers. In unbuffered mode we
// always bypass read-ahead buffer at ReadaheadRandomAccessFile
// When read-ahead is required we then make use of compaction_readahead_size
// value and always try to read ahead. With read-ahead we always
// pre-allocate buffer to the size instead of growing it up to a limit.
//
// This option is currently honored only on Windows
//
// Default: 1 Mb
//
// Special value: 0 - means do not maintain per instance buffer. Allocate
// per request buffer and avoid locking.
size_t random_access_max_buffer_size = 1024 * 1024;
// This is the maximum buffer size that is used by WritableFileWriter.
// With direct IO, we need to maintain an aligned buffer for writes.
// We allow the buffer to grow until it's size hits the limit in buffered
-20
View File
@@ -249,26 +249,6 @@ jlong Java_org_rocksdb_EnvOptions_compactionReadaheadSize(JNIEnv *, jclass,
return ENV_OPTIONS_GET(jhandle, compaction_readahead_size);
}
/*
* Class: org_rocksdb_EnvOptions
* Method: setRandomAccessMaxBufferSize
* Signature: (JJ)V
*/
void Java_org_rocksdb_EnvOptions_setRandomAccessMaxBufferSize(
JNIEnv *, jclass, jlong jhandle, jlong random_access_max_buffer_size) {
ENV_OPTIONS_SET_SIZE_T(jhandle, random_access_max_buffer_size);
}
/*
* Class: org_rocksdb_EnvOptions
* Method: randomAccessMaxBufferSize
* Signature: (J)J
*/
jlong Java_org_rocksdb_EnvOptions_randomAccessMaxBufferSize(JNIEnv *, jclass,
jlong jhandle) {
return ENV_OPTIONS_GET(jhandle, random_access_max_buffer_size);
}
/*
* Class: org_rocksdb_EnvOptions
* Method: setWritableFileMaxBufferSize
-46
View File
@@ -1577,29 +1577,6 @@ jlong Java_org_rocksdb_Options_compactionReadaheadSize(JNIEnv*, jclass,
return static_cast<jlong>(opt->compaction_readahead_size);
}
/*
* Class: org_rocksdb_Options
* Method: setRandomAccessMaxBufferSize
* Signature: (JJ)V
*/
void Java_org_rocksdb_Options_setRandomAccessMaxBufferSize(
JNIEnv*, jclass, jlong jhandle, jlong jrandom_access_max_buffer_size) {
auto* opt = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
opt->random_access_max_buffer_size =
static_cast<size_t>(jrandom_access_max_buffer_size);
}
/*
* Class: org_rocksdb_Options
* Method: randomAccessMaxBufferSize
* Signature: (J)J
*/
jlong Java_org_rocksdb_Options_randomAccessMaxBufferSize(JNIEnv*, jclass,
jlong jhandle) {
auto* opt = reinterpret_cast<ROCKSDB_NAMESPACE::Options*>(jhandle);
return static_cast<jlong>(opt->random_access_max_buffer_size);
}
/*
* Class: org_rocksdb_Options
* Method: setWritableFileMaxBufferSize
@@ -7112,29 +7089,6 @@ jlong Java_org_rocksdb_DBOptions_compactionReadaheadSize(JNIEnv*, jclass,
return static_cast<jlong>(opt->compaction_readahead_size);
}
/*
* Class: org_rocksdb_DBOptions
* Method: setRandomAccessMaxBufferSize
* Signature: (JJ)V
*/
void Java_org_rocksdb_DBOptions_setRandomAccessMaxBufferSize(
JNIEnv*, jclass, jlong jhandle, jlong jrandom_access_max_buffer_size) {
auto* opt = reinterpret_cast<ROCKSDB_NAMESPACE::DBOptions*>(jhandle);
opt->random_access_max_buffer_size =
static_cast<size_t>(jrandom_access_max_buffer_size);
}
/*
* Class: org_rocksdb_DBOptions
* Method: randomAccessMaxBufferSize
* Signature: (J)J
*/
jlong Java_org_rocksdb_DBOptions_randomAccessMaxBufferSize(JNIEnv*, jclass,
jlong jhandle) {
auto* opt = reinterpret_cast<ROCKSDB_NAMESPACE::DBOptions*>(jhandle);
return static_cast<jlong>(opt->random_access_max_buffer_size);
}
/*
* Class: org_rocksdb_DBOptions
* Method: setWritableFileMaxBufferSize
@@ -772,19 +772,6 @@ public class DBOptions extends RocksObject
return dailyOffpeakTimeUTC(nativeHandle_);
}
@Override
public DBOptions setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) {
assert (isOwningHandle());
setRandomAccessMaxBufferSize(nativeHandle_, randomAccessMaxBufferSize);
return this;
}
@Override
public long randomAccessMaxBufferSize() {
assert (isOwningHandle());
return randomAccessMaxBufferSize(nativeHandle_);
}
@Override
public DBOptions setWritableFileMaxBufferSize(final long writableFileMaxBufferSize) {
assert(isOwningHandle());
@@ -1364,9 +1351,6 @@ public class DBOptions extends RocksObject
private static native void setDailyOffpeakTimeUTC(
final long handle, final String dailyOffpeakTimeUTC);
private static native String dailyOffpeakTimeUTC(final long handle);
private static native void setRandomAccessMaxBufferSize(
final long handle, final long randomAccessMaxBufferSize);
private static native long randomAccessMaxBufferSize(final long handle);
private static native void setWritableFileMaxBufferSize(
final long handle, final long writableFileMaxBufferSize);
private static native long writableFileMaxBufferSize(final long handle);
@@ -938,54 +938,6 @@ public interface DBOptionsInterface<T extends DBOptionsInterface<T>> {
*/
long dbWriteBufferSize();
/**
* This is a maximum buffer size that is used by WinMmapReadableFile in
* unbuffered disk I/O mode. We need to maintain an aligned buffer for
* reads. We allow the buffer to grow until the specified value and then
* for bigger requests allocate one shot buffers. In unbuffered mode we
* always bypass read-ahead buffer at ReadaheadRandomAccessFile
* When read-ahead is required we then make use of
* {@link MutableDBOptionsInterface#compactionReadaheadSize()} value and
* always try to read ahead.
* With read-ahead we always pre-allocate buffer to the size instead of
* growing it up to a limit.
*
* This option is currently honored only on Windows
*
* Default: 1 Mb
*
* Special value: 0 - means do not maintain per instance buffer. Allocate
* per request buffer and avoid locking.
*
* @param randomAccessMaxBufferSize the maximum size of the random access
* buffer
*
* @return the reference to the current options.
*/
T setRandomAccessMaxBufferSize(long randomAccessMaxBufferSize);
/**
* This is a maximum buffer size that is used by WinMmapReadableFile in
* unbuffered disk I/O mode. We need to maintain an aligned buffer for
* reads. We allow the buffer to grow until the specified value and then
* for bigger requests allocate one shot buffers. In unbuffered mode we
* always bypass read-ahead buffer at ReadaheadRandomAccessFile
* When read-ahead is required we then make use of
* {@link MutableDBOptionsInterface#compactionReadaheadSize()} value and
* always try to read ahead. With read-ahead we always pre-allocate buffer
* to the size instead of growing it up to a limit.
*
* This option is currently honored only on Windows
*
* Default: 1 Mb
*
* Special value: 0 - means do not maintain per instance buffer. Allocate
* per request buffer and avoid locking.
*
* @return the maximum size of the random access buffer
*/
long randomAccessMaxBufferSize();
/**
* Use adaptive mutex, which spins in the user space before resorting
* to kernel. This could reduce context switch when the mutex is not
@@ -250,28 +250,6 @@ public class EnvOptions extends RocksObject {
return compactionReadaheadSize(nativeHandle_);
}
/**
* See {@link DBOptions#setRandomAccessMaxBufferSize(long)}.
*
* @param randomAccessMaxBufferSize the max buffer size for random access.
*
* @return the reference to these options.
*/
public EnvOptions setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) {
setRandomAccessMaxBufferSize(nativeHandle_, randomAccessMaxBufferSize);
return this;
}
/**
* See {@link DBOptions#randomAccessMaxBufferSize()}.
*
* @return the max buffer size for random access.
*/
public long randomAccessMaxBufferSize() {
assert (isOwningHandle());
return randomAccessMaxBufferSize(nativeHandle_);
}
/**
* See {@link DBOptions#setWritableFileMaxBufferSize(long)}.
*
@@ -350,9 +328,6 @@ public class EnvOptions extends RocksObject {
private static native void setCompactionReadaheadSize(
final long handle, final long compactionReadaheadSize);
private static native long compactionReadaheadSize(final long handle);
private static native void setRandomAccessMaxBufferSize(
final long handle, final long randomAccessMaxBufferSize);
private static native long randomAccessMaxBufferSize(final long handle);
private static native void setWritableFileMaxBufferSize(
final long handle, final long writableFileMaxBufferSize);
private static native long writableFileMaxBufferSize(final long handle);
@@ -860,19 +860,6 @@ public class Options extends RocksObject
return dailyOffpeakTimeUTC(nativeHandle_);
}
@Override
public Options setRandomAccessMaxBufferSize(final long randomAccessMaxBufferSize) {
assert (isOwningHandle());
setRandomAccessMaxBufferSize(nativeHandle_, randomAccessMaxBufferSize);
return this;
}
@Override
public long randomAccessMaxBufferSize() {
assert (isOwningHandle());
return randomAccessMaxBufferSize(nativeHandle_);
}
@Override
public Options setWritableFileMaxBufferSize(final long writableFileMaxBufferSize) {
assert(isOwningHandle());
@@ -2271,9 +2258,6 @@ public class Options extends RocksObject
private static native long compactionReadaheadSize(final long handle);
private static native void setDailyOffpeakTimeUTC(final long handle, final String offpeakTimeUTC);
private static native String dailyOffpeakTimeUTC(final long handle);
private static native void setRandomAccessMaxBufferSize(
final long handle, final long randomAccessMaxBufferSize);
private static native long randomAccessMaxBufferSize(final long handle);
private static native void setWritableFileMaxBufferSize(
final long handle, final long writableFileMaxBufferSize);
private static native long writableFileMaxBufferSize(final long handle);
@@ -462,15 +462,6 @@ public class DBOptionsTest {
}
}
@Test
public void randomAccessMaxBufferSize() {
try (final DBOptions opt = new DBOptions()) {
final long longValue = rand.nextLong();
opt.setRandomAccessMaxBufferSize(longValue);
assertThat(opt.randomAccessMaxBufferSize()).isEqualTo(longValue);
}
}
@Test
public void writableFileMaxBufferSize() {
try(final DBOptions opt = new DBOptions()) {
@@ -111,15 +111,6 @@ public class EnvOptionsTest {
}
}
@Test
public void randomAccessMaxBufferSize() {
try (final EnvOptions envOptions = new EnvOptions()) {
final int intValue = rand.nextInt(2147483647);
envOptions.setRandomAccessMaxBufferSize(intValue);
assertThat(envOptions.randomAccessMaxBufferSize()).isEqualTo(intValue);
}
}
@Test
public void writableFileMaxBufferSize() {
try (final EnvOptions envOptions = new EnvOptions()) {
@@ -708,15 +708,6 @@ public class OptionsTest {
}
}
@Test
public void randomAccessMaxBufferSize() {
try (final Options opt = new Options()) {
final long longValue = rand.nextLong();
opt.setRandomAccessMaxBufferSize(longValue);
assertThat(opt.randomAccessMaxBufferSize()).isEqualTo(longValue);
}
}
@Test
public void writableFileMaxBufferSize() {
try (final Options opt = new Options()) {
+1 -6
View File
@@ -254,8 +254,7 @@ static std::unordered_map<std::string, OptionTypeInfo>
{0, OptionType::kBoolean, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"random_access_max_buffer_size",
{offsetof(struct ImmutableDBOptions, random_access_max_buffer_size),
OptionType::kSizeT, OptionVerificationType::kNormal,
{0, OptionType::kSizeT, OptionVerificationType::kDeprecated,
OptionTypeFlags::kNone}},
{"use_adaptive_mutex",
{offsetof(struct ImmutableDBOptions, use_adaptive_mutex),
@@ -755,7 +754,6 @@ ImmutableDBOptions::ImmutableDBOptions(const DBOptions& options)
advise_random_on_open(options.advise_random_on_open),
db_write_buffer_size(options.db_write_buffer_size),
write_buffer_manager(options.write_buffer_manager),
random_access_max_buffer_size(options.random_access_max_buffer_size),
use_adaptive_mutex(options.use_adaptive_mutex),
listeners(options.listeners),
enable_thread_tracking(options.enable_thread_tracking),
@@ -905,9 +903,6 @@ void ImmutableDBOptions::Dump(Logger* log) const {
db_write_buffer_size);
ROCKS_LOG_HEADER(log, " Options.write_buffer_manager: %p",
write_buffer_manager.get());
ROCKS_LOG_HEADER(
log, " Options.random_access_max_buffer_size: %" ROCKSDB_PRIszt,
random_access_max_buffer_size);
ROCKS_LOG_HEADER(log, " Options.use_adaptive_mutex: %d",
use_adaptive_mutex);
ROCKS_LOG_HEADER(log, " Options.rate_limiter: %p",
-1
View File
@@ -62,7 +62,6 @@ struct ImmutableDBOptions {
bool advise_random_on_open;
size_t db_write_buffer_size;
std::shared_ptr<WriteBufferManager> write_buffer_manager;
size_t random_access_max_buffer_size;
bool use_adaptive_mutex;
std::vector<std::shared_ptr<EventListener>> listeners;
bool enable_thread_tracking;
-2
View File
@@ -124,8 +124,6 @@ void BuildDBOptions(const ImmutableDBOptions& immutable_db_options,
options.write_buffer_manager = immutable_db_options.write_buffer_manager;
options.compaction_readahead_size =
mutable_db_options.compaction_readahead_size;
options.random_access_max_buffer_size =
immutable_db_options.random_access_max_buffer_size;
options.writable_file_max_buffer_size =
mutable_db_options.writable_file_max_buffer_size;
options.use_adaptive_mutex = immutable_db_options.use_adaptive_mutex;
-1
View File
@@ -430,7 +430,6 @@ TEST_F(OptionsSettableTest, DBOptionsAllFieldsSettable) {
"use_direct_reads=false;"
"use_direct_io_for_flush_and_compaction=false;"
"max_log_file_size=4607;"
"random_access_max_buffer_size=1048576;"
"advise_random_on_open=true;"
"fail_if_options_file_error=false;"
"enable_pipelined_write=false;"
-4
View File
@@ -178,7 +178,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
{"advise_random_on_open", "true"},
{"use_adaptive_mutex", "false"},
{"compaction_readahead_size", "100"},
{"random_access_max_buffer_size", "3145728"},
{"writable_file_max_buffer_size", "314159"},
{"bytes_per_sync", "47"},
{"wal_bytes_per_sync", "48"},
@@ -362,7 +361,6 @@ TEST_F(OptionsTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.advise_random_on_open, true);
ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
@@ -2486,7 +2484,6 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
{"advise_random_on_open", "true"},
{"use_adaptive_mutex", "false"},
{"compaction_readahead_size", "100"},
{"random_access_max_buffer_size", "3145728"},
{"writable_file_max_buffer_size", "314159"},
{"bytes_per_sync", "47"},
{"wal_bytes_per_sync", "48"},
@@ -2674,7 +2671,6 @@ TEST_F(OptionsOldApiTest, GetOptionsFromMapTest) {
ASSERT_EQ(new_db_opt.advise_random_on_open, true);
ASSERT_EQ(new_db_opt.use_adaptive_mutex, false);
ASSERT_EQ(new_db_opt.compaction_readahead_size, 100);
ASSERT_EQ(new_db_opt.random_access_max_buffer_size, 3145728);
ASSERT_EQ(new_db_opt.writable_file_max_buffer_size, 314159);
ASSERT_EQ(new_db_opt.bytes_per_sync, static_cast<uint64_t>(47));
ASSERT_EQ(new_db_opt.wal_bytes_per_sync, static_cast<uint64_t>(48));
@@ -13,7 +13,6 @@
allow_ingest_behind=false
db_write_buffer_size=0
db_log_dir=
random_access_max_buffer_size=1048576
[CFOptions "default"]
ttl=0
@@ -10,10 +10,6 @@ conditions=log-1-true:log-2-true:log-3-true
suggestions=inc-bg-flush
conditions=log-1-true:log-4-false:log-3-true
[Rule "multiple-conds-all-false"]
suggestions=l0-l1-ratio-health-check
conditions=log-4-false:options-1-false
[Condition "log-1-true"]
source=LOG
regex=Stopping writes because we have \d+ immutable memtables \(waiting for flush\), max_write_buffer_number is set to \d+
@@ -30,11 +26,6 @@ regex=Stopping writes because we have \d+ level-0 files
source=LOG
regex=Stalling writes because of estimated pending compaction bytes \d+
[Condition "options-1-false"]
source=OPTIONS
options=CFOptions.level0_file_num_compaction_trigger:CFOptions.write_buffer_size:DBOptions.random_access_max_buffer_size
evaluate=int(options[0])*int(options[1])-int(options[2])<0 # should evaluate to a boolean
[Suggestion "inc-bg-flush"]
option=DBOptions.max_background_flushes
action=increase
+1 -2
View File
@@ -125,7 +125,7 @@ class TestConditionsConjunctions(unittest.TestCase):
# Check for the conditions
conds_triggered = ["log-1-true", "log-2-true", "log-3-true"]
conds_not_triggered = ["log-4-false", "options-1-false"]
conds_not_triggered = ["log-4-false"]
for cond in conds_triggered:
self.assertTrue(conditions_dict[cond].is_triggered(), repr(cond))
for cond in conds_not_triggered:
@@ -136,7 +136,6 @@ class TestConditionsConjunctions(unittest.TestCase):
rules_not_triggered = [
"single-condition-false",
"multiple-conds-one-false",
"multiple-conds-all-false",
]
for rule_name in rules_triggered:
rule = rules_dict[rule_name]
-4
View File
@@ -761,9 +761,6 @@ DEFINE_uint64(compaction_readahead_size,
DEFINE_int32(log_readahead_size, 0, "WAL and manifest readahead size");
DEFINE_int32(random_access_max_buffer_size, 1024 * 1024,
"Maximum windows randomaccess buffer size");
DEFINE_int32(writable_file_max_buffer_size, 1024 * 1024,
"Maximum write buffer for Writable File");
@@ -4307,7 +4304,6 @@ class Benchmark {
options.max_file_opening_threads = FLAGS_file_opening_threads;
options.compaction_readahead_size = FLAGS_compaction_readahead_size;
options.log_readahead_size = FLAGS_log_readahead_size;
options.random_access_max_buffer_size = FLAGS_random_access_max_buffer_size;
options.writable_file_max_buffer_size = FLAGS_writable_file_max_buffer_size;
options.use_fsync = FLAGS_use_fsync;
options.num_levels = FLAGS_num_levels;
-1
View File
@@ -222,7 +222,6 @@ const std::string options_file_content = R"OPTIONS_FILE(
stats_dump_period_sec=600
allow_fallocate=true
max_log_file_size=83886080
random_access_max_buffer_size=1048576
advise_random_on_open=true
dump_malloc_stats=true
@@ -0,0 +1 @@
Clean up all the references to `random_access_max_buffer_size`, related rules and all the clients wrappers. This option has been officially deprecated in 5.4.0.