mirror of
https://github.com/facebook/rocksdb.git
synced 2026-07-07 14:47:40 +08:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Yi Wu
|
849dca7221 |
@@ -45,8 +45,6 @@ etags
|
||||
rocksdb_dump
|
||||
rocksdb_undump
|
||||
db_test2
|
||||
trace_analyzer
|
||||
trace_analyzer_test
|
||||
|
||||
java/out
|
||||
java/target
|
||||
|
||||
+14
-57
@@ -15,23 +15,13 @@ cache:
|
||||
- apt
|
||||
|
||||
addons:
|
||||
apt:
|
||||
sources:
|
||||
- ubuntu-toolchain-r-test
|
||||
packages:
|
||||
- curl
|
||||
- g++-8
|
||||
- libbz2-dev
|
||||
- libgflags-dev
|
||||
- libsnappy-dev
|
||||
- mingw-w64
|
||||
- zlib1g-dev
|
||||
apt:
|
||||
packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
|
||||
env:
|
||||
- TEST_GROUP=platform_dependent # 16-18 minutes
|
||||
- TEST_GROUP=1 # 33-35 minutes
|
||||
- TEST_GROUP=2 # 18-20 minutes
|
||||
- TEST_GROUP=3 # 20-22 minutes
|
||||
- TEST_GROUP=4 # 12-14 minutes
|
||||
- TEST_GROUP=2 # 30-32 minutes
|
||||
- TEST_GROUP=3 # ? minutes - under development
|
||||
# Run java tests
|
||||
- JOB_NAME=java_test # 4-11 minutes
|
||||
# Build ROCKSDB_LITE
|
||||
@@ -39,7 +29,6 @@ env:
|
||||
# Build examples
|
||||
- JOB_NAME=examples # 5-7 minutes
|
||||
- JOB_NAME=cmake # 3-5 minutes
|
||||
- JOB_NAME=cmake-gcc8 # 3-5 minutes
|
||||
- JOB_NAME=cmake-mingw # 3 minutes
|
||||
|
||||
matrix:
|
||||
@@ -50,10 +39,6 @@ matrix:
|
||||
env: TEST_GROUP=2
|
||||
- os: osx
|
||||
env: TEST_GROUP=3
|
||||
- os: osx
|
||||
env: TEST_GROUP=4
|
||||
- os: osx
|
||||
env: JOB_NAME=cmake-gcc8
|
||||
- os : osx
|
||||
env: JOB_NAME=cmake-mingw
|
||||
- os : linux
|
||||
@@ -64,12 +49,9 @@ matrix:
|
||||
# https://docs.travis-ci.com/user/caching/#ccache-cache
|
||||
install:
|
||||
- if [ "${TRAVIS_OS_NAME}" == osx ]; then
|
||||
brew install ccache zstd lz4 snappy xz;
|
||||
brew install ccache;
|
||||
PATH=$PATH:/usr/local/opt/ccache/libexec;
|
||||
fi
|
||||
- if [ "${JOB_NAME}" == cmake-gcc8 ]; then
|
||||
CC=gcc-8 && CXX=g++-8;
|
||||
fi
|
||||
- if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
|
||||
mkdir cmake-dist && curl -sfSL https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
|
||||
fi
|
||||
@@ -82,40 +64,15 @@ before_script:
|
||||
script:
|
||||
- ${CXX} --version
|
||||
- if [ `command -v ccache` ]; then ccache -C; fi
|
||||
- case $TEST_GROUP in
|
||||
platform_dependent)
|
||||
OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some
|
||||
;;
|
||||
1)
|
||||
OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=full_filter_block_test make -j4 check_some
|
||||
;;
|
||||
2)
|
||||
OPT=-DTRAVIS V=1 make -j4 tools && OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=full_filter_block_test ROCKSDBTESTS_END=write_batch_with_index_test make -j4 check_some
|
||||
;;
|
||||
3)
|
||||
OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_batch_with_index_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some
|
||||
;;
|
||||
4)
|
||||
OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some
|
||||
;;
|
||||
esac
|
||||
- case $JOB_NAME in
|
||||
java_test)
|
||||
OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest
|
||||
;;
|
||||
lite_build)
|
||||
OPT='-DTRAVIS -DROCKSDB_LITE' V=1 make -j4 static_lib tools
|
||||
;;
|
||||
examples)
|
||||
OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
|
||||
;;
|
||||
cmake-mingw)
|
||||
mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
|
||||
;;
|
||||
cmake*)
|
||||
mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release && make -j4 rocksdb rocksdbjni
|
||||
;;
|
||||
esac
|
||||
- if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
|
||||
- if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
|
||||
- if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some; fi
|
||||
- if [ "${TEST_GROUP}" == '3' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some; fi
|
||||
- if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
|
||||
- if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib tools; fi
|
||||
- if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
|
||||
- if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake -DJNI=1 .. && make -j4 rocksdb rocksdbjni; fi
|
||||
- if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni; fi
|
||||
notifications:
|
||||
email:
|
||||
- leveldb@fb.com
|
||||
|
||||
+74
-131
@@ -16,7 +16,7 @@
|
||||
# See thirdparty.inc for more information.
|
||||
# sample command: cmake -G "Visual Studio 15 Win64" -DWITH_GFLAGS=1 -DWITH_SNAPPY=1 -DWITH_JEMALLOC=1 -DWITH_JNI=1 ..
|
||||
# 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
|
||||
# or simply /m to use all avail cores)
|
||||
# or simply /m ot use all avail cores)
|
||||
# msbuild rocksdb.sln
|
||||
#
|
||||
# rocksdb.sln build features exclusions of test only code in Release. If you build ALL_BUILD then everything
|
||||
@@ -48,11 +48,6 @@ option(WITH_JEMALLOC "build with JeMalloc" OFF)
|
||||
option(WITH_SNAPPY "build with SNAPPY" OFF)
|
||||
option(WITH_LZ4 "build with lz4" OFF)
|
||||
option(WITH_ZLIB "build with zlib" OFF)
|
||||
option(WITH_ZSTD "build with zstd" OFF)
|
||||
option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files, regardles of the system code page" OFF)
|
||||
if (WITH_WINDOWS_UTF8_FILENAMES)
|
||||
add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
|
||||
endif()
|
||||
if(MSVC)
|
||||
# Defaults currently different for GFLAGS.
|
||||
# We will address find_package work a little later
|
||||
@@ -61,7 +56,7 @@ if(MSVC)
|
||||
include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
|
||||
else()
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
|
||||
# FreeBSD has jemalloc as default malloc
|
||||
# FreeBSD has jemaloc as default malloc
|
||||
# but it does not have all the jemalloc files in include/...
|
||||
set(WITH_JEMALLOC ON)
|
||||
else()
|
||||
@@ -69,10 +64,9 @@ else()
|
||||
find_package(JeMalloc REQUIRED)
|
||||
add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
|
||||
include_directories(${JEMALLOC_INCLUDE_DIR})
|
||||
list(APPEND THIRDPARTY_LIBS ${JEMALLOC_LIBRARIES})
|
||||
endif()
|
||||
endif()
|
||||
|
||||
|
||||
# No config file for this
|
||||
option(WITH_GFLAGS "build with GFlags" ON)
|
||||
if(WITH_GFLAGS)
|
||||
@@ -94,13 +88,7 @@ else()
|
||||
if(WITH_ZLIB)
|
||||
find_package(zlib REQUIRED)
|
||||
add_definitions(-DZLIB)
|
||||
if(ZLIB_INCLUDE_DIRS)
|
||||
# CMake 3
|
||||
include_directories(${ZLIB_INCLUDE_DIRS})
|
||||
else()
|
||||
# CMake 2
|
||||
include_directories(${ZLIB_INCLUDE_DIR})
|
||||
endif()
|
||||
include_directories(${ZLIB_INCLUDE_DIR})
|
||||
list(APPEND THIRDPARTY_LIBS ${ZLIB_LIBRARIES})
|
||||
endif()
|
||||
|
||||
@@ -119,6 +107,7 @@ else()
|
||||
list(APPEND THIRDPARTY_LIBS ${LZ4_LIBRARIES})
|
||||
endif()
|
||||
|
||||
option(WITH_ZSTD "build with zstd" OFF)
|
||||
if(WITH_ZSTD)
|
||||
find_package(zstd REQUIRED)
|
||||
add_definitions(-DZSTD)
|
||||
@@ -143,17 +132,19 @@ endif()
|
||||
|
||||
string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
|
||||
|
||||
|
||||
# Read rocksdb version from version.h header file.
|
||||
file(READ include/rocksdb/version.h version_header_file)
|
||||
string(REGEX MATCH "#define ROCKSDB_MAJOR ([0-9]+)" _ ${version_header_file})
|
||||
set(ROCKSDB_VERSION_MAJOR ${CMAKE_MATCH_1})
|
||||
string(REGEX MATCH "#define ROCKSDB_MINOR ([0-9]+)" _ ${version_header_file})
|
||||
set(ROCKSDB_VERSION_MINOR ${CMAKE_MATCH_1})
|
||||
string(REGEX MATCH "#define ROCKSDB_PATCH ([0-9]+)" _ ${version_header_file})
|
||||
set(ROCKSDB_VERSION_PATCH ${CMAKE_MATCH_1})
|
||||
set(ROCKSDB_VERSION ${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH})
|
||||
|
||||
set(SH_CMD "sh")
|
||||
execute_process(COMMAND
|
||||
${SH_CMD} -c "build_tools/version.sh full"
|
||||
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
||||
OUTPUT_VARIABLE ROCKSDB_VERSION
|
||||
)
|
||||
string(STRIP "${ROCKSDB_VERSION}" ROCKSDB_VERSION)
|
||||
execute_process(COMMAND
|
||||
${SH_CMD} -c "build_tools/version.sh major"
|
||||
WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
|
||||
OUTPUT_VARIABLE ROCKSDB_VERSION_MAJOR
|
||||
)
|
||||
string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
|
||||
|
||||
option(WITH_MD_LIBRARY "build with MD" ON)
|
||||
if(WIN32 AND MSVC)
|
||||
@@ -180,7 +171,7 @@ else()
|
||||
endif()
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
|
||||
if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -fno-omit-frame-pointer")
|
||||
include(CheckCXXCompilerFlag)
|
||||
CHECK_CXX_COMPILER_FLAG("-momit-leaf-frame-pointer" HAVE_OMIT_LEAF_FRAME_POINTER)
|
||||
if(HAVE_OMIT_LEAF_FRAME_POINTER)
|
||||
@@ -238,7 +229,6 @@ int main() {
|
||||
unset(CMAKE_REQUIRED_FLAGS)
|
||||
if(HAVE_SSE42)
|
||||
add_definitions(-DHAVE_SSE42)
|
||||
add_definitions(-DHAVE_PCLMUL)
|
||||
elseif(FORCE_SSE42)
|
||||
message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
|
||||
endif()
|
||||
@@ -295,69 +285,35 @@ if(WITH_UBSAN)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
option(WITH_NUMA "build with NUMA policy support" OFF)
|
||||
if(WITH_NUMA)
|
||||
find_package(NUMA REQUIRED)
|
||||
find_package(NUMA)
|
||||
if(NUMA_FOUND)
|
||||
add_definitions(-DNUMA)
|
||||
include_directories(${NUMA_INCLUDE_DIR})
|
||||
list(APPEND THIRDPARTY_LIBS ${NUMA_LIBRARIES})
|
||||
endif()
|
||||
|
||||
option(WITH_TBB "build with Threading Building Blocks (TBB)" OFF)
|
||||
if(WITH_TBB)
|
||||
find_package(TBB REQUIRED)
|
||||
find_package(TBB)
|
||||
if(TBB_FOUND)
|
||||
add_definitions(-DTBB)
|
||||
include_directories(${TBB_INCLUDE_DIR})
|
||||
list(APPEND THIRDPARTY_LIBS ${TBB_LIBRARIES})
|
||||
endif()
|
||||
|
||||
# Stall notifications eat some performance from inserts
|
||||
option(DISABLE_STALL_NOTIF "Build with stall notifications" OFF)
|
||||
if(DISABLE_STALL_NOTIF)
|
||||
add_definitions(-DROCKSDB_DISABLE_STALL_NOTIFICATION)
|
||||
endif()
|
||||
|
||||
|
||||
if(DEFINED USE_RTTI)
|
||||
if(USE_RTTI)
|
||||
message(STATUS "Enabling RTTI")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -DROCKSDB_USE_RTTI")
|
||||
else()
|
||||
if(MSVC)
|
||||
message(STATUS "Disabling RTTI in Release builds. Always on in Debug.")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-")
|
||||
else()
|
||||
message(STATUS "Disabling RTTI in Release builds")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -fno-rtti")
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti")
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
message(STATUS "Enabling RTTI in Debug builds only (default)")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DROCKSDB_USE_RTTI")
|
||||
if(MSVC)
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /GR-")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Used to run CI build and tests so we can run faster
|
||||
option(OPTDBG "Build optimized debug build with MSVC" OFF)
|
||||
option(WITH_RUNTIME_DEBUG "build with debug version of runtime library" ON)
|
||||
set(OPTIMIZE_DEBUG_DEFAULT 0) # Debug build is unoptimized by default use -DOPTDBG=1 to optimize
|
||||
|
||||
if(DEFINED OPTDBG)
|
||||
set(OPTIMIZE_DEBUG ${OPTDBG})
|
||||
else()
|
||||
set(OPTIMIZE_DEBUG ${OPTIMIZE_DEBUG_DEFAULT})
|
||||
endif()
|
||||
|
||||
if(MSVC)
|
||||
if(OPTDBG)
|
||||
if((${OPTIMIZE_DEBUG} EQUAL 1))
|
||||
message(STATUS "Debug optimization is enabled")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "/Oxt /${RUNTIME_LIBRARY}d")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm")
|
||||
endif()
|
||||
if(WITH_RUNTIME_DEBUG)
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d")
|
||||
else()
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}")
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm /${RUNTIME_LIBRARY}d")
|
||||
endif()
|
||||
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} /Oxt /Zp8 /Gm- /Gy /${RUNTIME_LIBRARY}")
|
||||
|
||||
@@ -372,7 +328,7 @@ endif()
|
||||
option(ROCKSDB_LITE "Build RocksDBLite version" OFF)
|
||||
if(ROCKSDB_LITE)
|
||||
add_definitions(-DROCKSDB_LITE)
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions -Os")
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-exceptions")
|
||||
endif()
|
||||
|
||||
if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
|
||||
@@ -416,7 +372,7 @@ if(WITH_FALLOCATE)
|
||||
#include <linux/falloc.h>
|
||||
int main() {
|
||||
int fd = open(\"/dev/null\", 0);
|
||||
fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
|
||||
fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
|
||||
}
|
||||
" HAVE_FALLOCATE)
|
||||
if(HAVE_FALLOCATE)
|
||||
@@ -477,7 +433,6 @@ set(SOURCES
|
||||
db/compaction_iterator.cc
|
||||
db/compaction_job.cc
|
||||
db/compaction_picker.cc
|
||||
db/compaction_picker_fifo.cc
|
||||
db/compaction_picker_universal.cc
|
||||
db/convenience.cc
|
||||
db/db_filesnapshot.cc
|
||||
@@ -489,11 +444,9 @@ set(SOURCES
|
||||
db/db_impl_debug.cc
|
||||
db/db_impl_experimental.cc
|
||||
db/db_impl_readonly.cc
|
||||
db/db_impl_secondary.cc
|
||||
db/db_info_dumper.cc
|
||||
db/db_iter.cc
|
||||
db/dbformat.cc
|
||||
db/error_handler.cc
|
||||
db/event_helpers.cc
|
||||
db/experimental.cc
|
||||
db/external_sst_file_ingestion_job.cc
|
||||
@@ -502,17 +455,15 @@ set(SOURCES
|
||||
db/flush_scheduler.cc
|
||||
db/forward_iterator.cc
|
||||
db/internal_stats.cc
|
||||
db/in_memory_stats_history.cc
|
||||
db/logs_with_prep_tracker.cc
|
||||
db/log_reader.cc
|
||||
db/log_writer.cc
|
||||
db/malloc_stats.cc
|
||||
db/managed_iterator.cc
|
||||
db/memtable.cc
|
||||
db/memtable_list.cc
|
||||
db/merge_helper.cc
|
||||
db/merge_operator.cc
|
||||
db/range_del_aggregator.cc
|
||||
db/range_tombstone_fragmenter.cc
|
||||
db/repair.cc
|
||||
db/snapshot_impl.cc
|
||||
db/table_cache.cc
|
||||
@@ -532,6 +483,7 @@ set(SOURCES
|
||||
env/env_hdfs.cc
|
||||
env/mock_env.cc
|
||||
memtable/alloc_tracker.cc
|
||||
memtable/hash_cuckoo_rep.cc
|
||||
memtable/hash_linklist_rep.cc
|
||||
memtable/hash_skiplist_rep.cc
|
||||
memtable/skiplistrep.cc
|
||||
@@ -568,8 +520,6 @@ set(SOURCES
|
||||
table/cuckoo_table_builder.cc
|
||||
table/cuckoo_table_factory.cc
|
||||
table/cuckoo_table_reader.cc
|
||||
table/data_block_hash_index.cc
|
||||
table/data_block_footer.cc
|
||||
table/flush_block_policy.cc
|
||||
table/format.cc
|
||||
table/full_filter_block.cc
|
||||
@@ -585,7 +535,6 @@ set(SOURCES
|
||||
table/plain_table_index.cc
|
||||
table/plain_table_key_coding.cc
|
||||
table/plain_table_reader.cc
|
||||
table/sst_file_reader.cc
|
||||
table/sst_file_writer.cc
|
||||
table/table_properties.cc
|
||||
table/two_level_iterator.cc
|
||||
@@ -594,16 +543,13 @@ set(SOURCES
|
||||
tools/ldb_cmd.cc
|
||||
tools/ldb_tool.cc
|
||||
tools/sst_dump_tool.cc
|
||||
tools/trace_analyzer_tool.cc
|
||||
util/arena.cc
|
||||
util/auto_roll_logger.cc
|
||||
util/bloom.cc
|
||||
util/coding.cc
|
||||
util/compaction_job_stats_impl.cc
|
||||
util/comparator.cc
|
||||
util/compression_context_cache.cc
|
||||
util/concurrent_arena.cc
|
||||
util/concurrent_task_limiter_impl.cc
|
||||
util/crc32c.cc
|
||||
util/delete_scheduler.cc
|
||||
util/dynamic_bloom.cc
|
||||
@@ -613,7 +559,6 @@ set(SOURCES
|
||||
util/filename.cc
|
||||
util/filter_policy.cc
|
||||
util/hash.cc
|
||||
util/jemalloc_nodump_allocator.cc
|
||||
util/log_buffer.cc
|
||||
util/murmurhash.cc
|
||||
util/random.cc
|
||||
@@ -621,36 +566,42 @@ set(SOURCES
|
||||
util/slice.cc
|
||||
util/sst_file_manager_impl.cc
|
||||
util/status.cc
|
||||
util/status_message.cc
|
||||
util/string_util.cc
|
||||
util/sync_point.cc
|
||||
util/sync_point_impl.cc
|
||||
util/testutil.cc
|
||||
util/thread_local.cc
|
||||
util/threadpool_imp.cc
|
||||
util/trace_replay.cc
|
||||
util/transaction_test_util.cc
|
||||
util/xxhash.cc
|
||||
utilities/backupable/backupable_db.cc
|
||||
utilities/blob_db/blob_compaction_filter.cc
|
||||
utilities/blob_db/blob_db.cc
|
||||
utilities/blob_db/blob_db_impl.cc
|
||||
utilities/blob_db/blob_db_impl_filesnapshot.cc
|
||||
utilities/blob_db/blob_dump_tool.cc
|
||||
utilities/blob_db/blob_file.cc
|
||||
utilities/blob_db/blob_log_reader.cc
|
||||
utilities/blob_db/blob_log_writer.cc
|
||||
utilities/blob_db/blob_log_format.cc
|
||||
utilities/blob_db/ttl_extractor.cc
|
||||
utilities/cassandra/cassandra_compaction_filter.cc
|
||||
utilities/cassandra/format.cc
|
||||
utilities/cassandra/merge_operator.cc
|
||||
utilities/checkpoint/checkpoint_impl.cc
|
||||
utilities/col_buf_decoder.cc
|
||||
utilities/col_buf_encoder.cc
|
||||
utilities/column_aware_encoding_util.cc
|
||||
utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
|
||||
utilities/date_tiered/date_tiered_db_impl.cc
|
||||
utilities/debug.cc
|
||||
utilities/document/document_db.cc
|
||||
utilities/document/json_document.cc
|
||||
utilities/document/json_document_builder.cc
|
||||
utilities/env_mirror.cc
|
||||
utilities/env_timed.cc
|
||||
utilities/geodb/geodb_impl.cc
|
||||
utilities/leveldb_options/leveldb_options.cc
|
||||
utilities/lua/rocks_lua_compaction_filter.cc
|
||||
utilities/memory/memory_util.cc
|
||||
utilities/merge_operators/bytesxor.cc
|
||||
utilities/merge_operators/max.cc
|
||||
utilities/merge_operators/put.cc
|
||||
utilities/merge_operators/string_append/stringappend.cc
|
||||
@@ -663,9 +614,10 @@ set(SOURCES
|
||||
utilities/persistent_cache/block_cache_tier_metadata.cc
|
||||
utilities/persistent_cache/persistent_cache_tier.cc
|
||||
utilities/persistent_cache/volatile_tier_impl.cc
|
||||
utilities/redis/redis_lists.cc
|
||||
utilities/simulator_cache/sim_cache.cc
|
||||
utilities/spatialdb/spatial_db.cc
|
||||
utilities/table_properties_collectors/compact_on_deletion_collector.cc
|
||||
utilities/trace/file_trace_reader_writer.cc
|
||||
utilities/transactions/optimistic_transaction_db_impl.cc
|
||||
utilities/transactions/optimistic_transaction.cc
|
||||
utilities/transactions/pessimistic_transaction.cc
|
||||
@@ -677,17 +629,17 @@ set(SOURCES
|
||||
utilities/transactions/transaction_util.cc
|
||||
utilities/transactions/write_prepared_txn.cc
|
||||
utilities/transactions/write_prepared_txn_db.cc
|
||||
utilities/transactions/write_unprepared_txn.cc
|
||||
utilities/transactions/write_unprepared_txn_db.cc
|
||||
utilities/ttl/db_ttl_impl.cc
|
||||
utilities/write_batch_with_index/write_batch_with_index.cc
|
||||
utilities/write_batch_with_index/write_batch_with_index_internal.cc
|
||||
$<TARGET_OBJECTS:build_version>)
|
||||
|
||||
if(HAVE_SSE42 AND NOT MSVC)
|
||||
set_source_files_properties(
|
||||
util/crc32c.cc
|
||||
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
|
||||
if(HAVE_SSE42 AND NOT FORCE_SSE42)
|
||||
if(NOT MSVC)
|
||||
set_source_files_properties(
|
||||
util/crc32c.cc
|
||||
PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(HAVE_POWER8)
|
||||
@@ -703,12 +655,8 @@ if(WIN32)
|
||||
port/win/env_default.cc
|
||||
port/win/port_win.cc
|
||||
port/win/win_logger.cc
|
||||
port/win/win_thread.cc)
|
||||
|
||||
if(WITH_XPRESS)
|
||||
list(APPEND SOURCES
|
||||
port/win/win_thread.cc
|
||||
port/win/xpress_win.cc)
|
||||
endif()
|
||||
|
||||
if(WITH_JEMALLOC)
|
||||
list(APPEND SOURCES
|
||||
@@ -725,14 +673,6 @@ endif()
|
||||
set(ROCKSDB_STATIC_LIB rocksdb${ARTIFACT_SUFFIX})
|
||||
set(ROCKSDB_SHARED_LIB rocksdb-shared${ARTIFACT_SUFFIX})
|
||||
set(ROCKSDB_IMPORT_LIB ${ROCKSDB_SHARED_LIB})
|
||||
|
||||
option(WITH_LIBRADOS "Build with librados" OFF)
|
||||
if(WITH_LIBRADOS)
|
||||
list(APPEND SOURCES
|
||||
utilities/env_librados.cc)
|
||||
list(APPEND THIRDPARTY_LIBS rados)
|
||||
endif()
|
||||
|
||||
if(WIN32)
|
||||
set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
|
||||
set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
||||
@@ -751,6 +691,13 @@ else()
|
||||
OUTPUT_NAME "rocksdb")
|
||||
endif()
|
||||
|
||||
option(WITH_LIBRADOS "Build with librados" OFF)
|
||||
if(WITH_LIBRADOS)
|
||||
list(APPEND SOURCES
|
||||
utilities/env_librados.cc)
|
||||
list(APPEND THIRDPARTY_LIBS rados)
|
||||
endif()
|
||||
|
||||
add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
|
||||
target_link_libraries(${ROCKSDB_STATIC_LIB}
|
||||
${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
|
||||
@@ -819,7 +766,6 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
|
||||
TARGETS ${ROCKSDB_SHARED_LIB}
|
||||
EXPORT RocksDBTargets
|
||||
COMPONENT runtime
|
||||
ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
|
||||
LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}"
|
||||
INCLUDES DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}"
|
||||
@@ -866,7 +812,6 @@ if(WITH_TESTS)
|
||||
db/db_inplace_update_test.cc
|
||||
db/db_io_failure_test.cc
|
||||
db/db_iter_test.cc
|
||||
db/db_iter_stress_test.cc
|
||||
db/db_iterator_test.cc
|
||||
db/db_log_iter_test.cc
|
||||
db/db_memtable_test.cc
|
||||
@@ -874,7 +819,6 @@ if(WITH_TESTS)
|
||||
db/db_options_test.cc
|
||||
db/db_properties_test.cc
|
||||
db/db_range_del_test.cc
|
||||
db/db_secondary_test.cc
|
||||
db/db_sst_test.cc
|
||||
db/db_statistics_test.cc
|
||||
db/db_table_properties_test.cc
|
||||
@@ -886,8 +830,6 @@ if(WITH_TESTS)
|
||||
db/db_write_test.cc
|
||||
db/dbformat_test.cc
|
||||
db/deletefile_test.cc
|
||||
db/error_handler_test.cc
|
||||
db/obsolete_files_test.cc
|
||||
db/external_sst_file_basic_test.cc
|
||||
db/external_sst_file_test.cc
|
||||
db/fault_injection_test.cc
|
||||
@@ -904,8 +846,6 @@ if(WITH_TESTS)
|
||||
db/perf_context_test.cc
|
||||
db/plain_table_db_test.cc
|
||||
db/prefix_test.cc
|
||||
db/range_del_aggregator_test.cc
|
||||
db/range_tombstone_fragmenter_test.cc
|
||||
db/repair_test.cc
|
||||
db/table_properties_collector_test.cc
|
||||
db/version_builder_test.cc
|
||||
@@ -931,15 +871,12 @@ if(WITH_TESTS)
|
||||
table/cleanable_test.cc
|
||||
table/cuckoo_table_builder_test.cc
|
||||
table/cuckoo_table_reader_test.cc
|
||||
table/data_block_hash_index_test.cc
|
||||
table/full_filter_block_test.cc
|
||||
table/merger_test.cc
|
||||
table/sst_file_reader_test.cc
|
||||
table/table_test.cc
|
||||
tools/ldb_cmd_test.cc
|
||||
tools/reduce_levels_test.cc
|
||||
tools/sst_dump_test.cc
|
||||
tools/trace_analyzer_test.cc
|
||||
util/arena_test.cc
|
||||
util/auto_roll_logger_test.cc
|
||||
util/autovector_test.cc
|
||||
@@ -954,7 +891,6 @@ if(WITH_TESTS)
|
||||
util/hash_test.cc
|
||||
util/heap_test.cc
|
||||
util/rate_limiter_test.cc
|
||||
util/repeatable_thread_test.cc
|
||||
util/slice_transform_test.cc
|
||||
util/timer_queue_test.cc
|
||||
util/thread_list_test.cc
|
||||
@@ -966,6 +902,12 @@ if(WITH_TESTS)
|
||||
utilities/cassandra/cassandra_row_merge_test.cc
|
||||
utilities/cassandra/cassandra_serialize_test.cc
|
||||
utilities/checkpoint/checkpoint_test.cc
|
||||
utilities/column_aware_encoding_test.cc
|
||||
utilities/date_tiered/date_tiered_test.cc
|
||||
utilities/document/document_db_test.cc
|
||||
utilities/document/json_document_test.cc
|
||||
utilities/geodb/geodb_test.cc
|
||||
utilities/lua/rocks_lua_test.cc
|
||||
utilities/memory/memory_test.cc
|
||||
utilities/merge_operators/string_append/stringappend_test.cc
|
||||
utilities/object_registry_test.cc
|
||||
@@ -973,12 +915,13 @@ if(WITH_TESTS)
|
||||
utilities/options/options_util_test.cc
|
||||
utilities/persistent_cache/hash_table_test.cc
|
||||
utilities/persistent_cache/persistent_cache_test.cc
|
||||
utilities/redis/redis_lists_test.cc
|
||||
utilities/spatialdb/spatial_db_test.cc
|
||||
utilities/simulator_cache/sim_cache_test.cc
|
||||
utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
|
||||
utilities/transactions/optimistic_transaction_test.cc
|
||||
utilities/transactions/transaction_test.cc
|
||||
utilities/transactions/write_prepared_transaction_test.cc
|
||||
utilities/transactions/write_unprepared_transaction_test.cc
|
||||
utilities/ttl/ttl_test.cc
|
||||
utilities/write_batch_with_index/write_batch_with_index_test.cc
|
||||
)
|
||||
@@ -989,9 +932,9 @@ if(WITH_TESTS)
|
||||
set(BENCHMARKS
|
||||
cache/cache_bench.cc
|
||||
memtable/memtablerep_bench.cc
|
||||
db/range_del_aggregator_bench.cc
|
||||
tools/db_bench.cc
|
||||
table/table_reader_bench.cc
|
||||
utilities/column_aware_encoding_exp.cc
|
||||
utilities/persistent_cache/hash_table_bench.cc)
|
||||
add_library(testharness OBJECT util/testharness.cc)
|
||||
foreach(sourcefile ${BENCHMARKS})
|
||||
|
||||
+2
-249
@@ -1,255 +1,9 @@
|
||||
# Rocksdb Change Log
|
||||
|
||||
## 6.1.1 (4/9/2019)
|
||||
### New Features
|
||||
* When reading from option file/string/map, customized comparators and/or merge operators can be filled according to object registry.
|
||||
### Public API Change
|
||||
### Bug Fixes
|
||||
* Fix a bug in 2PC where a sequence of txn prepare, memtable flush, and crash could result in losing the prepared transaction.
|
||||
* Fix a bug in Encryption Env which could cause encrypted files to be read beyond file boundaries.
|
||||
|
||||
## 6.1.0 (3/27/2019)
|
||||
### New Features
|
||||
* Introduce two more stats levels, kExceptHistogramOrTimers and kExceptTimers.
|
||||
* Added a feature to perform data-block sampling for compressibility, and report stats to user.
|
||||
* Add support for trace filtering.
|
||||
* Add DBOptions.avoid_unnecessary_blocking_io. If true, we avoid file deletion when destorying ColumnFamilyHandle and Iterator. Instead, a job is scheduled to delete the files in background.
|
||||
|
||||
### Public API Change
|
||||
* Remove bundled fbson library.
|
||||
* statistics.stats_level_ becomes atomic. It is preferred to use statistics.set_stats_level() and statistics.get_stats_level() to access it.
|
||||
* Introduce a new IOError subcode, PathNotFound, to indicate trying to open a nonexistent file or directory for read.
|
||||
* Add initial support for multiple db instances sharing the same data in single-writer, multi-reader mode.
|
||||
* Removed some "using std::xxx" from public headers.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix JEMALLOC_CXX_THROW macro missing from older Jemalloc versions, causing build failures on some platforms.
|
||||
* Fix SstFileReader not able to open file ingested with write_glbal_seqno=true.
|
||||
|
||||
## 6.0.0 (2/19/2019)
|
||||
### New Features
|
||||
* Enabled checkpoint on readonly db (DBImplReadOnly).
|
||||
* Make DB ignore dropped column families while committing results of atomic flush.
|
||||
* RocksDB may choose to preopen some files even if options.max_open_files != -1. This may make DB open slightly longer.
|
||||
* For users of dictionary compression with ZSTD v0.7.0+, we now reuse the same digested dictionary when compressing each of an SST file's data blocks for faster compression speeds.
|
||||
* For all users of dictionary compression who set `cache_index_and_filter_blocks == true`, we now store dictionary data used for decompression in the block cache for better control over memory usage. For users of ZSTD v1.1.4+ who compile with -DZSTD_STATIC_LINKING_ONLY, this includes a digested dictionary, which is used to increase decompression speed.
|
||||
* Add support for block checksums verification for external SST files before ingestion.
|
||||
* Introduce stats history which periodically saves Statistics snapshots and added `GetStatsHistory` API to retrieve these snapshots.
|
||||
* Add a place holder in manifest which indicate a record from future that can be safely ignored.
|
||||
* Add support for trace sampling.
|
||||
* Enable properties block checksum verification for block-based tables.
|
||||
* For all users of dictionary compression, we now generate a separate dictionary for compressing each bottom-level SST file. Previously we reused a single dictionary for a whole compaction to bottom level. The new approach achieves better compression ratios; however, it uses more memory and CPU for buffering/sampling data blocks and training dictionaries.
|
||||
* Add whole key bloom filter support in memtable.
|
||||
* Files written by `SstFileWriter` will now use dictionary compression if it is configured in the file writer's `CompressionOptions`.
|
||||
|
||||
### Public API Change
|
||||
* Disallow CompactionFilter::IgnoreSnapshots() = false, because it is not very useful and the behavior is confusing. The filter will filter everything if there is no snapshot declared by the time the compaction starts. However, users can define a snapshot after the compaction starts and before it finishes and this new snapshot won't be repeatable, because after the compaction finishes, some keys may be dropped.
|
||||
* CompactionPri = kMinOverlappingRatio also uses compensated file size, which boosts file with lots of tombstones to be compacted first.
|
||||
* Transaction::GetForUpdate is extended with a do_validate parameter with default value of true. If false it skips validating the snapshot before doing the read. Similarly ::Merge, ::Put, ::Delete, and ::SingleDelete are extended with assume_tracked with default value of false. If true it indicates that call is assumed to be after a ::GetForUpdate.
|
||||
* `TableProperties::num_entries` and `TableProperties::num_deletions` now also account for number of range tombstones.
|
||||
* Remove geodb, spatial_db, document_db, json_document, date_tiered_db, and redis_lists.
|
||||
* With "ldb ----try_load_options", when wal_dir specified by the option file doesn't exist, ignore it.
|
||||
* Change time resolution in FileOperationInfo.
|
||||
* Deleting Blob files also go through SStFileManager.
|
||||
* Remove CuckooHash memtable.
|
||||
* The counter stat `number.block.not_compressed` now also counts blocks not compressed due to poor compression ratio.
|
||||
* Remove ttl option from `CompactionOptionsFIFO`. The option has been deprecated and ttl in `ColumnFamilyOptions` is used instead.
|
||||
* Support SST file ingestion across multiple column families via DB::IngestExternalFiles. See the function's comment about atomicity.
|
||||
* Remove Lua compaction filter.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix a deadlock caused by compaction and file ingestion waiting for each other in the event of write stalls.
|
||||
* Fix a memory leak when files with range tombstones are read in mmap mode and block cache is enabled
|
||||
* Fix handling of corrupt range tombstone blocks such that corruptions cannot cause deleted keys to reappear
|
||||
* Lock free MultiGet
|
||||
* Fix incorrect `NotFound` point lookup result when querying the endpoint of a file that has been extended by a range tombstone.
|
||||
* Fix with pipelined write, write leaders's callback failure lead to the whole write group fail.
|
||||
|
||||
### Change Default Options
|
||||
* Change options.compaction_pri's default to kMinOverlappingRatio
|
||||
|
||||
## 5.18.0 (11/30/2018)
|
||||
### New Features
|
||||
* Introduced `JemallocNodumpAllocator` memory allocator. When being use, block cache will be excluded from core dump.
|
||||
* Introduced `PerfContextByLevel` as part of `PerfContext` which allows storing perf context at each level. Also replaced `__thread` with `thread_local` keyword for perf_context. Added per-level perf context for bloom filter and `Get` query.
|
||||
* With level_compaction_dynamic_level_bytes = true, level multiplier may be adjusted automatically when Level 0 to 1 compaction is lagged behind.
|
||||
* Introduced DB option `atomic_flush`. If true, RocksDB supports flushing multiple column families and atomically committing the result to MANIFEST. Useful when WAL is disabled.
|
||||
* Added `num_deletions` and `num_merge_operands` members to `TableProperties`.
|
||||
* Added "rocksdb.min-obsolete-sst-number-to-keep" DB property that reports the lower bound on SST file numbers that are being kept from deletion, even if the SSTs are obsolete.
|
||||
* Add xxhash64 checksum support
|
||||
* Introduced `MemoryAllocator`, which lets the user specify custom memory allocator for block based table.
|
||||
* Improved `DeleteRange` to prevent read performance degradation. The feature is no longer marked as experimental.
|
||||
|
||||
### Public API Change
|
||||
* `DBOptions::use_direct_reads` now affects reads issued by `BackupEngine` on the database's SSTs.
|
||||
* `NO_ITERATORS` is divided into two counters `NO_ITERATOR_CREATED` and `NO_ITERATOR_DELETE`. Both of them are only increasing now, just as other counters.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix corner case where a write group leader blocked due to write stall blocks other writers in queue with WriteOptions::no_slowdown set.
|
||||
* Fix in-memory range tombstone truncation to avoid erroneously covering newer keys at a lower level, and include range tombstones in compacted files whose largest key is the range tombstone's start key.
|
||||
* Properly set the stop key for a truncated manual CompactRange
|
||||
* Fix slow flush/compaction when DB contains many snapshots. The problem became noticeable to us in DBs with 100,000+ snapshots, though it will affect others at different thresholds.
|
||||
* Fix the bug that WriteBatchWithIndex's SeekForPrev() doesn't see the entries with the same key.
|
||||
* Fix the bug where user comparator was sometimes fed with InternalKey instead of the user key. The bug manifests when during GenerateBottommostFiles.
|
||||
* Fix a bug in WritePrepared txns where if the number of old snapshots goes beyond the snapshot cache size (128 default) the rest will not be checked when evicting a commit entry from the commit cache.
|
||||
* Fixed Get correctness bug in the presence of range tombstones where merge operands covered by a range tombstone always result in NotFound.
|
||||
* Start populating `NO_FILE_CLOSES` ticker statistic, which was always zero previously.
|
||||
* The default value of NewBloomFilterPolicy()'s argument use_block_based_builder is changed to false. Note that this new default may cause large temp memory usage when building very large SST files.
|
||||
|
||||
## 5.17.0 (10/05/2018)
|
||||
### Public API Change
|
||||
* `OnTableFileCreated` will now be called for empty files generated during compaction. In that case, `TableFileCreationInfo::file_path` will be "(nil)" and `TableFileCreationInfo::file_size` will be zero.
|
||||
* Add `FlushOptions::allow_write_stall`, which controls whether Flush calls start working immediately, even if it causes user writes to stall, or will wait until flush can be performed without causing write stall (similar to `CompactRangeOptions::allow_write_stall`). Note that the default value is false, meaning we add delay to Flush calls until stalling can be avoided when possible. This is behavior change compared to previous RocksDB versions, where Flush calls didn't check if they might cause stall or not.
|
||||
* Application using PessimisticTransactionDB is expected to rollback/commit recovered transactions before starting new ones. This assumption is used to skip concurrency control during recovery.
|
||||
* Expose column family id to `OnCompactionCompleted`.
|
||||
|
||||
### New Features
|
||||
* TransactionOptions::skip_concurrency_control allows pessimistic transactions to skip the overhead of concurrency control. Could be used for optimizing certain transactions or during recovery.
|
||||
|
||||
### Bug Fixes
|
||||
* Avoid creating empty SSTs and subsequently deleting them in certain cases during compaction.
|
||||
* Sync CURRENT file contents during checkpoint.
|
||||
|
||||
## 5.16.3 (10/1/2018)
|
||||
### Bug Fixes
|
||||
* Fix crash caused when `CompactFiles` run with `CompactionOptions::compression == CompressionType::kDisableCompressionOption`. Now that setting causes the compression type to be chosen according to the column family-wide compression options.
|
||||
|
||||
## 5.16.2 (9/21/2018)
|
||||
### Bug Fixes
|
||||
* Fix bug in partition filters with format_version=4.
|
||||
|
||||
## 5.16.1 (9/17/2018)
|
||||
### Bug Fixes
|
||||
* Remove trace_analyzer_tool from rocksdb_lib target in TARGETS file.
|
||||
* Fix RocksDB Java build and tests.
|
||||
* Remove sync point in Block destructor.
|
||||
|
||||
## 5.16.0 (8/21/2018)
|
||||
### Public API Change
|
||||
* The merge operands are passed to `MergeOperator::ShouldMerge` in the reversed order relative to how they were merged (passed to FullMerge or FullMergeV2) for performance reasons
|
||||
* GetAllKeyVersions() to take an extra argument of `max_num_ikeys`.
|
||||
* Using ZSTD dictionary trainer (i.e., setting `CompressionOptions::zstd_max_train_bytes` to a nonzero value) now requires ZSTD version 1.1.3 or later.
|
||||
|
||||
### New Features
|
||||
* Changes the format of index blocks by delta encoding the index values, which are the block handles. This saves the encoding of BlockHandle::offset of the non-head index entries in each restart interval. The feature is backward compatible but not forward compatible. It is disabled by default unless format_version 4 or above is used.
|
||||
* Add a new tool: trace_analyzer. Trace_analyzer analyzes the trace file generated by using trace_replay API. It can convert the binary format trace file to a human readable txt file, output the statistics of the analyzed query types such as access statistics and size statistics, combining the dumped whole key space file to analyze, support query correlation analyzing, and etc. Current supported query types are: Get, Put, Delete, SingleDelete, DeleteRange, Merge, Iterator (Seek, SeekForPrev only).
|
||||
* Add hash index support to data blocks, which helps reducing the cpu utilization of point-lookup operations. This feature is backward compatible with the data block created without the hash index. It is disabled by default unless BlockBasedTableOptions::data_block_index_type is set to data_block_index_type = kDataBlockBinaryAndHash.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix a bug in misreporting the estimated partition index size in properties block.
|
||||
|
||||
## 5.15.0 (7/17/2018)
|
||||
### Public API Change
|
||||
* Remove managed iterator. ReadOptions.managed is not effective anymore.
|
||||
* For bottommost_compression, a compatible CompressionOptions is added via `bottommost_compression_opts`. To keep backward compatible, a new boolean `enabled` is added to CompressionOptions. For compression_opts, it will be always used no matter what value of `enabled` is. For bottommost_compression_opts, it will only be used when user set `enabled=true`, otherwise, compression_opts will be used for bottommost_compression as default.
|
||||
* With LRUCache, when high_pri_pool_ratio > 0, midpoint insertion strategy will be enabled to put low-pri items to the tail of low-pri list (the midpoint) when they first inserted into the cache. This is to make cache entries never get hit age out faster, improving cache efficiency when large background scan presents.
|
||||
* For users of `Statistics` objects created via `CreateDBStatistics()`, the format of the string returned by its `ToString()` method has changed.
|
||||
* The "rocksdb.num.entries" table property no longer counts range deletion tombstones as entries.
|
||||
|
||||
### New Features
|
||||
* Changes the format of index blocks by storing the key in their raw form rather than converting them to InternalKey. This saves 8 bytes per index key. The feature is backward compatible but not forward compatible. It is disabled by default unless format_version 3 or above is used.
|
||||
* Avoid memcpy when reading mmap files with OpenReadOnly and max_open_files==-1.
|
||||
* Support dynamically changing `ColumnFamilyOptions::ttl` via `SetOptions()`.
|
||||
* Add a new table property, "rocksdb.num.range-deletions", which counts the number of range deletion tombstones in the table.
|
||||
* Improve the performance of iterators doing long range scans by using readahead, when using direct IO.
|
||||
* pin_top_level_index_and_filter (default true) in BlockBasedTableOptions can be used in combination with cache_index_and_filter_blocks to prefetch and pin the top-level index of partitioned index and filter blocks in cache. It has no impact when cache_index_and_filter_blocks is false.
|
||||
* Write properties meta-block at the end of block-based table to save read-ahead IO.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix deadlock with enable_pipelined_write=true and max_successive_merges > 0
|
||||
* Check conflict at output level in CompactFiles.
|
||||
* Fix corruption in non-iterator reads when mmap is used for file reads
|
||||
* Fix bug with prefix search in partition filters where a shared prefix would be ignored from the later partitions. The bug could report an eixstent key as missing. The bug could be triggered if prefix_extractor is set and partition filters is enabled.
|
||||
* Change default value of `bytes_max_delete_chunk` to 0 in NewSstFileManager() as it doesn't work well with checkpoints.
|
||||
* Fix a bug caused by not copying the block trailer with compressed SST file, direct IO, prefetcher and no compressed block cache.
|
||||
* Fix write can stuck indefinitely if enable_pipelined_write=true. The issue exists since pipelined write was introduced in 5.5.0.
|
||||
|
||||
## 5.14.0 (5/16/2018)
|
||||
### Public API Change
|
||||
* Add a BlockBasedTableOption to align uncompressed data blocks on the smaller of block size or page size boundary, to reduce flash reads by avoiding reads spanning 4K pages.
|
||||
* The background thread naming convention changed (on supporting platforms) to "rocksdb:<thread pool priority><thread number>", e.g., "rocksdb:low0".
|
||||
* Add a new ticker stat rocksdb.number.multiget.keys.found to count number of keys successfully read in MultiGet calls
|
||||
* Touch-up to write-related counters in PerfContext. New counters added: write_scheduling_flushes_compactions_time, write_thread_wait_nanos. Counters whose behavior was fixed or modified: write_memtable_time, write_pre_and_post_process_time, write_delay_time.
|
||||
* Posix Env's NewRandomRWFile() will fail if the file doesn't exist.
|
||||
* Now, `DBOptions::use_direct_io_for_flush_and_compaction` only applies to background writes, and `DBOptions::use_direct_reads` applies to both user reads and background reads. This conforms with Linux's `open(2)` manpage, which advises against simultaneously reading a file in buffered and direct modes, due to possibly undefined behavior and degraded performance.
|
||||
* Iterator::Valid() always returns false if !status().ok(). So, now when doing a Seek() followed by some Next()s, there's no need to check status() after every operation.
|
||||
* Iterator::Seek()/SeekForPrev()/SeekToFirst()/SeekToLast() always resets status().
|
||||
* Introduced `CompressionOptions::kDefaultCompressionLevel`, which is a generic way to tell RocksDB to use the compression library's default level. It is now the default value for `CompressionOptions::level`. Previously the level defaulted to -1, which gave poor compression ratios in ZSTD.
|
||||
|
||||
### New Features
|
||||
* Introduce TTL for level compaction so that all files older than ttl go through the compaction process to get rid of old data.
|
||||
* TransactionDBOptions::write_policy can be configured to enable WritePrepared 2PC transactions. Read more about them in the wiki.
|
||||
* Add DB properties "rocksdb.block-cache-capacity", "rocksdb.block-cache-usage", "rocksdb.block-cache-pinned-usage" to show block cache usage.
|
||||
* Add `Env::LowerThreadPoolCPUPriority(Priority)` method, which lowers the CPU priority of background (esp. compaction) threads to minimize interference with foreground tasks.
|
||||
* Fsync parent directory after deleting a file in delete scheduler.
|
||||
* In level-based compaction, if bottom-pri thread pool was setup via `Env::SetBackgroundThreads()`, compactions to the bottom level will be delegated to that thread pool.
|
||||
* `prefix_extractor` has been moved from ImmutableCFOptions to MutableCFOptions, meaning it can be dynamically changed without a DB restart.
|
||||
|
||||
### Bug Fixes
|
||||
* Fsync after writing global seq number to the ingestion file in ExternalSstFileIngestionJob.
|
||||
* Fix WAL corruption caused by race condition between user write thread and FlushWAL when two_write_queue is not set.
|
||||
* Fix `BackupableDBOptions::max_valid_backups_to_open` to not delete backup files when refcount cannot be accurately determined.
|
||||
* Fix memory leak when pin_l0_filter_and_index_blocks_in_cache is used with partitioned filters
|
||||
* Disable rollback of merge operands in WritePrepared transactions to work around an issue in MyRocks. It can be enabled back by setting TransactionDBOptions::rollback_merge_operands to true.
|
||||
* Fix wrong results by ReverseBytewiseComparator::FindShortSuccessor()
|
||||
|
||||
### Java API Changes
|
||||
* Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
|
||||
* Added SstFileManager to the Java API to allow managing SST files across DB instances.
|
||||
|
||||
## 5.13.0 (3/20/2018)
|
||||
### Public API Change
|
||||
* RocksDBOptionsParser::Parse()'s `ignore_unknown_options` argument will only be effective if the option file shows it is generated using a higher version of RocksDB than the current version.
|
||||
* Remove CompactionEventListener.
|
||||
|
||||
### New Features
|
||||
* SstFileManager now can cancel compactions if they will result in max space errors. SstFileManager users can also use SetCompactionBufferSize to specify how much space must be leftover during a compaction for auxiliary file functions such as logging and flushing.
|
||||
* Avoid unnecessarily flushing in `CompactRange()` when the range specified by the user does not overlap unflushed memtables.
|
||||
* If `ColumnFamilyOptions::max_subcompactions` is set greater than one, we now parallelize large manual level-based compactions.
|
||||
* Add "rocksdb.live-sst-files-size" DB property to return total bytes of all SST files belong to the latest LSM tree.
|
||||
* NewSstFileManager to add an argument bytes_max_delete_chunk with default 64MB. With this argument, a file larger than 64MB will be ftruncated multiple times based on this size.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix a leak in prepared_section_completed_ where the zeroed entries would not removed from the map.
|
||||
* Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread.
|
||||
|
||||
## 5.12.0 (2/14/2018)
|
||||
## Unreleased
|
||||
### Public API Change
|
||||
* Iterator::SeekForPrev is now a pure virtual method. This is to prevent user who implement the Iterator interface fail to implement SeekForPrev by mistake.
|
||||
* Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`.
|
||||
* Add `CompactRangeOptions::allow_write_stall`, which makes `CompactRange` start working immediately, even if it causes user writes to stall. The default value is false, meaning we add delay to `CompactRange` calls until stalling can be avoided when possible. Note this delay is not present in previous RocksDB versions.
|
||||
* Creating checkpoint with empty directory now returns `Status::InvalidArgument`; previously, it returned `Status::IOError`.
|
||||
* Adds a BlockBasedTableOption to turn off index block compression.
|
||||
* Close() method now returns a status when closing a db.
|
||||
|
||||
### New Features
|
||||
* Improve the performance of iterators doing long range scans by using readahead.
|
||||
* Add new function `DeleteFilesInRanges()` to delete files in multiple ranges at once for better performance.
|
||||
* FreeBSD build support for RocksDB and RocksJava.
|
||||
* Improved performance of long range scans with readahead.
|
||||
* Updated to and now continuously tested in Visual Studio 2017.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix `DisableFileDeletions()` followed by `GetSortedWalFiles()` to not return obsolete WAL files that `PurgeObsoleteFiles()` is going to delete.
|
||||
* Fix Handle error return from WriteBuffer() during WAL file close and DB close.
|
||||
* Fix advance reservation of arena block addresses.
|
||||
* Fix handling of empty string as checkpoint directory.
|
||||
|
||||
## 5.11.0 (01/08/2018)
|
||||
### Public API Change
|
||||
* Add `autoTune` and `getBytesPerSecond()` to RocksJava RateLimiter
|
||||
|
||||
### New Features
|
||||
* Add a new histogram stat called rocksdb.db.flush.micros for memtable flush.
|
||||
* Add "--use_txn" option to use transactional API in db_stress.
|
||||
* Disable onboard cache for compaction output in Windows platform.
|
||||
* Improve the performance of iterators doing long range scans by using readahead.
|
||||
|
||||
### Bug Fixes
|
||||
* Fix a stack-use-after-scope bug in ForwardIterator.
|
||||
* Fix builds on platforms including Linux, Windows, and PowerPC.
|
||||
* Fix buffer overrun in backup engine for DBs with huge number of files.
|
||||
* Fix a mislabel bug for bottom-pri compaction threads.
|
||||
* Fix DB::Flush() keep waiting after flush finish under certain condition.
|
||||
|
||||
## 5.10.0 (12/11/2017)
|
||||
@@ -272,8 +26,7 @@
|
||||
* `BackupableDBOptions::max_valid_backups_to_open == 0` now means no backups will be opened during BackupEngine initialization. Previously this condition disabled limiting backups opened.
|
||||
* `DBOptions::preserve_deletes` is a new option that allows one to specify that DB should not drop tombstones for regular deletes if they have sequence number larger than what was set by the new API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)`. Disabled by default.
|
||||
* API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)` was added, users who wish to preserve deletes are expected to periodically call this function to advance the cutoff seqnum (all deletes made before this seqnum can be dropped by DB). It's user responsibility to figure out how to advance the seqnum in the way so the tombstones are kept for the desired period of time, yet are eventually processed in time and don't eat up too much space.
|
||||
* `ReadOptions::iter_start_seqnum` was added;
|
||||
if set to something > 0 user will see 2 changes in iterators behavior 1) only keys written with sequence larger than this parameter would be returned and 2) the `Slice` returned by iter->key() now points to the memory that keep User-oriented representation of the internal key, rather than user key. New struct `FullKey` was added to represent internal keys, along with a new helper function `ParseFullKey(const Slice& internal_key, FullKey* result);`.
|
||||
* `ReadOptions::iter_start_seqnum` was added; if set to something > 0 user will see 2 changes in iterators behavior 1) only keys written with sequence larger than this parameter would be returned and 2) the `Slice` returned by iter->key() now points to the the memory that keep User-oriented representation of the internal key, rather than user key. New struct `FullKey` was added to represent internal keys, along with a new helper function `ParseFullKey(const Slice& internal_key, FullKey* result);`.
|
||||
* Deprecate trash_dir param in NewSstFileManager, right now we will rename deleted files to <name>.trash instead of moving them to trash directory
|
||||
* Allow setting a custom trash/DB size ratio limit in the SstFileManager, after which files that are to be scheduled for deletion are deleted immediately, regardless of any delete ratelimit.
|
||||
* Return an error on write if write_options.sync = true and write_options.disableWAL = true to warn user of inconsistent options. Previously we will not write to WAL and not respecting the sync options in this case.
|
||||
|
||||
-23
@@ -43,8 +43,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
|
||||
command line flags processing. You can compile rocksdb library even
|
||||
if you don't have gflags installed.
|
||||
|
||||
* If you wish to build the RocksJava static target, then cmake is required for building Snappy.
|
||||
|
||||
## Supported platforms
|
||||
|
||||
* **Linux - Ubuntu**
|
||||
@@ -143,27 +141,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
|
||||
export JAVA_HOME=/usr/local/openjdk7
|
||||
gmake rocksdbjava
|
||||
|
||||
* **OpenBSD** (6.3/-current):
|
||||
|
||||
* As RocksDB is not available in the ports yet you have to build it on your own:
|
||||
|
||||
* Install the dependencies for RocksDB:
|
||||
|
||||
pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch
|
||||
|
||||
* Build RocksDB from source:
|
||||
|
||||
cd ~
|
||||
git clone https://github.com/facebook/rocksdb.git
|
||||
cd rocksdb
|
||||
gmake static_lib
|
||||
|
||||
* Build RocksJava from source (optional):
|
||||
|
||||
cd rocksdb
|
||||
export JAVA_HOME=/usr/local/jdk-1.8.0
|
||||
export PATH=$PATH:/usr/local/jdk-1.8.0/bin
|
||||
gmake rocksdbjava
|
||||
|
||||
* **iOS**:
|
||||
* Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
|
||||
|
||||
@@ -1,9 +1,7 @@
|
||||
This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
|
||||
|
||||
* Java - https://github.com/facebook/rocksdb/tree/master/java
|
||||
* Python
|
||||
* http://python-rocksdb.readthedocs.io/en/latest/
|
||||
* http://pyrocksdb.readthedocs.org/en/latest/ (unmaintained)
|
||||
* Python - http://pyrocksdb.readthedocs.org/en/latest/
|
||||
* Perl - https://metacpan.org/pod/RocksDB
|
||||
* Node.js - https://npmjs.org/package/rocksdb
|
||||
* Go - https://github.com/tecbot/gorocksdb
|
||||
@@ -12,11 +10,8 @@ This is the list of all known third-party language bindings for RocksDB. If some
|
||||
* PHP - https://github.com/Photonios/rocksdb-php
|
||||
* C# - https://github.com/warrenfalk/rocksdb-sharp
|
||||
* Rust
|
||||
* https://github.com/pingcap/rust-rocksdb (used in production fork of https://github.com/spacejam/rust-rocksdb)
|
||||
* https://github.com/spacejam/rust-rocksdb
|
||||
* https://github.com/bh1xuw/rust-rocks
|
||||
* D programming language - https://github.com/b1naryth1ef/rocksdb
|
||||
* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
|
||||
* Elixir - https://github.com/urbint/rox
|
||||
* Nim - https://github.com/status-im/nim-rocksdb
|
||||
* Swift and Objective-C (iOS/OSX) - https://github.com/iabudiab/ObjectiveRocks
|
||||
|
||||
@@ -76,49 +76,20 @@ ifeq ($(MAKECMDGOALS),install)
|
||||
endif
|
||||
|
||||
ifeq ($(MAKECMDGOALS),rocksdbjavastatic)
|
||||
ifneq ($(DEBUG_LEVEL),2)
|
||||
DEBUG_LEVEL=0
|
||||
endif
|
||||
DEBUG_LEVEL=0
|
||||
endif
|
||||
|
||||
ifeq ($(MAKECMDGOALS),rocksdbjavastaticrelease)
|
||||
DEBUG_LEVEL=0
|
||||
endif
|
||||
|
||||
ifeq ($(MAKECMDGOALS),rocksdbjavastaticreleasedocker)
|
||||
DEBUG_LEVEL=0
|
||||
endif
|
||||
|
||||
ifeq ($(MAKECMDGOALS),rocksdbjavastaticpublish)
|
||||
DEBUG_LEVEL=0
|
||||
endif
|
||||
|
||||
# Lite build flag.
|
||||
LITE ?= 0
|
||||
ifeq ($(LITE), 0)
|
||||
ifneq ($(filter -DROCKSDB_LITE,$(OPT)),)
|
||||
# Be backward compatible and support older format where OPT=-DROCKSDB_LITE is
|
||||
# specified instead of LITE=1 on the command line.
|
||||
LITE=1
|
||||
endif
|
||||
else ifeq ($(LITE), 1)
|
||||
ifeq ($(filter -DROCKSDB_LITE,$(OPT)),)
|
||||
OPT += -DROCKSDB_LITE
|
||||
endif
|
||||
endif
|
||||
|
||||
# Figure out optimize level.
|
||||
ifneq ($(DEBUG_LEVEL), 2)
|
||||
ifeq ($(LITE), 0)
|
||||
OPT += -O2
|
||||
else
|
||||
OPT += -Os
|
||||
endif
|
||||
endif
|
||||
|
||||
# compile with -O2 if debug level is not 2
|
||||
ifneq ($(DEBUG_LEVEL), 2)
|
||||
OPT += -fno-omit-frame-pointer
|
||||
OPT += -O2 -fno-omit-frame-pointer
|
||||
# Skip for archs that don't support -momit-leaf-frame-pointer
|
||||
ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1))
|
||||
OPT += -momit-leaf-frame-pointer
|
||||
@@ -198,7 +169,7 @@ include make_config.mk
|
||||
CLEAN_FILES += make_config.mk
|
||||
|
||||
missing_make_config_paths := $(shell \
|
||||
grep "\./\S*\|/\S*" -o $(CURDIR)/make_config.mk | \
|
||||
grep "\/\S*" -o $(CURDIR)/make_config.mk | \
|
||||
while read path; \
|
||||
do [ -e $$path ] || echo $$path; \
|
||||
done | sort | uniq)
|
||||
@@ -251,9 +222,6 @@ ifdef COMPILE_WITH_TSAN
|
||||
PROFILING_FLAGS =
|
||||
# LUA is not supported under TSAN
|
||||
LUA_PATH =
|
||||
# Limit keys for crash test under TSAN to avoid error:
|
||||
# "ThreadSanitizer: DenseSlabAllocator overflow. Dying."
|
||||
CRASH_TEST_EXT_ARGS += --max_key=1000000
|
||||
endif
|
||||
|
||||
# AIX doesn't work with -pg
|
||||
@@ -264,13 +232,9 @@ endif
|
||||
# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
|
||||
ifdef COMPILE_WITH_UBSAN
|
||||
DISABLE_JEMALLOC=1
|
||||
# Suppress alignment warning because murmurhash relies on casting unaligned
|
||||
# memory to integer. Fixing it may cause performance regression. 3-way crc32
|
||||
# relies on it too, although it can be rewritten to eliminate with minimal
|
||||
# performance regression.
|
||||
EXEC_LDFLAGS += -fsanitize=undefined -fno-sanitize-recover=all
|
||||
PLATFORM_CCFLAGS += -fsanitize=undefined -fno-sanitize-recover=all -DROCKSDB_UBSAN_RUN
|
||||
PLATFORM_CXXFLAGS += -fsanitize=undefined -fno-sanitize-recover=all -DROCKSDB_UBSAN_RUN
|
||||
EXEC_LDFLAGS += -fsanitize=undefined
|
||||
PLATFORM_CCFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
|
||||
PLATFORM_CXXFLAGS += -fsanitize=undefined -DROCKSDB_UBSAN_RUN
|
||||
endif
|
||||
|
||||
ifdef ROCKSDB_VALGRIND_RUN
|
||||
@@ -308,11 +272,7 @@ endif
|
||||
default: all
|
||||
|
||||
WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
|
||||
-Wunused-parameter
|
||||
|
||||
ifeq ($(PLATFORM), OS_OPENBSD)
|
||||
WARNING_FLAGS += -Wno-unused-lambda-capture
|
||||
endif
|
||||
-Wno-unused-parameter
|
||||
|
||||
ifndef DISABLE_WARNING_AS_ERROR
|
||||
WARNING_FLAGS += -Werror
|
||||
@@ -341,7 +301,7 @@ endif
|
||||
ifeq ("$(wildcard $(LUA_LIB))", "") # LUA_LIB does not exist
|
||||
$(error $(LUA_LIB) does not exist. Try to specify both LUA_PATH and LUA_LIB manually)
|
||||
endif
|
||||
EXEC_LDFLAGS += $(LUA_LIB)
|
||||
LDFLAGS += $(LUA_LIB)
|
||||
|
||||
endif
|
||||
|
||||
@@ -401,9 +361,7 @@ VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full
|
||||
|
||||
BENCHTOOLOBJECTS = $(BENCH_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
|
||||
|
||||
ANALYZETOOLOBJECTS = $(ANALYZER_LIB_SOURCES:.cc=.o)
|
||||
|
||||
EXPOBJECTS = $(LIBOBJECTS) $(TESTUTIL)
|
||||
EXPOBJECTS = $(EXP_LIB_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
|
||||
|
||||
TESTS = \
|
||||
db_basic_test \
|
||||
@@ -431,7 +389,6 @@ TESTS = \
|
||||
db_blob_index_test \
|
||||
db_bloom_filter_test \
|
||||
db_iter_test \
|
||||
db_iter_stress_test \
|
||||
db_log_iter_test \
|
||||
db_compaction_filter_test \
|
||||
db_compaction_test \
|
||||
@@ -443,15 +400,14 @@ TESTS = \
|
||||
db_merge_operator_test \
|
||||
db_options_test \
|
||||
db_range_del_test \
|
||||
db_secondary_test \
|
||||
db_sst_test \
|
||||
db_tailing_iter_test \
|
||||
db_universal_compaction_test \
|
||||
db_io_failure_test \
|
||||
db_properties_test \
|
||||
db_table_properties_test \
|
||||
db_statistics_test \
|
||||
db_write_test \
|
||||
error_handler_test \
|
||||
autovector_test \
|
||||
blob_db_test \
|
||||
cleanable_test \
|
||||
@@ -459,7 +415,6 @@ TESTS = \
|
||||
table_properties_collector_test \
|
||||
arena_test \
|
||||
block_test \
|
||||
data_block_hash_index_test \
|
||||
cache_test \
|
||||
corruption_test \
|
||||
slice_transform_test \
|
||||
@@ -483,6 +438,7 @@ TESTS = \
|
||||
merger_test \
|
||||
util_merge_operators_test \
|
||||
options_file_test \
|
||||
redis_test \
|
||||
reduce_levels_test \
|
||||
plain_table_db_test \
|
||||
comparator_db_test \
|
||||
@@ -496,8 +452,12 @@ TESTS = \
|
||||
cassandra_row_merge_test \
|
||||
cassandra_serialize_test \
|
||||
ttl_test \
|
||||
date_tiered_test \
|
||||
backupable_db_test \
|
||||
document_db_test \
|
||||
json_document_test \
|
||||
sim_cache_test \
|
||||
spatial_db_test \
|
||||
version_edit_test \
|
||||
version_set_test \
|
||||
compaction_picker_test \
|
||||
@@ -507,8 +467,8 @@ TESTS = \
|
||||
write_batch_with_index_test \
|
||||
write_controller_test\
|
||||
deletefile_test \
|
||||
obsolete_files_test \
|
||||
table_test \
|
||||
geodb_test \
|
||||
delete_scheduler_test \
|
||||
options_test \
|
||||
options_settable_test \
|
||||
@@ -525,6 +485,7 @@ TESTS = \
|
||||
compaction_job_test \
|
||||
thread_list_test \
|
||||
sst_dump_test \
|
||||
column_aware_encoding_test \
|
||||
compact_files_test \
|
||||
optimistic_transaction_test \
|
||||
write_callback_test \
|
||||
@@ -536,19 +497,13 @@ TESTS = \
|
||||
ldb_cmd_test \
|
||||
persistent_cache_test \
|
||||
statistics_test \
|
||||
lua_test \
|
||||
range_del_aggregator_test \
|
||||
lru_cache_test \
|
||||
object_registry_test \
|
||||
repair_test \
|
||||
env_timed_test \
|
||||
write_prepared_transaction_test \
|
||||
write_unprepared_transaction_test \
|
||||
db_universal_compaction_test \
|
||||
trace_analyzer_test \
|
||||
repeatable_thread_test \
|
||||
range_tombstone_fragmenter_test \
|
||||
range_del_aggregator_test \
|
||||
sst_file_reader_test \
|
||||
db_secondary_test \
|
||||
|
||||
PARALLEL_TEST = \
|
||||
backupable_db_test \
|
||||
@@ -567,12 +522,7 @@ PARALLEL_TEST = \
|
||||
table_test \
|
||||
transaction_test \
|
||||
write_prepared_transaction_test \
|
||||
write_unprepared_transaction_test \
|
||||
|
||||
# options_settable_test doesn't pass with UBSAN as we use hack in the test
|
||||
ifdef COMPILE_WITH_UBSAN
|
||||
TESTS := $(shell echo $(TESTS) | sed 's/\boptions_settable_test\b//g')
|
||||
endif
|
||||
SUBSET := $(TESTS)
|
||||
ifdef ROCKSDBTESTS_START
|
||||
SUBSET := $(shell echo $(SUBSET) | sed 's/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/')
|
||||
@@ -592,13 +542,12 @@ TOOLS = \
|
||||
rocksdb_dump \
|
||||
rocksdb_undump \
|
||||
blob_dump \
|
||||
trace_analyzer \
|
||||
|
||||
TEST_LIBS = \
|
||||
librocksdb_env_basic_test.a
|
||||
|
||||
# TODO: add back forward_iterator_bench, after making it build in all environemnts.
|
||||
BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench persistent_cache_bench range_del_aggregator_bench
|
||||
BENCHMARKS = db_bench table_reader_bench cache_bench memtablerep_bench column_aware_encoding_exp persistent_cache_bench
|
||||
|
||||
# if user didn't config LIBNAME, set the default
|
||||
ifeq ($(LIBNAME),)
|
||||
@@ -681,15 +630,14 @@ ifeq ($(HAVE_POWER8),1)
|
||||
shared_all_libobjects = $(shared_libobjects) $(shared-ppc-objects)
|
||||
endif
|
||||
$(SHARED4): $(shared_all_libobjects)
|
||||
$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(shared_all_libobjects) $(LDFLAGS) -o $@
|
||||
$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED3) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(shared_libobjects) $(LDFLAGS) -o $@
|
||||
|
||||
endif # PLATFORM_SHARED_EXT
|
||||
|
||||
.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests package \
|
||||
release tags tags0 valgrind_check whitebox_crash_test format static_lib shared_lib all \
|
||||
release tags valgrind_check whitebox_crash_test format static_lib shared_lib all \
|
||||
dbg rocksdbjavastatic rocksdbjava install install-static install-shared uninstall \
|
||||
analyze tools tools_lib \
|
||||
blackbox_crash_test_with_atomic_flush whitebox_crash_test_with_atomic_flush
|
||||
analyze tools tools_lib
|
||||
|
||||
|
||||
all: $(LIBRARY) $(BENCHMARKS) tools tools_lib test_libs $(TESTS)
|
||||
@@ -718,7 +666,7 @@ coverage:
|
||||
COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) J=1 all check
|
||||
cd coverage && ./coverage_test.sh
|
||||
# Delete intermediate files
|
||||
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
|
||||
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
|
||||
|
||||
ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
|
||||
# Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
|
||||
@@ -819,6 +767,7 @@ J ?= 100%
|
||||
# Use this regexp to select the subset of tests whose names match.
|
||||
tests-regexp = .
|
||||
|
||||
t_run = $(wildcard t/run-*)
|
||||
.PHONY: check_0
|
||||
check_0:
|
||||
$(AM_V_GEN)export TEST_TMPDIR=$(TMPD); \
|
||||
@@ -828,13 +777,13 @@ check_0:
|
||||
test -t 1 && eta=--eta || eta=; \
|
||||
{ \
|
||||
printf './%s\n' $(filter-out $(PARALLEL_TEST),$(TESTS)); \
|
||||
find t -name 'run-*' -print; \
|
||||
printf '%s\n' $(t_run); \
|
||||
} \
|
||||
| $(prioritize_long_running_tests) \
|
||||
| grep -E '$(tests-regexp)' \
|
||||
| build_tools/gnu_parallel -j$(J) --plain --joblog=LOG $$eta --gnu '{} >& t/log-{/}'
|
||||
|
||||
valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionStressTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized|MySQLStyleTransactionTest.TransactionStressTest
|
||||
valgrind-blacklist-regexp = InlineSkipTest.ConcurrentInsert|TransactionTest.DeadlockStress|DBCompactionTest.SuggestCompactRangeNoTwoLevel0Compactions|BackupableDBTest.RateLimiting|DBTest.CloseSpeedup|DBTest.ThreadStatusFlush|DBTest.RateLimitingTest|DBTest.EncodeDecompressedBlockSizeTest|FaultInjectionTest.UninstalledCompaction|HarnessTest.Randomized|ExternalSSTFileTest.CompactDuringAddFileRandom|ExternalSSTFileTest.IngestFileWithGlobalSeqnoRandomized
|
||||
|
||||
.PHONY: valgrind_check_0
|
||||
valgrind_check_0:
|
||||
@@ -845,7 +794,7 @@ valgrind_check_0:
|
||||
test -t 1 && eta=--eta || eta=; \
|
||||
{ \
|
||||
printf './%s\n' $(filter-out $(PARALLEL_TEST) %skiplist_test options_settable_test, $(TESTS)); \
|
||||
find t -name 'run-*' -print; \
|
||||
printf '%s\n' $(t_run); \
|
||||
} \
|
||||
| $(prioritize_long_running_tests) \
|
||||
| grep -E '$(tests-regexp)' \
|
||||
@@ -864,7 +813,7 @@ CLEAN_FILES += t LOG $(TMPD)
|
||||
# regardless of their duration. As with any use of "watch", hit ^C to
|
||||
# interrupt.
|
||||
watch-log:
|
||||
$(WATCH) --interval=0 'sort -k7,7nr -k4,4gr LOG|$(quoted_perl_command)'
|
||||
watch --interval=0 'sort -k7,7nr -k4,4gr LOG|$(quoted_perl_command)'
|
||||
|
||||
# If J != 1 and GNU parallel is installed, run the tests in parallel,
|
||||
# via the check_0 rule above. Otherwise, run them sequentially.
|
||||
@@ -897,15 +846,10 @@ ldb_tests: ldb
|
||||
|
||||
crash_test: whitebox_crash_test blackbox_crash_test
|
||||
|
||||
crash_test_with_atomic_flush: whitebox_crash_test_with_atomic_flush blackbox_crash_test_with_atomic_flush
|
||||
|
||||
blackbox_crash_test: db_stress
|
||||
python -u tools/db_crashtest.py --simple blackbox $(CRASH_TEST_EXT_ARGS)
|
||||
python -u tools/db_crashtest.py blackbox $(CRASH_TEST_EXT_ARGS)
|
||||
|
||||
blackbox_crash_test_with_atomic_flush: db_stress
|
||||
python -u tools/db_crashtest.py --enable_atomic_flush blackbox $(CRASH_TEST_EXT_ARGS)
|
||||
|
||||
ifeq ($(CRASH_TEST_KILL_ODD),)
|
||||
CRASH_TEST_KILL_ODD=888887
|
||||
endif
|
||||
@@ -916,10 +860,6 @@ whitebox_crash_test: db_stress
|
||||
python -u tools/db_crashtest.py whitebox --random_kill_odd \
|
||||
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
|
||||
|
||||
whitebox_crash_test_with_atomic_flush: db_stress
|
||||
python -u tools/db_crashtest.py --enable_atomic_flush whitebox --random_kill_odd \
|
||||
$(CRASH_TEST_KILL_ODD) $(CRASH_TEST_EXT_ARGS)
|
||||
|
||||
asan_check:
|
||||
$(MAKE) clean
|
||||
COMPILE_WITH_ASAN=1 $(MAKE) check -j32
|
||||
@@ -930,11 +870,6 @@ asan_crash_test:
|
||||
COMPILE_WITH_ASAN=1 $(MAKE) crash_test
|
||||
$(MAKE) clean
|
||||
|
||||
asan_crash_test_with_atomic_flush:
|
||||
$(MAKE) clean
|
||||
COMPILE_WITH_ASAN=1 $(MAKE) crash_test_with_atomic_flush
|
||||
$(MAKE) clean
|
||||
|
||||
ubsan_check:
|
||||
$(MAKE) clean
|
||||
COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
|
||||
@@ -945,11 +880,6 @@ ubsan_crash_test:
|
||||
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
|
||||
$(MAKE) clean
|
||||
|
||||
ubsan_crash_test_with_atomic_flush:
|
||||
$(MAKE) clean
|
||||
COMPILE_WITH_UBSAN=1 $(MAKE) crash_test_with_atomic_flush
|
||||
$(MAKE) clean
|
||||
|
||||
valgrind_test:
|
||||
ROCKSDB_VALGRIND_RUN=1 DISABLE_JEMALLOC=1 $(MAKE) valgrind_check
|
||||
|
||||
@@ -1037,10 +967,8 @@ unity.a: unity.o
|
||||
$(AM_V_AR)rm -f $@
|
||||
$(AM_V_at)$(AR) $(ARFLAGS) $@ unity.o
|
||||
|
||||
|
||||
TOOLLIBOBJECTS = $(TOOL_LIB_SOURCES:.cc=.o)
|
||||
# try compiling db_test with unity
|
||||
unity_test: db/db_test.o db/db_test_util.o $(TESTHARNESS) $(TOOLLIBOBJECTS) unity.a
|
||||
unity_test: db/db_test.o db/db_test_util.o $(TESTHARNESS) unity.a
|
||||
$(AM_LINK)
|
||||
./unity_test
|
||||
|
||||
@@ -1050,21 +978,14 @@ rocksdb.h rocksdb.cc: build_tools/amalgamate.py Makefile $(LIB_SOURCES) unity.cc
|
||||
clean:
|
||||
rm -f $(BENCHMARKS) $(TOOLS) $(TESTS) $(LIBRARY) $(SHARED)
|
||||
rm -rf $(CLEAN_FILES) ios-x86 ios-arm scan_build_report
|
||||
$(FIND) . -name "*.[oda]" -exec rm -f {} \;
|
||||
$(FIND) . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
|
||||
find . -name "*.[oda]" -exec rm -f {} \;
|
||||
find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
|
||||
rm -rf bzip2* snappy* zlib* lz4* zstd*
|
||||
cd java; $(MAKE) clean
|
||||
|
||||
tags:
|
||||
ctags -R .
|
||||
cscope -b `$(FIND) . -name '*.cc'` `$(FIND) . -name '*.h'` `$(FIND) . -name '*.c'`
|
||||
ctags -e -R -o etags *
|
||||
|
||||
tags0:
|
||||
ctags -R .
|
||||
cscope -b `$(FIND) . -name '*.cc' -and ! -name '*_test.cc'` \
|
||||
`$(FIND) . -name '*.c' -and ! -name '*_test.c'` \
|
||||
`$(FIND) . -name '*.h' -and ! -name '*_test.h'`
|
||||
ctags * -R
|
||||
cscope -b `find . -name '*.cc'` `find . -name '*.h'` `find . -name '*.c'`
|
||||
ctags -e -R -o etags *
|
||||
|
||||
format:
|
||||
@@ -1080,7 +1001,7 @@ $(LIBRARY): $(LIBOBJECTS)
|
||||
$(AM_V_AR)rm -f $@
|
||||
$(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS)
|
||||
|
||||
$(TOOLS_LIBRARY): $(BENCH_LIB_SOURCES:.cc=.o) $(TOOL_LIB_SOURCES:.cc=.o) $(LIB_SOURCES:.cc=.o) $(TESTUTIL) $(ANALYZER_LIB_SOURCES:.cc=.o)
|
||||
$(TOOLS_LIBRARY): $(BENCH_LIB_SOURCES:.cc=.o) $(TOOL_LIB_SOURCES:.cc=.o) $(LIB_SOURCES:.cc=.o) $(TESTUTIL)
|
||||
$(AM_V_AR)rm -f $@
|
||||
$(AM_V_at)$(AR) $(ARFLAGS) $@ $^
|
||||
|
||||
@@ -1091,9 +1012,6 @@ librocksdb_env_basic_test.a: env/env_basic_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
db_bench: tools/db_bench.o $(BENCHTOOLOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
trace_analyzer: tools/trace_analyzer.o $(ANALYZETOOLOBJECTS) $(LIBOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
cache_bench: cache/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1163,6 +1081,9 @@ cassandra_row_merge_test: utilities/cassandra/cassandra_row_merge_test.o utiliti
|
||||
cassandra_serialize_test: utilities/cassandra/cassandra_serialize_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
hash_table_test: utilities/persistent_cache/hash_table_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1244,9 +1165,6 @@ db_statistics_test: db/db_statistics_test.o db/db_test_util.o $(LIBOBJECTS) $(TE
|
||||
db_write_test: db/db_write_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
error_handler_test: db/error_handler_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
external_sst_file_basic_test: db/external_sst_file_basic_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1259,9 +1177,6 @@ db_tailing_iter_test: db/db_tailing_iter_test.o db/db_test_util.o $(LIBOBJECTS)
|
||||
db_iter_test: db/db_iter_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
db_iter_stress_test: db/db_iter_stress_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
db_universal_compaction_test: db/db_universal_compaction_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1301,9 +1216,18 @@ backupable_db_test: utilities/backupable/backupable_db_test.o $(LIBOBJECTS) $(TE
|
||||
checkpoint_test: utilities/checkpoint/checkpoint_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
document_db_test: utilities/document/document_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
json_document_test: utilities/document/json_document_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
sim_cache_test: utilities/simulator_cache/sim_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1321,6 +1245,9 @@ object_registry_test: utilities/object_registry_test.o $(LIBOBJECTS) $(TESTHARNE
|
||||
ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
date_tiered_test: utilities/date_tiered/date_tiered_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
write_batch_with_index_test: utilities/write_batch_with_index/write_batch_with_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1387,9 +1314,6 @@ table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
block_test: table/block_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
data_block_hash_index_test: table/data_block_hash_index_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
inlineskiplist_test: memtable/inlineskiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1444,7 +1368,7 @@ options_file_test: db/options_file_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
obsolete_files_test: db/obsolete_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
geodb_test: utilities/geodb/geodb_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
rocksdb_dump: tools/dump/rocksdb_dump.o $(LIBOBJECTS)
|
||||
@@ -1483,9 +1407,6 @@ options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHAR
|
||||
db_bench_tool_test: tools/db_bench_tool_test.o $(BENCHTOOLOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
trace_analyzer_test: tools/trace_analyzer_test.o $(LIBOBJECTS) $(ANALYZETOOLOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
event_logger_test: util/event_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1495,6 +1416,9 @@ timer_queue_test: util/timer_queue_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
sst_dump_test: tools/sst_dump_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
column_aware_encoding_test: utilities/column_aware_encoding_test.o $(TESTHARNESS) $(EXPOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
optimistic_transaction_test: utilities/transactions/optimistic_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1525,15 +1449,15 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
|
||||
write_prepared_transaction_test: utilities/transactions/write_prepared_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
write_unprepared_transaction_test: utilities/transactions/write_unprepared_transaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
sst_dump: tools/sst_dump.o $(LIBOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
blob_dump: tools/blob_dump.o $(LIBOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
column_aware_encoding_exp: utilities/column_aware_encoding_exp.o $(EXPOBJECTS)
|
||||
$(AM_LINK)
|
||||
|
||||
repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
@@ -1555,27 +1479,15 @@ statistics_test: monitoring/statistics_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
lru_cache_test: cache/lru_cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
lua_test: utilities/lua/rocks_lua_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
range_del_aggregator_bench: db/range_del_aggregator_bench.o $(LIBOBJECTS) $(TESTUTIL)
|
||||
$(AM_LINK)
|
||||
|
||||
blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
repeatable_thread_test: util/repeatable_thread_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
range_tombstone_fragmenter_test: db/range_tombstone_fragmenter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
sst_file_reader_test: table/sst_file_reader_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
db_secondary_test: db/db_secondary_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(AM_LINK)
|
||||
|
||||
#-------------------------------------------------
|
||||
# make install related stuff
|
||||
INSTALL_PATH ?= /usr/local
|
||||
@@ -1590,10 +1502,10 @@ uninstall:
|
||||
|
||||
install-headers:
|
||||
install -d $(INSTALL_PATH)/lib
|
||||
for header_dir in `$(FIND) "include/rocksdb" -type d`; do \
|
||||
for header_dir in `find "include/rocksdb" -type d`; do \
|
||||
install -d $(INSTALL_PATH)/$$header_dir; \
|
||||
done
|
||||
for header in `$(FIND) "include/rocksdb" -type f -name *.h`; do \
|
||||
for header in `find "include/rocksdb" -type f -name *.h`; do \
|
||||
install -C -m 644 $$header $(INSTALL_PATH)/$$header; \
|
||||
done
|
||||
|
||||
@@ -1620,12 +1532,6 @@ install: install-static
|
||||
JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
|
||||
ifeq ($(PLATFORM), OS_SOLARIS)
|
||||
ARCH := $(shell isainfo -b)
|
||||
else ifeq ($(PLATFORM), OS_OPENBSD)
|
||||
ifneq (,$(filter $(MACHINE), amd64 arm64 sparc64))
|
||||
ARCH := 64
|
||||
else
|
||||
ARCH := 32
|
||||
endif
|
||||
else
|
||||
ARCH := $(shell getconf LONG_BIT)
|
||||
endif
|
||||
@@ -1646,17 +1552,16 @@ ZLIB_SHA256 ?= c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1
|
||||
ZLIB_DOWNLOAD_BASE ?= http://zlib.net
|
||||
BZIP2_VER ?= 1.0.6
|
||||
BZIP2_SHA256 ?= a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd
|
||||
BZIP2_DOWNLOAD_BASE ?= https://web.archive.org/web/20180624184835/http://www.bzip.org
|
||||
SNAPPY_VER ?= 1.1.7
|
||||
SNAPPY_SHA256 ?= 3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4
|
||||
SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/archive
|
||||
LZ4_VER ?= 1.8.3
|
||||
LZ4_SHA256 ?= 33af5936ac06536805f9745e0b6d61da606a1f8b4cc5c04dd3cbaca3b9b4fc43
|
||||
BZIP2_DOWNLOAD_BASE ?= http://www.bzip.org
|
||||
SNAPPY_VER ?= 1.1.4
|
||||
SNAPPY_SHA256 ?= 134bfe122fd25599bb807bb8130e7ba6d9bdb851e0b16efcb83ac4f5d0b70057
|
||||
SNAPPY_DOWNLOAD_BASE ?= https://github.com/google/snappy/releases/download
|
||||
LZ4_VER ?= 1.8.0
|
||||
LZ4_SHA256 ?= 2ca482ea7a9bb103603108b5a7510b7592b90158c151ff50a28f1ca8389fccf6
|
||||
LZ4_DOWNLOAD_BASE ?= https://github.com/lz4/lz4/archive
|
||||
ZSTD_VER ?= 1.3.7
|
||||
ZSTD_SHA256 ?= 5dd1e90eb16c25425880c8a91327f63de22891ffed082fcc17e5ae84fce0d5fb
|
||||
ZSTD_VER ?= 1.3.3
|
||||
ZSTD_SHA256 ?= a77c47153ee7de02626c5b2a097005786b71688be61e9fb81806a011f90b297b
|
||||
ZSTD_DOWNLOAD_BASE ?= https://github.com/facebook/zstd/archive
|
||||
CURL_SSL_OPTS ?= --tlsv1
|
||||
|
||||
ifeq ($(PLATFORM), OS_MACOSX)
|
||||
ROCKSDBJNILIB = librocksdbjni-osx.jnilib
|
||||
@@ -1685,17 +1590,10 @@ ifeq ($(PLATFORM), OS_AIX)
|
||||
EXTRACT_SOURCES = gunzip < TAR_GZ | tar xvf -
|
||||
SNAPPY_MAKE_TARGET = libsnappy.la
|
||||
endif
|
||||
ifeq ($(PLATFORM), OS_OPENBSD)
|
||||
JAVA_INCLUDE = -I$(JAVA_HOME)/include -I$(JAVA_HOME)/include/openbsd
|
||||
ROCKSDBJNILIB = librocksdbjni-openbsd$(ARCH).so
|
||||
ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-openbsd$(ARCH).jar
|
||||
endif
|
||||
|
||||
libz.a:
|
||||
-rm -rf zlib-$(ZLIB_VER)
|
||||
ifeq (,$(wildcard ./zlib-$(ZLIB_VER).tar.gz))
|
||||
curl --output zlib-$(ZLIB_VER).tar.gz -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
|
||||
endif
|
||||
curl -O -L ${ZLIB_DOWNLOAD_BASE}/zlib-$(ZLIB_VER).tar.gz
|
||||
ZLIB_SHA256_ACTUAL=`$(SHA256_CMD) zlib-$(ZLIB_VER).tar.gz | cut -d ' ' -f 1`; \
|
||||
if [ "$(ZLIB_SHA256)" != "$$ZLIB_SHA256_ACTUAL" ]; then \
|
||||
echo zlib-$(ZLIB_VER).tar.gz checksum mismatch, expected=\"$(ZLIB_SHA256)\" actual=\"$$ZLIB_SHA256_ACTUAL\"; \
|
||||
@@ -1707,9 +1605,7 @@ endif
|
||||
|
||||
libbz2.a:
|
||||
-rm -rf bzip2-$(BZIP2_VER)
|
||||
ifeq (,$(wildcard ./bzip2-$(BZIP2_VER).tar.gz))
|
||||
curl --output bzip2-$(BZIP2_VER).tar.gz -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
|
||||
endif
|
||||
curl -O -L ${BZIP2_DOWNLOAD_BASE}/$(BZIP2_VER)/bzip2-$(BZIP2_VER).tar.gz
|
||||
BZIP2_SHA256_ACTUAL=`$(SHA256_CMD) bzip2-$(BZIP2_VER).tar.gz | cut -d ' ' -f 1`; \
|
||||
if [ "$(BZIP2_SHA256)" != "$$BZIP2_SHA256_ACTUAL" ]; then \
|
||||
echo bzip2-$(BZIP2_VER).tar.gz checksum mismatch, expected=\"$(BZIP2_SHA256)\" actual=\"$$BZIP2_SHA256_ACTUAL\"; \
|
||||
@@ -1721,24 +1617,21 @@ endif
|
||||
|
||||
libsnappy.a:
|
||||
-rm -rf snappy-$(SNAPPY_VER)
|
||||
ifeq (,$(wildcard ./snappy-$(SNAPPY_VER).tar.gz))
|
||||
curl --output snappy-$(SNAPPY_VER).tar.gz -L ${CURL_SSL_OPTS} ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER).tar.gz
|
||||
endif
|
||||
curl -O -L ${SNAPPY_DOWNLOAD_BASE}/$(SNAPPY_VER)/snappy-$(SNAPPY_VER).tar.gz
|
||||
SNAPPY_SHA256_ACTUAL=`$(SHA256_CMD) snappy-$(SNAPPY_VER).tar.gz | cut -d ' ' -f 1`; \
|
||||
if [ "$(SNAPPY_SHA256)" != "$$SNAPPY_SHA256_ACTUAL" ]; then \
|
||||
echo snappy-$(SNAPPY_VER).tar.gz checksum mismatch, expected=\"$(SNAPPY_SHA256)\" actual=\"$$SNAPPY_SHA256_ACTUAL\"; \
|
||||
exit 1; \
|
||||
fi
|
||||
tar xvzf snappy-$(SNAPPY_VER).tar.gz
|
||||
mkdir snappy-$(SNAPPY_VER)/build
|
||||
cd snappy-$(SNAPPY_VER)/build && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON .. && $(MAKE) ${SNAPPY_MAKE_TARGET}
|
||||
cp snappy-$(SNAPPY_VER)/build/libsnappy.a .
|
||||
cd snappy-$(SNAPPY_VER) && CFLAGS='${EXTRA_CFLAGS}' CXXFLAGS='${EXTRA_CXXFLAGS}' LDFLAGS='${EXTRA_LDFLAGS}' ./configure --with-pic --enable-static --disable-shared
|
||||
cd snappy-$(SNAPPY_VER) && $(MAKE) ${SNAPPY_MAKE_TARGET}
|
||||
cp snappy-$(SNAPPY_VER)/.libs/libsnappy.a .
|
||||
|
||||
liblz4.a:
|
||||
-rm -rf lz4-$(LZ4_VER)
|
||||
ifeq (,$(wildcard ./lz4-$(LZ4_VER).tar.gz))
|
||||
curl --output lz4-$(LZ4_VER).tar.gz -L ${CURL_SSL_OPTS} ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
|
||||
endif
|
||||
curl -O -L ${LZ4_DOWNLOAD_BASE}/v$(LZ4_VER).tar.gz
|
||||
mv v$(LZ4_VER).tar.gz lz4-$(LZ4_VER).tar.gz
|
||||
LZ4_SHA256_ACTUAL=`$(SHA256_CMD) lz4-$(LZ4_VER).tar.gz | cut -d ' ' -f 1`; \
|
||||
if [ "$(LZ4_SHA256)" != "$$LZ4_SHA256_ACTUAL" ]; then \
|
||||
echo lz4-$(LZ4_VER).tar.gz checksum mismatch, expected=\"$(LZ4_SHA256)\" actual=\"$$LZ4_SHA256_ACTUAL\"; \
|
||||
@@ -1750,9 +1643,8 @@ endif
|
||||
|
||||
libzstd.a:
|
||||
-rm -rf zstd-$(ZSTD_VER)
|
||||
ifeq (,$(wildcard ./zstd-$(ZSTD_VER).tar.gz))
|
||||
curl --output zstd-$(ZSTD_VER).tar.gz -L ${CURL_SSL_OPTS} ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
|
||||
endif
|
||||
curl -O -L ${ZSTD_DOWNLOAD_BASE}/v$(ZSTD_VER).tar.gz
|
||||
mv v$(ZSTD_VER).tar.gz zstd-$(ZSTD_VER).tar.gz
|
||||
ZSTD_SHA256_ACTUAL=`$(SHA256_CMD) zstd-$(ZSTD_VER).tar.gz | cut -d ' ' -f 1`; \
|
||||
if [ "$(ZSTD_SHA256)" != "$$ZSTD_SHA256_ACTUAL" ]; then \
|
||||
echo zstd-$(ZSTD_VER).tar.gz checksum mismatch, expected=\"$(ZSTD_SHA256)\" actual=\"$$ZSTD_SHA256_ACTUAL\"; \
|
||||
@@ -1799,9 +1691,7 @@ rocksdbjavastatic: $(java_static_all_libobjects)
|
||||
-o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) \
|
||||
$(java_static_all_libobjects) $(COVERAGEFLAGS) \
|
||||
$(JAVA_COMPRESSIONS) $(JAVA_STATIC_LDFLAGS)
|
||||
cd java/target;if [ "$(DEBUG_LEVEL)" == "0" ]; then \
|
||||
strip $(STRIPFLAGS) $(ROCKSDBJNILIB); \
|
||||
fi
|
||||
cd java/target;strip $(STRIPFLAGS) $(ROCKSDBJNILIB)
|
||||
cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
|
||||
cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
|
||||
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
|
||||
@@ -1814,26 +1704,20 @@ rocksdbjavastaticrelease: rocksdbjavastatic
|
||||
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
|
||||
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
|
||||
|
||||
rocksdbjavastaticreleasedocker: rocksdbjavastatic rocksdbjavastaticdockerx86 rocksdbjavastaticdockerx86_64
|
||||
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
|
||||
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
|
||||
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
|
||||
|
||||
rocksdbjavastaticdockerx86:
|
||||
mkdir -p java/target
|
||||
DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
|
||||
if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
|
||||
docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
|
||||
fi
|
||||
docker start -a rocksdb_linux_x86-be
|
||||
|
||||
rocksdbjavastaticdockerx86_64:
|
||||
mkdir -p java/target
|
||||
rocksdbjavastaticreleasedocker: rocksdbjavastatic
|
||||
DOCKER_LINUX_X64_CONTAINER=`docker ps -aqf name=rocksdb_linux_x64-be`; \
|
||||
if [ -z "$$DOCKER_LINUX_X64_CONTAINER" ]; then \
|
||||
docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
|
||||
fi
|
||||
docker start -a rocksdb_linux_x64-be
|
||||
DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
|
||||
if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
|
||||
docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
|
||||
fi
|
||||
docker start -a rocksdb_linux_x86-be
|
||||
cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md
|
||||
cd java/target;jar -uf $(ROCKSDB_JAR_ALL) librocksdbjni-*.so librocksdbjni-*.jnilib
|
||||
cd java/target/classes;jar -uf ../$(ROCKSDB_JAR_ALL) org/rocksdb/*.class org/rocksdb/util/*.class
|
||||
|
||||
rocksdbjavastaticdockerppc64le:
|
||||
mkdir -p java/target
|
||||
@@ -1889,7 +1773,7 @@ $(java_libobjects): jl/%.o: %.cc
|
||||
rocksdbjava: $(java_all_libobjects)
|
||||
$(AM_V_GEN)cd java;$(MAKE) javalib;
|
||||
$(AM_V_at)rm -f ./java/target/$(ROCKSDBJNILIB)
|
||||
$(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(java_all_libobjects) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(AM_V_at)$(CXX) $(CXXFLAGS) -I./java/. $(JAVA_INCLUDE) -shared -fPIC -o ./java/target/$(ROCKSDBJNILIB) $(JNI_NATIVE_SOURCES) $(java_libobjects) $(JAVA_LDFLAGS) $(COVERAGEFLAGS)
|
||||
$(AM_V_at)cd java;jar -cf target/$(ROCKSDB_JAR) HISTORY*.md
|
||||
$(AM_V_at)cd java/target;jar -uf $(ROCKSDB_JAR) $(ROCKSDBJNILIB)
|
||||
$(AM_V_at)cd java/target/classes;jar -uf ../$(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
|
||||
@@ -1921,8 +1805,7 @@ commit_prereq: build_tools/rocksdb-lego-determinator \
|
||||
ifeq ($(PLATFORM), IOS)
|
||||
# For iOS, create universal object files to be used on both the simulator and
|
||||
# a device.
|
||||
XCODEROOT=$(shell xcode-select -print-path)
|
||||
PLATFORMSROOT=$(XCODEROOT)/Platforms
|
||||
PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms
|
||||
SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer
|
||||
DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer
|
||||
IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString)
|
||||
@@ -1959,7 +1842,7 @@ endif
|
||||
# Source files dependencies detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(ANALYZER_LIB_SOURCES)
|
||||
all_sources = $(LIB_SOURCES) $(MAIN_SOURCES) $(MOCK_LIB_SOURCES) $(TOOL_LIB_SOURCES) $(BENCH_LIB_SOURCES) $(TEST_LIB_SOURCES) $(EXP_LIB_SOURCES)
|
||||
DEPFILES = $(all_sources:.cc=.cc.d)
|
||||
|
||||
# Add proper dependency support so changing a .h file forces a .cc file to
|
||||
|
||||
@@ -4,8 +4,9 @@
|
||||
[](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
|
||||
[](http://140.211.168.68:8080/job/Rocksdb)
|
||||
|
||||
|
||||
RocksDB is developed and maintained by Facebook Database Engineering Team.
|
||||
It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com)
|
||||
It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
|
||||
and Jeff Dean (jeff@google.com)
|
||||
|
||||
This code is a library that forms the core building block for a fast
|
||||
@@ -25,7 +26,3 @@ rely on the details of any other header files in this package. Those
|
||||
internal APIs may be changed without warning.
|
||||
|
||||
Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/
|
||||
|
||||
## License
|
||||
|
||||
RocksDB is dual-licensed under both the GPLv2 (found in the COPYING file in the root directory) and Apache 2.0 License (found in the LICENSE.Apache file in the root directory). You may select, at your option, one of the above-listed licenses.
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@ RocksDBLite is a project focused on mobile use cases, which don't need a lot of
|
||||
Some examples of the features disabled by ROCKSDB_LITE:
|
||||
* compiled-in support for LDB tool
|
||||
* No backupable DB
|
||||
* No support for replication (which we provide in form of TransactionalIterator)
|
||||
* No support for replication (which we provide in form of TrasactionalIterator)
|
||||
* No advanced monitoring tools
|
||||
* No special-purpose memtables that are highly optimized for specific use cases
|
||||
* No Transactions
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
|
||||
load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
|
||||
load(":defs.bzl", "test_binary")
|
||||
|
||||
REPO_PATH = package_name() + "/"
|
||||
|
||||
ROCKSDB_COMPILER_FLAGS = [
|
||||
BUCK_BINS = "buck-out/gen/" + REPO_PATH
|
||||
|
||||
TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
|
||||
|
||||
rocksdb_compiler_flags = [
|
||||
"-fno-builtin-memcmp",
|
||||
"-DROCKSDB_PLATFORM_POSIX",
|
||||
"-DROCKSDB_LIB_IO_POSIX",
|
||||
@@ -20,19 +20,14 @@ ROCKSDB_COMPILER_FLAGS = [
|
||||
"-DBZIP2",
|
||||
"-DLZ4",
|
||||
"-DZSTD",
|
||||
"-DZSTD_STATIC_LINKING_ONLY",
|
||||
"-DGFLAGS=gflags",
|
||||
"-DNUMA",
|
||||
"-DTBB",
|
||||
# Needed to compile in fbcode
|
||||
"-Wno-expansion-to-defined",
|
||||
# Added missing flags from output of build_detect_platform
|
||||
"-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
|
||||
"-DROCKSDB_BACKTRACE",
|
||||
"-Wnarrowing",
|
||||
]
|
||||
|
||||
ROCKSDB_EXTERNAL_DEPS = [
|
||||
rocksdb_external_deps = [
|
||||
("bzip2", None, "bz2"),
|
||||
("snappy", None, "snappy"),
|
||||
("zlib", None, "z"),
|
||||
@@ -44,17 +39,14 @@ ROCKSDB_EXTERNAL_DEPS = [
|
||||
("googletest", None, "gtest"),
|
||||
]
|
||||
|
||||
ROCKSDB_PREPROCESSOR_FLAGS = [
|
||||
rocksdb_preprocessor_flags = [
|
||||
# Directories with files for #include
|
||||
"-I" + REPO_PATH + "include/",
|
||||
"-I" + REPO_PATH,
|
||||
]
|
||||
|
||||
ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
|
||||
"x86_64": [
|
||||
"-DHAVE_SSE42",
|
||||
"-DHAVE_PCLMUL",
|
||||
],
|
||||
rocksdb_arch_preprocessor_flags = {
|
||||
"x86_64": ["-DHAVE_SSE42"],
|
||||
}
|
||||
|
||||
build_mode = read_config("fbcode", "build_mode")
|
||||
@@ -63,15 +55,8 @@ is_opt_mode = build_mode.startswith("opt")
|
||||
|
||||
# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
|
||||
# doesn't harm and avoid forgetting to add it.
|
||||
ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
|
||||
|
||||
sanitizer = read_config("fbcode", "sanitizer")
|
||||
|
||||
# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
|
||||
# whether the binary is linked with jemalloc at runtime.
|
||||
ROCKSDB_COMPILER_FLAGS += (["-DROCKSDB_JEMALLOC"] if sanitizer == "" else [])
|
||||
|
||||
ROCKSDB_EXTERNAL_DEPS += ([("jemalloc", None, "headers")] if sanitizer == "" else [])
|
||||
if is_opt_mode:
|
||||
rocksdb_compiler_flags.append("-DNDEBUG")
|
||||
|
||||
cpp_library(
|
||||
name = "rocksdb_lib",
|
||||
@@ -87,7 +72,6 @@ cpp_library(
|
||||
"db/compaction_iterator.cc",
|
||||
"db/compaction_job.cc",
|
||||
"db/compaction_picker.cc",
|
||||
"db/compaction_picker_fifo.cc",
|
||||
"db/compaction_picker_universal.cc",
|
||||
"db/convenience.cc",
|
||||
"db/db_filesnapshot.cc",
|
||||
@@ -98,12 +82,10 @@ cpp_library(
|
||||
"db/db_impl_files.cc",
|
||||
"db/db_impl_open.cc",
|
||||
"db/db_impl_readonly.cc",
|
||||
"db/db_impl_secondary.cc",
|
||||
"db/db_impl_write.cc",
|
||||
"db/db_info_dumper.cc",
|
||||
"db/db_iter.cc",
|
||||
"db/dbformat.cc",
|
||||
"db/error_handler.cc",
|
||||
"db/event_helpers.cc",
|
||||
"db/experimental.cc",
|
||||
"db/external_sst_file_ingestion_job.cc",
|
||||
@@ -111,18 +93,16 @@ cpp_library(
|
||||
"db/flush_job.cc",
|
||||
"db/flush_scheduler.cc",
|
||||
"db/forward_iterator.cc",
|
||||
"db/in_memory_stats_history.cc",
|
||||
"db/internal_stats.cc",
|
||||
"db/log_reader.cc",
|
||||
"db/log_writer.cc",
|
||||
"db/logs_with_prep_tracker.cc",
|
||||
"db/malloc_stats.cc",
|
||||
"db/managed_iterator.cc",
|
||||
"db/memtable.cc",
|
||||
"db/memtable_list.cc",
|
||||
"db/merge_helper.cc",
|
||||
"db/merge_operator.cc",
|
||||
"db/range_del_aggregator.cc",
|
||||
"db/range_tombstone_fragmenter.cc",
|
||||
"db/repair.cc",
|
||||
"db/snapshot_impl.cc",
|
||||
"db/table_cache.cc",
|
||||
@@ -144,6 +124,7 @@ cpp_library(
|
||||
"env/io_posix.cc",
|
||||
"env/mock_env.cc",
|
||||
"memtable/alloc_tracker.cc",
|
||||
"memtable/hash_cuckoo_rep.cc",
|
||||
"memtable/hash_linklist_rep.cc",
|
||||
"memtable/hash_skiplist_rep.cc",
|
||||
"memtable/skiplistrep.cc",
|
||||
@@ -182,8 +163,6 @@ cpp_library(
|
||||
"table/cuckoo_table_builder.cc",
|
||||
"table/cuckoo_table_factory.cc",
|
||||
"table/cuckoo_table_reader.cc",
|
||||
"table/data_block_footer.cc",
|
||||
"table/data_block_hash_index.cc",
|
||||
"table/flush_block_policy.cc",
|
||||
"table/format.cc",
|
||||
"table/full_filter_block.cc",
|
||||
@@ -199,7 +178,6 @@ cpp_library(
|
||||
"table/plain_table_index.cc",
|
||||
"table/plain_table_key_coding.cc",
|
||||
"table/plain_table_reader.cc",
|
||||
"table/sst_file_reader.cc",
|
||||
"table/sst_file_writer.cc",
|
||||
"table/table_properties.cc",
|
||||
"table/two_level_iterator.cc",
|
||||
@@ -214,9 +192,7 @@ cpp_library(
|
||||
"util/coding.cc",
|
||||
"util/compaction_job_stats_impl.cc",
|
||||
"util/comparator.cc",
|
||||
"util/compression_context_cache.cc",
|
||||
"util/concurrent_arena.cc",
|
||||
"util/concurrent_task_limiter_impl.cc",
|
||||
"util/crc32c.cc",
|
||||
"util/delete_scheduler.cc",
|
||||
"util/dynamic_bloom.cc",
|
||||
@@ -226,7 +202,6 @@ cpp_library(
|
||||
"util/filename.cc",
|
||||
"util/filter_policy.cc",
|
||||
"util/hash.cc",
|
||||
"util/jemalloc_nodump_allocator.cc",
|
||||
"util/log_buffer.cc",
|
||||
"util/murmurhash.cc",
|
||||
"util/random.cc",
|
||||
@@ -234,36 +209,39 @@ cpp_library(
|
||||
"util/slice.cc",
|
||||
"util/sst_file_manager_impl.cc",
|
||||
"util/status.cc",
|
||||
"util/status_message.cc",
|
||||
"util/string_util.cc",
|
||||
"util/sync_point.cc",
|
||||
"util/sync_point_impl.cc",
|
||||
"util/thread_local.cc",
|
||||
"util/threadpool_imp.cc",
|
||||
"util/trace_replay.cc",
|
||||
"util/transaction_test_util.cc",
|
||||
"util/xxhash.cc",
|
||||
"utilities/backupable/backupable_db.cc",
|
||||
"utilities/blob_db/blob_compaction_filter.cc",
|
||||
"utilities/blob_db/blob_db.cc",
|
||||
"utilities/blob_db/blob_db_impl.cc",
|
||||
"utilities/blob_db/blob_db_impl_filesnapshot.cc",
|
||||
"utilities/blob_db/blob_dump_tool.cc",
|
||||
"utilities/blob_db/blob_file.cc",
|
||||
"utilities/blob_db/blob_log_format.cc",
|
||||
"utilities/blob_db/blob_log_reader.cc",
|
||||
"utilities/blob_db/blob_log_writer.cc",
|
||||
"utilities/blob_db/ttl_extractor.cc",
|
||||
"utilities/cassandra/cassandra_compaction_filter.cc",
|
||||
"utilities/cassandra/format.cc",
|
||||
"utilities/cassandra/merge_operator.cc",
|
||||
"utilities/checkpoint/checkpoint_impl.cc",
|
||||
"utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc",
|
||||
"utilities/convenience/info_log_finder.cc",
|
||||
"utilities/date_tiered/date_tiered_db_impl.cc",
|
||||
"utilities/debug.cc",
|
||||
"utilities/document/document_db.cc",
|
||||
"utilities/document/json_document.cc",
|
||||
"utilities/document/json_document_builder.cc",
|
||||
"utilities/env_mirror.cc",
|
||||
"utilities/env_timed.cc",
|
||||
"utilities/geodb/geodb_impl.cc",
|
||||
"utilities/leveldb_options/leveldb_options.cc",
|
||||
"utilities/lua/rocks_lua_compaction_filter.cc",
|
||||
"utilities/memory/memory_util.cc",
|
||||
"utilities/merge_operators/bytesxor.cc",
|
||||
"utilities/merge_operators/max.cc",
|
||||
"utilities/merge_operators/put.cc",
|
||||
"utilities/merge_operators/string_append/stringappend.cc",
|
||||
@@ -276,9 +254,10 @@ cpp_library(
|
||||
"utilities/persistent_cache/block_cache_tier_metadata.cc",
|
||||
"utilities/persistent_cache/persistent_cache_tier.cc",
|
||||
"utilities/persistent_cache/volatile_tier_impl.cc",
|
||||
"utilities/redis/redis_lists.cc",
|
||||
"utilities/simulator_cache/sim_cache.cc",
|
||||
"utilities/spatialdb/spatial_db.cc",
|
||||
"utilities/table_properties_collectors/compact_on_deletion_collector.cc",
|
||||
"utilities/trace/file_trace_reader_writer.cc",
|
||||
"utilities/transactions/optimistic_transaction.cc",
|
||||
"utilities/transactions/optimistic_transaction_db_impl.cc",
|
||||
"utilities/transactions/pessimistic_transaction.cc",
|
||||
@@ -290,18 +269,16 @@ cpp_library(
|
||||
"utilities/transactions/transaction_util.cc",
|
||||
"utilities/transactions/write_prepared_txn.cc",
|
||||
"utilities/transactions/write_prepared_txn_db.cc",
|
||||
"utilities/transactions/write_unprepared_txn.cc",
|
||||
"utilities/transactions/write_unprepared_txn_db.cc",
|
||||
"utilities/ttl/db_ttl_impl.cc",
|
||||
"utilities/write_batch_with_index/write_batch_with_index.cc",
|
||||
"utilities/write_batch_with_index/write_batch_with_index_internal.cc",
|
||||
],
|
||||
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [],
|
||||
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
cpp_library(
|
||||
@@ -309,44 +286,45 @@ cpp_library(
|
||||
srcs = [
|
||||
"db/db_test_util.cc",
|
||||
"table/mock_table.cc",
|
||||
"tools/trace_analyzer_tool.cc",
|
||||
"util/fault_injection_test_env.cc",
|
||||
"util/testharness.cc",
|
||||
"util/testutil.cc",
|
||||
"utilities/cassandra/test_utils.cc",
|
||||
"utilities/col_buf_decoder.cc",
|
||||
"utilities/col_buf_encoder.cc",
|
||||
"utilities/column_aware_encoding_util.cc",
|
||||
],
|
||||
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [":rocksdb_lib"],
|
||||
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
cpp_library(
|
||||
name = "rocksdb_tools_lib",
|
||||
srcs = [
|
||||
"tools/db_bench_tool.cc",
|
||||
"tools/trace_analyzer_tool.cc",
|
||||
"util/testutil.cc",
|
||||
],
|
||||
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [":rocksdb_lib"],
|
||||
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
cpp_library(
|
||||
name = "env_basic_test_lib",
|
||||
srcs = ["env/env_basic_test.cc"],
|
||||
auto_headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
headers = AutoHeaders.RECURSIVE_GLOB,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [":rocksdb_test_lib"],
|
||||
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
# [test_name, test_src, test_type]
|
||||
@@ -436,6 +414,11 @@ ROCKS_TESTS = [
|
||||
"util/coding_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"column_aware_encoding_test",
|
||||
"utilities/column_aware_encoding_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"column_family_test",
|
||||
"db/column_family_test.cc",
|
||||
@@ -502,8 +485,8 @@ ROCKS_TESTS = [
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"data_block_hash_index_test",
|
||||
"table/data_block_hash_index_test.cc",
|
||||
"date_tiered_test",
|
||||
"utilities/date_tiered/date_tiered_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
@@ -561,11 +544,6 @@ ROCKS_TESTS = [
|
||||
"db/db_io_failure_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"db_iter_stress_test",
|
||||
"db/db_iter_stress_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"db_iter_test",
|
||||
"db/db_iter_test.cc",
|
||||
@@ -606,11 +584,6 @@ ROCKS_TESTS = [
|
||||
"db/db_range_del_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"db_secondary_test",
|
||||
"db/db_secondary_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"db_sst_test",
|
||||
"db/db_sst_test.cc",
|
||||
@@ -671,6 +644,11 @@ ROCKS_TESTS = [
|
||||
"db/deletefile_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"document_db_test",
|
||||
"utilities/document/document_db_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"dynamic_bloom_test",
|
||||
"util/dynamic_bloom_test.cc",
|
||||
@@ -691,11 +669,6 @@ ROCKS_TESTS = [
|
||||
"utilities/env_timed_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"error_handler_test",
|
||||
"db/error_handler_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"event_logger_test",
|
||||
"util/event_logger_test.cc",
|
||||
@@ -746,6 +719,11 @@ ROCKS_TESTS = [
|
||||
"table/full_filter_block_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"geodb_test",
|
||||
"utilities/geodb/geodb_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"hash_table_test",
|
||||
"utilities/persistent_cache/hash_table_test.cc",
|
||||
@@ -776,6 +754,11 @@ ROCKS_TESTS = [
|
||||
"monitoring/iostats_context_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"json_document_test",
|
||||
"utilities/document/json_document_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"ldb_cmd_test",
|
||||
"tools/ldb_cmd_test.cc",
|
||||
@@ -836,11 +819,6 @@ ROCKS_TESTS = [
|
||||
"utilities/object_registry_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"obsolete_files_test",
|
||||
"db/obsolete_files_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"optimistic_transaction_test",
|
||||
"utilities/transactions/optimistic_transaction_test.cc",
|
||||
@@ -901,11 +879,6 @@ ROCKS_TESTS = [
|
||||
"db/range_del_aggregator_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"range_tombstone_fragmenter_test",
|
||||
"db/range_tombstone_fragmenter_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"rate_limiter_test",
|
||||
"util/rate_limiter_test.cc",
|
||||
@@ -921,11 +894,6 @@ ROCKS_TESTS = [
|
||||
"db/repair_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"repeatable_thread_test",
|
||||
"util/repeatable_thread_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"sim_cache_test",
|
||||
"utilities/simulator_cache/sim_cache_test.cc",
|
||||
@@ -942,13 +910,13 @@ ROCKS_TESTS = [
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"sst_dump_test",
|
||||
"tools/sst_dump_test.cc",
|
||||
"spatial_db_test",
|
||||
"utilities/spatialdb/spatial_db_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"sst_file_reader_test",
|
||||
"table/sst_file_reader_test.cc",
|
||||
"sst_dump_test",
|
||||
"tools/sst_dump_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
@@ -986,11 +954,6 @@ ROCKS_TESTS = [
|
||||
"util/timer_queue_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"trace_analyzer_test",
|
||||
"tools/trace_analyzer_test.cc",
|
||||
"serial",
|
||||
],
|
||||
[
|
||||
"transaction_test",
|
||||
"utilities/transactions/transaction_test.cc",
|
||||
@@ -1056,26 +1019,43 @@ ROCKS_TESTS = [
|
||||
"utilities/transactions/write_prepared_transaction_test.cc",
|
||||
"parallel",
|
||||
],
|
||||
[
|
||||
"write_unprepared_transaction_test",
|
||||
"utilities/transactions/write_unprepared_transaction_test.cc",
|
||||
"parallel",
|
||||
],
|
||||
]
|
||||
|
||||
# Generate a test rule for each entry in ROCKS_TESTS
|
||||
# Do not build the tests in opt mode, since SyncPoint and other test code
|
||||
# will not be included.
|
||||
[
|
||||
test_binary(
|
||||
parallelism = parallelism,
|
||||
rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
test_cc = test_cc,
|
||||
test_name = test_name,
|
||||
)
|
||||
for test_name, test_cc, parallelism in ROCKS_TESTS
|
||||
if not is_opt_mode
|
||||
]
|
||||
if not is_opt_mode:
|
||||
for test_cfg in ROCKS_TESTS:
|
||||
test_name = test_cfg[0]
|
||||
test_cc = test_cfg[1]
|
||||
ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
|
||||
test_bin = test_name + "_bin"
|
||||
|
||||
cpp_binary (
|
||||
name = test_bin,
|
||||
srcs = [test_cc],
|
||||
deps = [":rocksdb_test_lib"],
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
custom_unittest(
|
||||
name = test_name,
|
||||
type = ttype,
|
||||
deps = [":" + test_bin],
|
||||
command = [TEST_RUNNER, BUCK_BINS + test_bin]
|
||||
)
|
||||
|
||||
custom_unittest(
|
||||
name = "make_rocksdbjavastatic",
|
||||
command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
|
||||
type = "simple",
|
||||
)
|
||||
|
||||
custom_unittest(
|
||||
name = "make_rocksdb_lite_release",
|
||||
command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
|
||||
type = "simple",
|
||||
)
|
||||
|
||||
@@ -12,7 +12,6 @@ At Facebook, we use RocksDB as storage engines in multiple data management servi
|
||||
6. LogDevice -- a distributed data store for logs [2]
|
||||
|
||||
[1] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
|
||||
|
||||
[2] https://code.facebook.com/posts/357056558062811/logdevice-a-distributed-data-store-for-logs/
|
||||
|
||||
## LinkedIn
|
||||
@@ -27,7 +26,7 @@ Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasu
|
||||
Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
|
||||
|
||||
## CockroachDB
|
||||
CockroachDB is an open-source geo-replicated transactional database. They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
|
||||
CockroachDB is an open-source geo-replicated transactional database (still in development). They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
|
||||
|
||||
## DNANexus
|
||||
DNANexus is using RocksDB to speed up processing of genomics data.
|
||||
@@ -89,6 +88,3 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed
|
||||
|
||||
## ProfaneDB
|
||||
[ProfaneDB](https://profanedb.gitlab.io/) is a database for Protocol Buffers, and uses RocksDB for storage. It is accessible via gRPC, and the schema is defined using directly `.proto` files.
|
||||
|
||||
## IOTA Foundation
|
||||
[IOTA Foundation](https://www.iota.org/) is using RocksDB in the [IOTA Reference Implementation (IRI)](https://github.com/iotaledger/iri) to store the local state of the Tangle. The Tangle is the first open-source distributed ledger powering the future of the Internet of Things.
|
||||
Vendored
-5
@@ -14,11 +14,6 @@ Vagrant.configure("2") do |config|
|
||||
box.vm.box = "chef/centos-6.5"
|
||||
end
|
||||
|
||||
config.vm.define "centos7" do |box|
|
||||
box.vm.box = "centos/7"
|
||||
box.vm.provision "shell", path: "build_tools/setup_centos7.sh"
|
||||
end
|
||||
|
||||
config.vm.define "FreeBSD10" do |box|
|
||||
box.vm.guest = :freebsd
|
||||
box.vm.box = "robin/freebsd-10"
|
||||
|
||||
+2
-2
@@ -43,9 +43,9 @@ We plan to use this port for our business purposes here at Bing and this provide
|
||||
|
||||
* Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`)
|
||||
* All posix specific headers were replaced to port/port.h which worked well
|
||||
* Replaced `dirent.h` for `port/port_dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
|
||||
* Replaced `dirent.h` for `port/dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
|
||||
* Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port`
|
||||
* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `zu` on posix systems and to `Iu` on windows.
|
||||
* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `%z` on posix systems and to Iu on windows.
|
||||
* in class member initialization were moved to a __ctors in some cases
|
||||
* `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file.
|
||||
* `constexpr` for functions was replaced to a template specialization (1 place)
|
||||
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
version: 1.0.{build}
|
||||
image: Visual Studio 2017
|
||||
image: Visual Studio 2015
|
||||
before_build:
|
||||
- md %APPVEYOR_BUILD_FOLDER%\build
|
||||
- cd %APPVEYOR_BUILD_FOLDER%\build
|
||||
- cmake -G "Visual Studio 15 Win64" -DOPTDBG=1 -DWITH_XPRESS=1 -DPORTABLE=1 -DJNI=1 ..
|
||||
- cmake -G "Visual Studio 14 Win64" -DOPTDBG=1 -DWITH_XPRESS=1 -DPORTABLE=1 ..
|
||||
- cd ..
|
||||
build:
|
||||
project: build\rocksdb.sln
|
||||
|
||||
@@ -3,11 +3,14 @@ from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
from targets_builder import TARGETSBuilder
|
||||
from optparse import OptionParser
|
||||
import os
|
||||
import fnmatch
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
from util import ColorString
|
||||
import util
|
||||
|
||||
# tests to export as libraries for inclusion in other projects
|
||||
_EXPORTED_TEST_LIBS = ["env_basic_test"]
|
||||
@@ -33,7 +36,7 @@ def parse_src_mk(repo_path):
|
||||
# get all .cc / .c files
|
||||
def get_cc_files(repo_path):
|
||||
cc_files = []
|
||||
for root, dirnames, filenames in os.walk(repo_path): # noqa: B007 T25377293 Grandfathered in
|
||||
for root, dirnames, filenames in os.walk(repo_path):
|
||||
root = root[(len(repo_path) + 1):]
|
||||
if "java" in root:
|
||||
# Skip java
|
||||
@@ -109,14 +112,12 @@ def generate_targets(repo_path):
|
||||
"rocksdb_test_lib",
|
||||
src_mk.get("MOCK_LIB_SOURCES", []) +
|
||||
src_mk.get("TEST_LIB_SOURCES", []) +
|
||||
src_mk.get("EXP_LIB_SOURCES", []) +
|
||||
src_mk.get("ANALYZER_LIB_SOURCES", []),
|
||||
src_mk.get("EXP_LIB_SOURCES", []),
|
||||
[":rocksdb_lib"])
|
||||
# rocksdb_tools_lib
|
||||
TARGETS.add_library(
|
||||
"rocksdb_tools_lib",
|
||||
src_mk.get("BENCH_LIB_SOURCES", []) +
|
||||
src_mk.get("ANALYZER_LIB_SOURCES", []) +
|
||||
["util/testutil.cc"],
|
||||
[":rocksdb_lib"])
|
||||
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
# Create a tmp directory for the test to use
|
||||
TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
|
||||
# shellcheck disable=SC2068
|
||||
TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"
|
||||
|
||||
@@ -10,7 +10,7 @@ def pretty_list(lst, indent=8):
|
||||
|
||||
if len(lst) == 1:
|
||||
return "\"%s\"" % lst[0]
|
||||
|
||||
|
||||
separator = "\",\n%s\"" % (" " * indent)
|
||||
res = separator.join(sorted(lst))
|
||||
res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 4))
|
||||
@@ -31,16 +31,13 @@ class TARGETSBuilder:
|
||||
self.targets_file.close()
|
||||
|
||||
def add_library(self, name, srcs, deps=None, headers=None):
|
||||
headers_attr_prefix = ""
|
||||
if headers is None:
|
||||
headers_attr_prefix = "auto_"
|
||||
headers = "AutoHeaders.RECURSIVE_GLOB"
|
||||
self.targets_file.write(targets_cfg.library_template.format(
|
||||
name=name,
|
||||
srcs=pretty_list(srcs),
|
||||
headers_attr_prefix=headers_attr_prefix,
|
||||
headers=headers,
|
||||
deps=pretty_list(deps)))
|
||||
self.targets_file.write(targets_cfg.library_template % (
|
||||
name,
|
||||
pretty_list(srcs),
|
||||
headers,
|
||||
pretty_list(deps)))
|
||||
self.total_lib = self.total_lib + 1
|
||||
|
||||
def add_binary(self, name, srcs, deps=None):
|
||||
|
||||
+58
-51
@@ -2,13 +2,13 @@ from __future__ import absolute_import
|
||||
from __future__ import division
|
||||
from __future__ import print_function
|
||||
from __future__ import unicode_literals
|
||||
rocksdb_target_header = """load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
|
||||
load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
|
||||
load(":defs.bzl", "test_binary")
|
||||
rocksdb_target_header = """REPO_PATH = package_name() + "/"
|
||||
|
||||
REPO_PATH = package_name() + "/"
|
||||
BUCK_BINS = "buck-out/gen/" + REPO_PATH
|
||||
|
||||
ROCKSDB_COMPILER_FLAGS = [
|
||||
TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
|
||||
|
||||
rocksdb_compiler_flags = [
|
||||
"-fno-builtin-memcmp",
|
||||
"-DROCKSDB_PLATFORM_POSIX",
|
||||
"-DROCKSDB_LIB_IO_POSIX",
|
||||
@@ -24,19 +24,14 @@ ROCKSDB_COMPILER_FLAGS = [
|
||||
"-DBZIP2",
|
||||
"-DLZ4",
|
||||
"-DZSTD",
|
||||
"-DZSTD_STATIC_LINKING_ONLY",
|
||||
"-DGFLAGS=gflags",
|
||||
"-DNUMA",
|
||||
"-DTBB",
|
||||
# Needed to compile in fbcode
|
||||
"-Wno-expansion-to-defined",
|
||||
# Added missing flags from output of build_detect_platform
|
||||
"-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
|
||||
"-DROCKSDB_BACKTRACE",
|
||||
"-Wnarrowing",
|
||||
]
|
||||
|
||||
ROCKSDB_EXTERNAL_DEPS = [
|
||||
rocksdb_external_deps = [
|
||||
("bzip2", None, "bz2"),
|
||||
("snappy", None, "snappy"),
|
||||
("zlib", None, "z"),
|
||||
@@ -48,17 +43,14 @@ ROCKSDB_EXTERNAL_DEPS = [
|
||||
("googletest", None, "gtest"),
|
||||
]
|
||||
|
||||
ROCKSDB_PREPROCESSOR_FLAGS = [
|
||||
rocksdb_preprocessor_flags = [
|
||||
# Directories with files for #include
|
||||
"-I" + REPO_PATH + "include/",
|
||||
"-I" + REPO_PATH,
|
||||
]
|
||||
|
||||
ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
|
||||
"x86_64": [
|
||||
"-DHAVE_SSE42",
|
||||
"-DHAVE_PCLMUL",
|
||||
],
|
||||
rocksdb_arch_preprocessor_flags = {
|
||||
"x86_64": ["-DHAVE_SSE42"],
|
||||
}
|
||||
|
||||
build_mode = read_config("fbcode", "build_mode")
|
||||
@@ -67,28 +59,21 @@ is_opt_mode = build_mode.startswith("opt")
|
||||
|
||||
# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
|
||||
# doesn't harm and avoid forgetting to add it.
|
||||
ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
|
||||
|
||||
sanitizer = read_config("fbcode", "sanitizer")
|
||||
|
||||
# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
|
||||
# whether the binary is linked with jemalloc at runtime.
|
||||
ROCKSDB_COMPILER_FLAGS += (["-DROCKSDB_JEMALLOC"] if sanitizer == "" else [])
|
||||
|
||||
ROCKSDB_EXTERNAL_DEPS += ([("jemalloc", None, "headers")] if sanitizer == "" else [])
|
||||
if is_opt_mode:
|
||||
rocksdb_compiler_flags.append("-DNDEBUG")
|
||||
"""
|
||||
|
||||
|
||||
library_template = """
|
||||
cpp_library(
|
||||
name = "{name}",
|
||||
srcs = [{srcs}],
|
||||
{headers_attr_prefix}headers = {headers},
|
||||
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
deps = [{deps}],
|
||||
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
name = "%s",
|
||||
srcs = [%s],
|
||||
headers = %s,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [%s],
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -96,11 +81,11 @@ binary_template = """
|
||||
cpp_binary(
|
||||
name = "%s",
|
||||
srcs = [%s],
|
||||
arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
deps = [%s],
|
||||
external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -119,17 +104,39 @@ ROCKS_TESTS = [
|
||||
# Generate a test rule for each entry in ROCKS_TESTS
|
||||
# Do not build the tests in opt mode, since SyncPoint and other test code
|
||||
# will not be included.
|
||||
[
|
||||
test_binary(
|
||||
parallelism = parallelism,
|
||||
rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
|
||||
rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
|
||||
rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
|
||||
rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
|
||||
test_cc = test_cc,
|
||||
test_name = test_name,
|
||||
)
|
||||
for test_name, test_cc, parallelism in ROCKS_TESTS
|
||||
if not is_opt_mode
|
||||
]
|
||||
if not is_opt_mode:
|
||||
for test_cfg in ROCKS_TESTS:
|
||||
test_name = test_cfg[0]
|
||||
test_cc = test_cfg[1]
|
||||
ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
|
||||
test_bin = test_name + "_bin"
|
||||
|
||||
cpp_binary (
|
||||
name = test_bin,
|
||||
srcs = [test_cc],
|
||||
deps = [":rocksdb_test_lib"],
|
||||
preprocessor_flags = rocksdb_preprocessor_flags,
|
||||
arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
|
||||
compiler_flags = rocksdb_compiler_flags,
|
||||
external_deps = rocksdb_external_deps,
|
||||
)
|
||||
|
||||
custom_unittest(
|
||||
name = test_name,
|
||||
type = ttype,
|
||||
deps = [":" + test_bin],
|
||||
command = [TEST_RUNNER, BUCK_BINS + test_bin]
|
||||
)
|
||||
|
||||
custom_unittest(
|
||||
name = "make_rocksdbjavastatic",
|
||||
command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
|
||||
type = "simple",
|
||||
)
|
||||
|
||||
custom_unittest(
|
||||
name = "make_rocksdb_lite_release",
|
||||
command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
|
||||
type = "simple",
|
||||
)
|
||||
"""
|
||||
|
||||
@@ -7,12 +7,12 @@
|
||||
// Name of the environment variables which need to be set by the entity which
|
||||
// triggers continuous runs so that code at the end of the file gets executed
|
||||
// and Sandcastle run starts.
|
||||
const ENV_POST_RECEIVE_HOOK = "POST_RECEIVE_HOOK";
|
||||
const ENV_HTTPS_APP_VALUE = "HTTPS_APP_VALUE";
|
||||
const ENV_HTTPS_TOKEN_VALUE = "HTTPS_TOKEN_VALUE";
|
||||
define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
|
||||
define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
|
||||
define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
|
||||
|
||||
const PRIMARY_TOKEN_FILE = '/home/krad/.sandcastle';
|
||||
const CONT_RUN_ALIAS = "leveldb";
|
||||
define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
|
||||
define("CONT_RUN_ALIAS", "leveldb");
|
||||
|
||||
//////////////////////////////////////////////////////////////////////
|
||||
/* Run tests in sandcastle */
|
||||
@@ -97,7 +97,7 @@ function getSteps($applyDiff, $diffID, $username, $test) {
|
||||
}
|
||||
|
||||
// fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
|
||||
// Git thinks it is an uncommitted change.
|
||||
// Git thinks it is an uncommited change.
|
||||
$fix_git_ignore = array(
|
||||
"name" => "Fix git ignore",
|
||||
"shell" => "echo fbcode >> .git/info/exclude",
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/sh
|
||||
#
|
||||
# Detects OS we're compiling on and outputs a file specified by the first
|
||||
# argument, which in turn gets read while processing Makefile.
|
||||
@@ -16,8 +16,6 @@
|
||||
# PLATFORM_CXXFLAGS C++ compiler flags. Will contain:
|
||||
# PLATFORM_SHARED_VERSIONED Set to 'true' if platform supports versioned
|
||||
# shared libraries, empty otherwise.
|
||||
# FIND Command for the find utility
|
||||
# WATCH Command for the watch utility
|
||||
#
|
||||
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
|
||||
#
|
||||
@@ -53,13 +51,11 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
|
||||
FBCODE_BUILD="true"
|
||||
# If we're compiling with TSAN we need pic build
|
||||
PIC_BUILD=$COMPILE_WITH_TSAN
|
||||
if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
||||
if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
else
|
||||
# we need this to build with MySQL. Don't use for other purposes.
|
||||
source "$PWD/build_tools/fbcode_config4.8.1.sh"
|
||||
elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
|
||||
source "$PWD/build_tools/fbcode_config_platform007.sh"
|
||||
else
|
||||
source "$PWD/build_tools/fbcode_config.sh"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -68,23 +64,11 @@ rm -f "$OUTPUT"
|
||||
touch "$OUTPUT"
|
||||
|
||||
if test -z "$CC"; then
|
||||
if [ -x "$(command -v cc)" ]; then
|
||||
CC=cc
|
||||
elif [ -x "$(command -v clang)" ]; then
|
||||
CC=clang
|
||||
else
|
||||
CC=cc
|
||||
fi
|
||||
CC=cc
|
||||
fi
|
||||
|
||||
if test -z "$CXX"; then
|
||||
if [ -x "$(command -v g++)" ]; then
|
||||
CXX=g++
|
||||
elif [ -x "$(command -v clang++)" ]; then
|
||||
CXX=clang++
|
||||
else
|
||||
CXX=g++
|
||||
fi
|
||||
CXX=g++
|
||||
fi
|
||||
|
||||
# Detect OS
|
||||
@@ -101,15 +85,7 @@ if test -z "$CLANG_SCAN_BUILD"; then
|
||||
fi
|
||||
|
||||
if test -z "$CLANG_ANALYZER"; then
|
||||
CLANG_ANALYZER=$(command -v clang++ 2> /dev/null)
|
||||
fi
|
||||
|
||||
if test -z "$FIND"; then
|
||||
FIND=find
|
||||
fi
|
||||
|
||||
if test -z "$WATCH"; then
|
||||
WATCH=watch
|
||||
CLANG_ANALYZER=$(which clang++ 2> /dev/null)
|
||||
fi
|
||||
|
||||
COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
|
||||
@@ -146,8 +122,6 @@ case "$TARGET_OS" in
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
|
||||
else
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
|
||||
fi
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
|
||||
# PORT_FILES=port/linux/linux_specific.cc
|
||||
@@ -180,12 +154,9 @@ case "$TARGET_OS" in
|
||||
;;
|
||||
OpenBSD)
|
||||
PLATFORM=OS_OPENBSD
|
||||
CXX=clang++
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
|
||||
# PORT_FILES=port/openbsd/openbsd_specific.cc
|
||||
FIND=gfind
|
||||
WATCH=gnuwatch
|
||||
;;
|
||||
DragonFly)
|
||||
PLATFORM=OS_DRAGONFLYBSD
|
||||
@@ -200,8 +171,6 @@ case "$TARGET_OS" in
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
|
||||
else
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
|
||||
fi
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
|
||||
# PORT_FILES=port/linux/linux_specific.cc
|
||||
@@ -234,7 +203,7 @@ else
|
||||
#include <linux/falloc.h>
|
||||
int main() {
|
||||
int fd = open("/dev/null", 0);
|
||||
fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
|
||||
fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
|
||||
}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
@@ -370,16 +339,6 @@ EOF
|
||||
# JEMALLOC can be enabled either using the flag (like here) or by
|
||||
# providing direct link to the jemalloc library
|
||||
WITH_JEMALLOC_FLAG=1
|
||||
# check for JEMALLOC installed with HomeBrew
|
||||
if [ "$PLATFORM" == "OS_MACOSX" ]; then
|
||||
if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then
|
||||
JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ')
|
||||
JEMALLOC_INCLUDE="-I/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/include"
|
||||
JEMALLOC_LIB="/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
|
||||
PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $JEMALLOC_LIB"
|
||||
JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS $JEMALLOC_LIB"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
|
||||
@@ -486,17 +445,6 @@ EOF
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
|
||||
fi
|
||||
fi
|
||||
|
||||
if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then
|
||||
# Test whether c++17 aligned-new is supported
|
||||
$CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o /dev/null 2>/dev/null <<EOF
|
||||
struct alignas(1024) t {int a;};
|
||||
int main() {}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
|
||||
@@ -518,8 +466,8 @@ if test "$USE_HDFS"; then
|
||||
echo "JAVA_HOME has to be set for HDFS usage."
|
||||
exit 1
|
||||
fi
|
||||
HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS -I$HADOOP_HOME/include"
|
||||
HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64 -L$HADOOP_HOME/lib/native"
|
||||
HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS"
|
||||
HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64"
|
||||
HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
|
||||
HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
|
||||
COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
|
||||
@@ -537,9 +485,7 @@ if test -z "$PORTABLE"; then
|
||||
elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
|
||||
# TODO: Handle this with approprite options.
|
||||
COMMON_FLAGS="$COMMON_FLAGS"
|
||||
elif [ "$TARGET_OS" == "IOS" ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS"
|
||||
elif [ "$TARGET_OS" != "AIX" ] && [ "$TARGET_OS" != "SunOS" ]; then
|
||||
elif [ "$TARGET_OS" != AIX ] && [ "$TARGET_OS" != SunOS ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -march=native "
|
||||
elif test "$USE_SSE"; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
|
||||
@@ -627,8 +573,6 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
|
||||
echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
|
||||
echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
|
||||
echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
|
||||
echo "FIND=$FIND" >> "$OUTPUT"
|
||||
echo "WATCH=$WATCH" >> "$OUTPUT"
|
||||
# This will enable some related identifiers for the preprocessor
|
||||
if test -n "$JEMALLOC"; then
|
||||
echo "JEMALLOC=1" >> "$OUTPUT"
|
||||
|
||||
@@ -13,12 +13,10 @@ error=0
|
||||
|
||||
function log {
|
||||
DATE=`date +%Y-%m-%d:%H:%M:%S`
|
||||
# shellcheck disable=SC2068
|
||||
echo $DATE $@
|
||||
}
|
||||
|
||||
function log_err {
|
||||
# shellcheck disable=SC2145
|
||||
log "ERROR: $@ Error code: $error."
|
||||
}
|
||||
|
||||
|
||||
+18
-18
@@ -1,18 +1,18 @@
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/112ec378fec7002ad3e09afde022e656049f7191/5.x/centos7-native/c447969
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/04999bdb3ce81a11073535dcb00b5e13dc1cbaf5/stable/centos7-native/c9f9104
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/92b0c8e5c8eecc71eb042594ce1ab3413799b385/5.x/gcc-5-glibc-2.23/339d858
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/3d8698d5973ba94f41620a80a67e4457fdf01e90/2.23/gcc-5-glibc-2.23/ca1d1c0
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/gcc-5-glibc-2.23/9bc6787
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/gcc-5-glibc-2.23/9bc6787
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/gcc-5-glibc-2.23/9bc6787
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/gcc-5-glibc-2.23/9bc6787
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/af6628a46758f1a15484a1760cd7294164bc5ba1/1.3.5/gcc-5-glibc-2.23/03859b5
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/gcc-5-glibc-2.23/9bc6787
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/b1a0e56c1e3e6929813a4331ade3a58ff083afbb/master/gcc-5-glibc-2.23/aa64d6b
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/gcc-5-glibc-2.23/9bc6787
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/gcc-5-glibc-2.23/b443de1
|
||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/gcc-5-glibc-2.23/9bc6787
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/55031de95a2b46c82948743419a603b3d6aefe28/2.29.1/centos7-native/da39a3e
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/gcc-5-glibc-2.23/9bc6787
|
||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.2.3/gcc-5-glibc-2.23/65372bd
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f20197cf3d4bd50339c9777aaa0b2ccadad9e2cb/2.23/gcc-5-glibc-2.23/ca1d1c0
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/6427ce8c7496e4ab06c2da81543b94c0de8be3d0/1.1.3/gcc-5-glibc-2.23/9bc6787
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/8f1e8b867d26efef93eac2fabbdb2e1d512665d7/1.2.8/gcc-5-glibc-2.23/9bc6787
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/70471c0571559fe0af7db6d7e8860b93a7eadfe1/1.0.6/gcc-5-glibc-2.23/9bc6787
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/453c89d6f0e68cdf1c151c769197fabedad9cac8/r131/gcc-5-glibc-2.23/9bc6787
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/00a40fa5f8bd2cd0622f2e868552793aef37ccf4/1.3.0/gcc-5-glibc-2.23/03859b5
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/47eef08f9acb77de982fbda6047c26d330739538/2.2.0/gcc-5-glibc-2.23/9bc6787
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/4414ddc78df8008b35cc4adac23590ad29148584/master/gcc-5-glibc-2.23/d506c82
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/9d7ae2693d05d62f9a579cb21e6b717cf257a75d/2.0.11/gcc-5-glibc-2.23/9bc6787
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/2b2dd58e3a52ccf2c1d827def59e5f740de0ad15/1.2/gcc-5-glibc-2.23/b443de1
|
||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/379addf7ab2468a2b4293b47456cfcd1c9cb318d/4.3/gcc-5-glibc-2.23/9bc6787
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/3f68f5fe65a85b7c2d3e66852268fbd1efdb3151/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b9fab0aec99d9c36408e810b2677e91c12807afd/2.28/centos6-native/da39a3e
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/423431d61786b20bcc3bde8972901130cb29e6b3/3.11.0/gcc-5-glibc-2.23/9bc6787
|
||||
LUA_BASE=/mnt/gvfs/third-party2/lua/3b0bb3bd9a0f690a069c479fcc0f7424fc7456d2/5.2.3/gcc-5-glibc-2.23/65372bd
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# shellcheck disable=SC2148
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
GCC_BASE=/mnt/gvfs/third-party2/gcc/6e8e715624fd15256a7970073387793dfcf79b46/7.x/centos7-native/b2ef2b6
|
||||
CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/ef37e1faa1c29782abfac1ae65a291b9b7966f6d/stable/centos7-native/c9f9104
|
||||
LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/c67031f0f739ac61575a061518d6ef5038f99f90/7.x/platform007/5620abc
|
||||
GLIBC_BASE=/mnt/gvfs/third-party2/glibc/60d6f124a78798b73944f5ba87c2306ae3460153/2.26/platform007/f259413
|
||||
SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
|
||||
ZLIB_BASE=/mnt/gvfs/third-party2/zlib/22c2d65676fb7c23cfa797c4f6937f38b026f3cf/1.2.8/platform007/ca4da3d
|
||||
BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
|
||||
LZ4_BASE=/mnt/gvfs/third-party2/lz4/907b498203d297947f3bb70b9466f47e100f1873/r131/platform007/ca4da3d
|
||||
ZSTD_BASE=/mnt/gvfs/third-party2/zstd/3ee276cbacfad3074e3f07bf826ac47f06970f4e/1.3.5/platform007/15a3614
|
||||
GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
|
||||
JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/9c910d36d6235cc40e8ff559358f1833452300ca/master/platform007/5b0f53e
|
||||
NUMA_BASE=/mnt/gvfs/third-party2/numa/9cbf2460284c669ed19c3ccb200a71f7dd7e53c7/2.0.11/platform007/ca4da3d
|
||||
LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/bf3d7497fe4e6d007354f0adffa16ce3003f8338/1.3/platform007/6f3e0a9
|
||||
TBB_BASE=/mnt/gvfs/third-party2/tbb/ff4e0b093534704d8abab678a4fd7f5ea7b094c7/2018_U5/platform007/ca4da3d
|
||||
KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/b5c4a61a5c483ba24722005ae07895971a2ac707/fb/platform007/da39a3e
|
||||
BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/92ff90349e2f43ea0a8246d8b1cf17b6869013e3/2.29.1/centos7-native/da39a3e
|
||||
VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/f3f697a28122e6bcd513273dd9c1ff23852fc59f/3.13.0/platform007/ca4da3d
|
||||
LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
|
||||
@@ -64,12 +64,8 @@ class MatchErrorParser(ErrorParserBase):
|
||||
|
||||
class CompilerErrorParser(MatchErrorParser):
|
||||
def __init__(self):
|
||||
# format (compile error):
|
||||
# '<filename>:<line #>:<column #>: error: <error msg>'
|
||||
# format (link error):
|
||||
# '<filename>:<line #>: error: <error msg>'
|
||||
# The below regex catches both
|
||||
super(CompilerErrorParser, self).__init__(r'\S+:\d+: error:')
|
||||
# format: '<filename>:<line #>:<column #>: error: <error msg>'
|
||||
super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:')
|
||||
|
||||
|
||||
class ScanBuildErrorParser(MatchErrorParser):
|
||||
@@ -132,14 +128,11 @@ _TEST_NAME_TO_PARSERS = {
|
||||
'lite': [CompilerErrorParser],
|
||||
'lite_test': [CompilerErrorParser, GTestErrorParser],
|
||||
'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
|
||||
'stress_crash_with_atomic_flush': [CompilerErrorParser, DbCrashErrorParser],
|
||||
'write_stress': [CompilerErrorParser, WriteStressErrorParser],
|
||||
'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
|
||||
'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
|
||||
'asan_crash_with_atomic_flush': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
|
||||
'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
|
||||
'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
|
||||
'ubsan_crash_with_atomic_flush': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
|
||||
'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
|
||||
'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
|
||||
'format_compatible': [CompilerErrorParser, CompatErrorParser],
|
||||
|
||||
@@ -43,15 +43,11 @@ if test -z $PIC_BUILD; then
|
||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
||||
CFLAGS+=" -DLZ4"
|
||||
fi
|
||||
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
else
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
|
||||
CFLAGS+=" -DZSTD"
|
||||
fi
|
||||
CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Set environment variables so that we can compile rocksdb using
|
||||
# fbcode settings. It uses the latest g++ and clang compilers and also
|
||||
# uses jemalloc
|
||||
# Environment variables that change the behavior of this script:
|
||||
# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
|
||||
|
||||
|
||||
BASEDIR=`dirname $BASH_SOURCE`
|
||||
source "$BASEDIR/dependencies_platform007.sh"
|
||||
|
||||
CFLAGS=""
|
||||
|
||||
# libgcc
|
||||
LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
|
||||
LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
|
||||
|
||||
# glibc
|
||||
GLIBC_INCLUDE="$GLIBC_BASE/include"
|
||||
GLIBC_LIBS=" -L $GLIBC_BASE/lib"
|
||||
|
||||
# snappy
|
||||
SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
|
||||
else
|
||||
SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DSNAPPY"
|
||||
|
||||
if test -z $PIC_BUILD; then
|
||||
# location of zlib headers and libraries
|
||||
ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
|
||||
ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
|
||||
CFLAGS+=" -DZLIB"
|
||||
|
||||
# location of bzip headers and libraries
|
||||
BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
|
||||
BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
|
||||
CFLAGS+=" -DBZIP2"
|
||||
|
||||
LZ4_INCLUDE=" -I $LZ4_BASE/include/"
|
||||
LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
|
||||
CFLAGS+=" -DLZ4"
|
||||
fi
|
||||
|
||||
ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
|
||||
else
|
||||
ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DZSTD"
|
||||
|
||||
# location of gflags headers and libraries
|
||||
GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
|
||||
else
|
||||
GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DGFLAGS=gflags"
|
||||
|
||||
# location of jemalloc
|
||||
JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
|
||||
JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
|
||||
|
||||
if test -z $PIC_BUILD; then
|
||||
# location of numa
|
||||
NUMA_INCLUDE=" -I $NUMA_BASE/include/"
|
||||
NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
|
||||
CFLAGS+=" -DNUMA"
|
||||
|
||||
# location of libunwind
|
||||
LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
|
||||
fi
|
||||
|
||||
# location of TBB
|
||||
TBB_INCLUDE=" -isystem $TBB_BASE/include/"
|
||||
if test -z $PIC_BUILD; then
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb.a"
|
||||
else
|
||||
TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
|
||||
fi
|
||||
CFLAGS+=" -DTBB"
|
||||
|
||||
# use Intel SSE support for checksum calculations
|
||||
export USE_SSE=1
|
||||
export PORTABLE=1
|
||||
|
||||
BINUTILS="$BINUTILS_BASE/bin"
|
||||
AR="$BINUTILS/ar"
|
||||
|
||||
DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
|
||||
|
||||
STDLIBS="-L $GCC_BASE/lib64"
|
||||
|
||||
CLANG_BIN="$CLANG_BASE/bin"
|
||||
CLANG_LIB="$CLANG_BASE/lib"
|
||||
CLANG_SRC="$CLANG_BASE/../../src"
|
||||
|
||||
CLANG_ANALYZER="$CLANG_BIN/clang++"
|
||||
CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
|
||||
|
||||
if [ -z "$USE_CLANG" ]; then
|
||||
# gcc
|
||||
CC="$GCC_BASE/bin/gcc"
|
||||
CXX="$GCC_BASE/bin/g++"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
JEMALLOC=1
|
||||
else
|
||||
# clang
|
||||
CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
|
||||
CC="$CLANG_BIN/clang"
|
||||
CXX="$CLANG_BIN/clang++"
|
||||
|
||||
KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
|
||||
|
||||
CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
|
||||
CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
|
||||
CFLAGS+=" -isystem $GLIBC_INCLUDE"
|
||||
CFLAGS+=" -isystem $LIBGCC_INCLUDE"
|
||||
CFLAGS+=" -isystem $CLANG_INCLUDE"
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
|
||||
CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
|
||||
CFLAGS+=" -Wno-expansion-to-defined "
|
||||
CXXFLAGS="-nostdinc++"
|
||||
fi
|
||||
|
||||
CFLAGS+=" $DEPS_INCLUDE"
|
||||
CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
|
||||
CXXFLAGS+=" $CFLAGS"
|
||||
|
||||
EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
|
||||
EXEC_LDFLAGS+=" -B$BINUTILS/gold"
|
||||
EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
|
||||
EXEC_LDFLAGS+=" $LIBUNWIND"
|
||||
EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
|
||||
# required by libtbb
|
||||
EXEC_LDFLAGS+=" -ldl"
|
||||
|
||||
PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
|
||||
|
||||
EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"
|
||||
|
||||
VALGRIND_VER="$VALGRIND_BASE/bin/"
|
||||
|
||||
# lua not supported because it's on track for deprecation, I think
|
||||
LUA_PATH=
|
||||
LUA_LIB=
|
||||
|
||||
export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
|
||||
@@ -5082,8 +5082,8 @@ sub openoutputfiles {
|
||||
# Set reading FD if using --group (--ungroup does not need)
|
||||
for my $fdno (1,2) {
|
||||
# Re-open the file for reading
|
||||
# so fdw can be closed separately
|
||||
# and fdr can be seeked separately (for --line-buffer)
|
||||
# so fdw can be closed seperately
|
||||
# and fdr can be seeked seperately (for --line-buffer)
|
||||
open(my $fdr,"<", $self->fh($fdno,'name')) ||
|
||||
::die_bug("fdr: Cannot open ".$self->fh($fdno,'name'));
|
||||
$self->set_fh($fdno,'r',$fdr);
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
# shellcheck disable=SC1113
|
||||
#/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
@@ -29,14 +28,12 @@ function package() {
|
||||
if dpkg --get-selections | grep --quiet $1; then
|
||||
log "$1 is already installed. skipping."
|
||||
else
|
||||
# shellcheck disable=SC2068
|
||||
apt-get install $@ -y
|
||||
fi
|
||||
elif [[ $OS = "centos" ]]; then
|
||||
if rpm -qa | grep --quiet $1; then
|
||||
log "$1 is already installed. skipping."
|
||||
else
|
||||
# shellcheck disable=SC2068
|
||||
yum install $@ -y
|
||||
fi
|
||||
fi
|
||||
@@ -55,7 +52,6 @@ function gem_install() {
|
||||
if gem list | grep --quiet $1; then
|
||||
log "$1 is already installed. skipping."
|
||||
else
|
||||
# shellcheck disable=SC2068
|
||||
gem install $@
|
||||
fi
|
||||
}
|
||||
@@ -129,5 +125,4 @@ function main() {
|
||||
include $LIB_DIR
|
||||
}
|
||||
|
||||
# shellcheck disable=SC2068
|
||||
main $@
|
||||
|
||||
@@ -85,12 +85,9 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
|
||||
GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
|
||||
ASAN="COMPILE_WITH_ASAN=1"
|
||||
CLANG="USE_CLANG=1"
|
||||
# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
|
||||
# using platform007 gives us gcc-8 or higher which has that bug fixed.
|
||||
TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
|
||||
LITE="OPT=\"-DROCKSDB_LITE -g\""
|
||||
TSAN="COMPILE_WITH_TSAN=1"
|
||||
UBSAN="COMPILE_WITH_UBSAN=1"
|
||||
TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"'
|
||||
NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd"
|
||||
DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
|
||||
HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
|
||||
SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
|
||||
@@ -346,7 +343,7 @@ LITE_BUILD_COMMANDS="[
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Build RocksDB debug version',
|
||||
'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
|
||||
'shell':'$LITE make J=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
@@ -355,22 +352,6 @@ LITE_BUILD_COMMANDS="[
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# Report RocksDB lite binary size to scuba
|
||||
REPORT_LITE_BINARY_SIZE_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb Lite Binary Size',
|
||||
'oncall':'$ONCALL',
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Report RocksDB Lite binary size to scuba',
|
||||
'shell':'tools/report_lite_binary_size.sh',
|
||||
'user':'root',
|
||||
},
|
||||
],
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB stress/crash test
|
||||
#
|
||||
@@ -383,43 +364,14 @@ STRESS_CRASH_TEST_COMMANDS="[
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Build and run RocksDB debug stress tests',
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
||||
'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
{
|
||||
'name':'Build and run RocksDB debug crash tests',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
}
|
||||
],
|
||||
$ARTIFACTS,
|
||||
$REPORT
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB stress/crash test with atomic flush
|
||||
#
|
||||
STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb Stress/Crash Test (atomic flush)',
|
||||
'oncall':'$ONCALL',
|
||||
'timeout': 86400,
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Build and run RocksDB debug stress tests',
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
{
|
||||
'name':'Build and run RocksDB debug crash tests with atomic flush',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
||||
'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
}
|
||||
@@ -484,29 +436,7 @@ ASAN_CRASH_TEST_COMMANDS="[
|
||||
{
|
||||
'name':'Build and run RocksDB debug asan_crash_test',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
],
|
||||
$REPORT
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB crash testing with atomic flush under address sanitizer
|
||||
#
|
||||
ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb crash test (atomic flush) under ASAN',
|
||||
'oncall':'$ONCALL',
|
||||
'timeout': 86400,
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
||||
'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
@@ -548,29 +478,7 @@ UBSAN_CRASH_TEST_COMMANDS="[
|
||||
{
|
||||
'name':'Build and run RocksDB debug ubsan_crash_test',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
],
|
||||
$REPORT
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB crash testing with atomic flush under undefined behavior sanitizer
|
||||
#
|
||||
UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb crash test (atomic flush) under UBSAN',
|
||||
'oncall':'$ONCALL',
|
||||
'timeout': 86400,
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush',
|
||||
'timeout': 86400,
|
||||
'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
||||
'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
@@ -636,29 +544,7 @@ TSAN_CRASH_TEST_COMMANDS="[
|
||||
{
|
||||
'name':'Compile and run',
|
||||
'timeout': 86400,
|
||||
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
],
|
||||
$REPORT
|
||||
}
|
||||
]"
|
||||
|
||||
#
|
||||
# RocksDB crash test with atomic flush under TSAN
|
||||
#
|
||||
TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
|
||||
{
|
||||
'name':'Rocksdb Crash Test with atomic flush under TSAN',
|
||||
'oncall':'$ONCALL',
|
||||
'timeout': 86400,
|
||||
'steps': [
|
||||
$CLEANUP_ENV,
|
||||
{
|
||||
'name':'Compile and run',
|
||||
'timeout': 86400,
|
||||
'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
|
||||
'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
|
||||
'user':'root',
|
||||
$PARSER
|
||||
},
|
||||
@@ -706,7 +592,7 @@ run_no_compression()
|
||||
rm -rf /dev/shm/rocksdb
|
||||
mkdir /dev/shm/rocksdb
|
||||
make clean
|
||||
cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
|
||||
cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
|
||||
mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
|
||||
cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
|
||||
mv .tmp.Makefile Makefile
|
||||
@@ -759,12 +645,12 @@ run_regression()
|
||||
|
||||
# === lite build ===
|
||||
make clean
|
||||
make LITE=1 -j$(nproc) static_lib
|
||||
OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
|
||||
send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
|
||||
strip librocksdb.a
|
||||
send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
|
||||
|
||||
make LITE=1 -j$(nproc) shared_lib
|
||||
OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
|
||||
send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
|
||||
strip `readlink -f librocksdb.so`
|
||||
send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
|
||||
@@ -842,15 +728,9 @@ case $1 in
|
||||
lite)
|
||||
echo $LITE_BUILD_COMMANDS
|
||||
;;
|
||||
report_lite_binary_size)
|
||||
echo $REPORT_LITE_BINARY_SIZE_COMMANDS
|
||||
;;
|
||||
stress_crash)
|
||||
echo $STRESS_CRASH_TEST_COMMANDS
|
||||
;;
|
||||
stress_crash_with_atomic_flush)
|
||||
echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
||||
;;
|
||||
write_stress)
|
||||
echo $WRITE_STRESS_COMMANDS
|
||||
;;
|
||||
@@ -860,18 +740,12 @@ case $1 in
|
||||
asan_crash)
|
||||
echo $ASAN_CRASH_TEST_COMMANDS
|
||||
;;
|
||||
asan_crash_with_atomic_flush)
|
||||
echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
||||
;;
|
||||
ubsan)
|
||||
echo $UBSAN_TEST_COMMANDS
|
||||
;;
|
||||
ubsan_crash)
|
||||
echo $UBSAN_CRASH_TEST_COMMANDS
|
||||
;;
|
||||
ubsan_crash_with_atomic_flush)
|
||||
echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
||||
;;
|
||||
valgrind)
|
||||
echo $VALGRIND_TEST_COMMANDS
|
||||
;;
|
||||
@@ -881,9 +755,6 @@ case $1 in
|
||||
tsan_crash)
|
||||
echo $TSAN_CRASH_TEST_COMMANDS
|
||||
;;
|
||||
tsan_crash_with_atomic_flush)
|
||||
echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
|
||||
;;
|
||||
format_compatible)
|
||||
echo $FORMAT_COMPATIBLE_COMMANDS
|
||||
;;
|
||||
|
||||
@@ -336,7 +336,7 @@ $InvokeTestAsync = {
|
||||
# Test limiting factor here
|
||||
[int]$count = 0
|
||||
# Overall status
|
||||
[bool]$script:success = $true;
|
||||
[bool]$success = $true;
|
||||
|
||||
function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
|
||||
{
|
||||
@@ -425,7 +425,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
|
||||
$log_content = @(Get-Content $log)
|
||||
|
||||
if($completed.State -ne "Completed") {
|
||||
$script:success = $false
|
||||
$success = $false
|
||||
Write-Warning $message
|
||||
$log_content | Write-Warning
|
||||
} else {
|
||||
@@ -449,7 +449,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
|
||||
}
|
||||
|
||||
if(!$pass_found) {
|
||||
$script:success = $false;
|
||||
$success = $false;
|
||||
Write-Warning $message
|
||||
$log_content | Write-Warning
|
||||
} else {
|
||||
@@ -473,7 +473,7 @@ New-TimeSpan -Start $StartDate -End $EndDate |
|
||||
}
|
||||
|
||||
|
||||
if(!$script:success) {
|
||||
if(!$success) {
|
||||
# This does not succeed killing off jobs quick
|
||||
# So we simply exit
|
||||
# Remove-Job -Job $jobs -Force
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
ROCKSDB_VERSION="5.10.3"
|
||||
ZSTD_VERSION="1.1.3"
|
||||
|
||||
echo "This script configures CentOS with everything needed to build and run RocksDB"
|
||||
|
||||
yum update -y && yum install epel-release -y
|
||||
|
||||
yum install -y \
|
||||
wget \
|
||||
gcc-c++ \
|
||||
snappy snappy-devel \
|
||||
zlib zlib-devel \
|
||||
bzip2 bzip2-devel \
|
||||
lz4-devel \
|
||||
libasan \
|
||||
gflags
|
||||
|
||||
mkdir -pv /usr/local/rocksdb-${ROCKSDB_VERSION}
|
||||
ln -sfT /usr/local/rocksdb-${ROCKSDB_VERSION} /usr/local/rocksdb
|
||||
|
||||
wget -qO /tmp/zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz
|
||||
wget -qO /tmp/rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
|
||||
|
||||
cd /tmp
|
||||
|
||||
tar xzvf zstd-${ZSTD_VERSION}.tar.gz
|
||||
tar xzvf rocksdb-${ROCKSDB_VERSION}.tar.gz -C /usr/local/
|
||||
|
||||
echo "Installing ZSTD..."
|
||||
pushd zstd-${ZSTD_VERSION}
|
||||
make && make install
|
||||
popd
|
||||
|
||||
echo "Compiling RocksDB..."
|
||||
cd /usr/local/rocksdb
|
||||
chown -R vagrant:vagrant /usr/local/rocksdb/
|
||||
sudo -u vagrant make static_lib
|
||||
cd examples/
|
||||
sudo -u vagrant make all
|
||||
sudo -u vagrant ./c_simple_example
|
||||
@@ -53,45 +53,6 @@ function get_lib_base()
|
||||
log_variable $__res_var
|
||||
}
|
||||
|
||||
###########################################################
|
||||
# platform007 dependencies #
|
||||
###########################################################
|
||||
|
||||
OUTPUT="$BASEDIR/dependencies_platform007.sh"
|
||||
|
||||
rm -f "$OUTPUT"
|
||||
touch "$OUTPUT"
|
||||
|
||||
echo "Writing dependencies to $OUTPUT"
|
||||
|
||||
# Compilers locations
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
|
||||
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
|
||||
|
||||
log_variable GCC_BASE
|
||||
log_variable CLANG_BASE
|
||||
|
||||
# Libraries locations
|
||||
get_lib_base libgcc 7.x platform007
|
||||
get_lib_base glibc 2.26 platform007
|
||||
get_lib_base snappy LATEST platform007
|
||||
get_lib_base zlib LATEST platform007
|
||||
get_lib_base bzip2 LATEST platform007
|
||||
get_lib_base lz4 LATEST platform007
|
||||
get_lib_base zstd LATEST platform007
|
||||
get_lib_base gflags LATEST platform007
|
||||
get_lib_base jemalloc LATEST platform007
|
||||
get_lib_base numa LATEST platform007
|
||||
get_lib_base libunwind LATEST platform007
|
||||
get_lib_base tbb LATEST platform007
|
||||
|
||||
get_lib_base kernel-headers fb platform007
|
||||
get_lib_base binutils LATEST centos7-native
|
||||
get_lib_base valgrind LATEST platform007
|
||||
get_lib_base lua 5.3.4 platform007
|
||||
|
||||
git diff $OUTPUT
|
||||
|
||||
###########################################################
|
||||
# 5.x dependencies #
|
||||
###########################################################
|
||||
@@ -104,8 +65,8 @@ touch "$OUTPUT"
|
||||
echo "Writing dependencies to $OUTPUT"
|
||||
|
||||
# Compilers locations
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos7-native/*/`
|
||||
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
|
||||
GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos6-native/*/`
|
||||
CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
|
||||
|
||||
log_variable GCC_BASE
|
||||
log_variable CLANG_BASE
|
||||
@@ -125,7 +86,7 @@ get_lib_base libunwind LATEST gcc-5-glibc-2.23
|
||||
get_lib_base tbb LATEST gcc-5-glibc-2.23
|
||||
|
||||
get_lib_base kernel-headers 4.0.9-36_fbk5_2933_gd092e3f gcc-5-glibc-2.23
|
||||
get_lib_base binutils LATEST centos7-native
|
||||
get_lib_base binutils LATEST centos6-native
|
||||
get_lib_base valgrind LATEST gcc-5-glibc-2.23
|
||||
get_lib_base lua 5.2.3 gcc-5-glibc-2.23
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/sh
|
||||
if [ "$#" = "0" ]; then
|
||||
echo "Usage: $0 major|minor|patch|full"
|
||||
exit 1
|
||||
|
||||
Vendored
+1
-1
@@ -52,7 +52,7 @@ namespace rocksdb {
|
||||
|
||||
class CacheBench;
|
||||
namespace {
|
||||
void deleter(const Slice& /*key*/, void* value) {
|
||||
void deleter(const Slice& key, void* value) {
|
||||
delete reinterpret_cast<char *>(value);
|
||||
}
|
||||
|
||||
|
||||
Vendored
+16
-16
@@ -40,9 +40,9 @@ static int DecodeValue(void* v) {
|
||||
const std::string kLRU = "lru";
|
||||
const std::string kClock = "clock";
|
||||
|
||||
void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
|
||||
void dumbDeleter(const Slice& key, void* value) {}
|
||||
|
||||
void eraseDeleter(const Slice& /*key*/, void* value) {
|
||||
void eraseDeleter(const Slice& key, void* value) {
|
||||
Cache* cache = reinterpret_cast<Cache*>(value);
|
||||
cache->Erase("foo");
|
||||
}
|
||||
@@ -64,8 +64,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
||||
|
||||
std::vector<int> deleted_keys_;
|
||||
std::vector<int> deleted_values_;
|
||||
std::shared_ptr<Cache> cache_;
|
||||
std::shared_ptr<Cache> cache2_;
|
||||
shared_ptr<Cache> cache_;
|
||||
shared_ptr<Cache> cache2_;
|
||||
|
||||
CacheTest()
|
||||
: cache_(NewCache(kCacheSize, kNumShardBits, false)),
|
||||
@@ -73,7 +73,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
||||
current_ = this;
|
||||
}
|
||||
|
||||
~CacheTest() override {}
|
||||
~CacheTest() {
|
||||
}
|
||||
|
||||
std::shared_ptr<Cache> NewCache(size_t capacity) {
|
||||
auto type = GetParam();
|
||||
@@ -98,7 +99,7 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
int Lookup(std::shared_ptr<Cache> cache, int key) {
|
||||
int Lookup(shared_ptr<Cache> cache, int key) {
|
||||
Cache::Handle* handle = cache->Lookup(EncodeKey(key));
|
||||
const int r = (handle == nullptr) ? -1 : DecodeValue(cache->Value(handle));
|
||||
if (handle != nullptr) {
|
||||
@@ -107,16 +108,16 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
||||
return r;
|
||||
}
|
||||
|
||||
void Insert(std::shared_ptr<Cache> cache, int key, int value,
|
||||
int charge = 1) {
|
||||
void Insert(shared_ptr<Cache> cache, int key, int value, int charge = 1) {
|
||||
cache->Insert(EncodeKey(key), EncodeValue(value), charge,
|
||||
&CacheTest::Deleter);
|
||||
}
|
||||
|
||||
void Erase(std::shared_ptr<Cache> cache, int key) {
|
||||
void Erase(shared_ptr<Cache> cache, int key) {
|
||||
cache->Erase(EncodeKey(key));
|
||||
}
|
||||
|
||||
|
||||
int Lookup(int key) {
|
||||
return Lookup(cache_, key);
|
||||
}
|
||||
@@ -144,7 +145,7 @@ class CacheTest : public testing::TestWithParam<std::string> {
|
||||
CacheTest* CacheTest::current_;
|
||||
|
||||
TEST_P(CacheTest, UsageTest) {
|
||||
// cache is std::shared_ptr and will be automatically cleaned up.
|
||||
// cache is shared_ptr and will be automatically cleaned up.
|
||||
const uint64_t kCapacity = 100000;
|
||||
auto cache = NewCache(kCapacity, 8, false);
|
||||
|
||||
@@ -172,7 +173,7 @@ TEST_P(CacheTest, UsageTest) {
|
||||
}
|
||||
|
||||
TEST_P(CacheTest, PinnedUsageTest) {
|
||||
// cache is std::shared_ptr and will be automatically cleaned up.
|
||||
// cache is shared_ptr and will be automatically cleaned up.
|
||||
const uint64_t kCapacity = 100000;
|
||||
auto cache = NewCache(kCapacity, 8, false);
|
||||
|
||||
@@ -306,7 +307,7 @@ TEST_P(CacheTest, EvictionPolicy) {
|
||||
Insert(200, 201);
|
||||
|
||||
// Frequently used entry must be kept around
|
||||
for (int i = 0; i < kCacheSize + 200; i++) {
|
||||
for (int i = 0; i < kCacheSize + 100; i++) {
|
||||
Insert(1000+i, 2000+i);
|
||||
ASSERT_EQ(101, Lookup(100));
|
||||
}
|
||||
@@ -359,7 +360,7 @@ TEST_P(CacheTest, EvictionPolicyRef) {
|
||||
Insert(303, 104);
|
||||
|
||||
// Insert entries much more than Cache capacity
|
||||
for (int i = 0; i < kCacheSize + 200; i++) {
|
||||
for (int i = 0; i < kCacheSize + 100; i++) {
|
||||
Insert(1000 + i, 2000 + i);
|
||||
}
|
||||
|
||||
@@ -469,7 +470,7 @@ class Value {
|
||||
};
|
||||
|
||||
namespace {
|
||||
void deleter(const Slice& /*key*/, void* value) {
|
||||
void deleter(const Slice& key, void* value) {
|
||||
delete static_cast<Value *>(value);
|
||||
}
|
||||
} // namespace
|
||||
@@ -687,8 +688,7 @@ TEST_P(CacheTest, DefaultShardBits) {
|
||||
}
|
||||
|
||||
#ifdef SUPPORT_CLOCK_CACHE
|
||||
std::shared_ptr<Cache> (*new_clock_cache_func)(size_t, int,
|
||||
bool) = NewClockCache;
|
||||
shared_ptr<Cache> (*new_clock_cache_func)(size_t, int, bool) = NewClockCache;
|
||||
INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
|
||||
testing::Values(kLRU, kClock));
|
||||
#else
|
||||
|
||||
Vendored
+32
-31
@@ -13,8 +13,8 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
std::shared_ptr<Cache> NewClockCache(size_t /*capacity*/, int /*num_shard_bits*/,
|
||||
bool /*strict_capacity_limit*/) {
|
||||
std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
|
||||
bool strict_capacity_limit) {
|
||||
// Clock cache not supported.
|
||||
return nullptr;
|
||||
}
|
||||
@@ -234,35 +234,38 @@ struct CleanupContext {
|
||||
};
|
||||
|
||||
// A cache shard which maintains its own CLOCK cache.
|
||||
class ClockCacheShard final : public CacheShard {
|
||||
class ClockCacheShard : public CacheShard {
|
||||
public:
|
||||
// Hash map type.
|
||||
typedef tbb::concurrent_hash_map<CacheKey, CacheHandle*, CacheKey> HashTable;
|
||||
|
||||
ClockCacheShard();
|
||||
~ClockCacheShard() override;
|
||||
~ClockCacheShard();
|
||||
|
||||
// Interfaces
|
||||
void SetCapacity(size_t capacity) override;
|
||||
void SetStrictCapacityLimit(bool strict_capacity_limit) override;
|
||||
Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
|
||||
void (*deleter)(const Slice& key, void* value),
|
||||
Cache::Handle** handle, Cache::Priority priority) override;
|
||||
Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
|
||||
virtual void SetCapacity(size_t capacity) override;
|
||||
virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
|
||||
virtual Status Insert(const Slice& key, uint32_t hash, void* value,
|
||||
size_t charge,
|
||||
void (*deleter)(const Slice& key, void* value),
|
||||
Cache::Handle** handle,
|
||||
Cache::Priority priority) override;
|
||||
virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
|
||||
// If the entry in in cache, increase reference count and return true.
|
||||
// Return false otherwise.
|
||||
//
|
||||
// Not necessary to hold mutex_ before being called.
|
||||
bool Ref(Cache::Handle* handle) override;
|
||||
bool Release(Cache::Handle* handle, bool force_erase = false) override;
|
||||
void Erase(const Slice& key, uint32_t hash) override;
|
||||
virtual bool Ref(Cache::Handle* handle) override;
|
||||
virtual bool Release(Cache::Handle* handle,
|
||||
bool force_erase = false) override;
|
||||
virtual void Erase(const Slice& key, uint32_t hash) override;
|
||||
bool EraseAndConfirm(const Slice& key, uint32_t hash,
|
||||
CleanupContext* context);
|
||||
size_t GetUsage() const override;
|
||||
size_t GetPinnedUsage() const override;
|
||||
void EraseUnRefEntries() override;
|
||||
void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
|
||||
bool thread_safe) override;
|
||||
virtual size_t GetUsage() const override;
|
||||
virtual size_t GetPinnedUsage() const override;
|
||||
virtual void EraseUnRefEntries() override;
|
||||
virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
|
||||
bool thread_safe) override;
|
||||
|
||||
private:
|
||||
static const uint32_t kInCacheBit = 1;
|
||||
@@ -364,9 +367,7 @@ ClockCacheShard::~ClockCacheShard() {
|
||||
for (auto& handle : list_) {
|
||||
uint32_t flags = handle.flags.load(std::memory_order_relaxed);
|
||||
if (InCache(flags) || CountRefs(flags) > 0) {
|
||||
if (handle.deleter != nullptr) {
|
||||
(*handle.deleter)(handle.key, handle.value);
|
||||
}
|
||||
(*handle.deleter)(handle.key, handle.value);
|
||||
delete[] handle.key.data();
|
||||
}
|
||||
}
|
||||
@@ -585,7 +586,7 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
|
||||
size_t charge,
|
||||
void (*deleter)(const Slice& key, void* value),
|
||||
Cache::Handle** out_handle,
|
||||
Cache::Priority /*priority*/) {
|
||||
Cache::Priority priority) {
|
||||
CleanupContext context;
|
||||
HashTable::accessor accessor;
|
||||
char* key_data = new char[key.size()];
|
||||
@@ -672,7 +673,7 @@ void ClockCacheShard::EraseUnRefEntries() {
|
||||
Cleanup(context);
|
||||
}
|
||||
|
||||
class ClockCache final : public ShardedCache {
|
||||
class ClockCache : public ShardedCache {
|
||||
public:
|
||||
ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit)
|
||||
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
|
||||
@@ -682,31 +683,31 @@ class ClockCache final : public ShardedCache {
|
||||
SetStrictCapacityLimit(strict_capacity_limit);
|
||||
}
|
||||
|
||||
~ClockCache() override { delete[] shards_; }
|
||||
virtual ~ClockCache() { delete[] shards_; }
|
||||
|
||||
const char* Name() const override { return "ClockCache"; }
|
||||
virtual const char* Name() const override { return "ClockCache"; }
|
||||
|
||||
CacheShard* GetShard(int shard) override {
|
||||
virtual CacheShard* GetShard(int shard) override {
|
||||
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
||||
}
|
||||
|
||||
const CacheShard* GetShard(int shard) const override {
|
||||
virtual const CacheShard* GetShard(int shard) const override {
|
||||
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
||||
}
|
||||
|
||||
void* Value(Handle* handle) override {
|
||||
virtual void* Value(Handle* handle) override {
|
||||
return reinterpret_cast<const CacheHandle*>(handle)->value;
|
||||
}
|
||||
|
||||
size_t GetCharge(Handle* handle) const override {
|
||||
virtual size_t GetCharge(Handle* handle) const override {
|
||||
return reinterpret_cast<const CacheHandle*>(handle)->charge;
|
||||
}
|
||||
|
||||
uint32_t GetHash(Handle* handle) const override {
|
||||
virtual uint32_t GetHash(Handle* handle) const override {
|
||||
return reinterpret_cast<const CacheHandle*>(handle)->hash;
|
||||
}
|
||||
|
||||
void DisownData() override { shards_ = nullptr; }
|
||||
virtual void DisownData() override { shards_ = nullptr; }
|
||||
|
||||
private:
|
||||
ClockCacheShard* shards_;
|
||||
|
||||
Vendored
+33
-53
@@ -99,22 +99,14 @@ void LRUHandleTable::Resize() {
|
||||
length_ = new_length;
|
||||
}
|
||||
|
||||
LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
|
||||
double high_pri_pool_ratio,
|
||||
bool use_adaptive_mutex)
|
||||
: capacity_(0),
|
||||
high_pri_pool_usage_(0),
|
||||
strict_capacity_limit_(strict_capacity_limit),
|
||||
high_pri_pool_ratio_(high_pri_pool_ratio),
|
||||
high_pri_pool_capacity_(0),
|
||||
usage_(0),
|
||||
lru_usage_(0),
|
||||
mutex_(use_adaptive_mutex) {
|
||||
LRUCacheShard::LRUCacheShard()
|
||||
: capacity_(0), high_pri_pool_usage_(0), strict_capacity_limit_(false),
|
||||
high_pri_pool_ratio_(0), high_pri_pool_capacity_(0), usage_(0),
|
||||
lru_usage_(0) {
|
||||
// Make empty circular linked list
|
||||
lru_.next = &lru_;
|
||||
lru_.prev = &lru_;
|
||||
lru_low_pri_ = &lru_;
|
||||
SetCapacity(capacity);
|
||||
}
|
||||
|
||||
LRUCacheShard::~LRUCacheShard() {}
|
||||
@@ -201,7 +193,7 @@ void LRUCacheShard::LRU_Remove(LRUHandle* e) {
|
||||
void LRUCacheShard::LRU_Insert(LRUHandle* e) {
|
||||
assert(e->next == nullptr);
|
||||
assert(e->prev == nullptr);
|
||||
if (high_pri_pool_ratio_ > 0 && (e->IsHighPri() || e->HasHit())) {
|
||||
if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
|
||||
// Inset "e" to head of LRU list.
|
||||
e->next = &lru_;
|
||||
e->prev = lru_.prev;
|
||||
@@ -248,6 +240,22 @@ void LRUCacheShard::EvictFromLRU(size_t charge,
|
||||
}
|
||||
}
|
||||
|
||||
void* LRUCacheShard::operator new(size_t size) {
|
||||
return port::cacheline_aligned_alloc(size);
|
||||
}
|
||||
|
||||
void* LRUCacheShard::operator new[](size_t size) {
|
||||
return port::cacheline_aligned_alloc(size);
|
||||
}
|
||||
|
||||
void LRUCacheShard::operator delete(void *memblock) {
|
||||
port::cacheline_aligned_free(memblock);
|
||||
}
|
||||
|
||||
void LRUCacheShard::operator delete[](void* memblock) {
|
||||
port::cacheline_aligned_free(memblock);
|
||||
}
|
||||
|
||||
void LRUCacheShard::SetCapacity(size_t capacity) {
|
||||
autovector<LRUHandle*> last_reference_list;
|
||||
{
|
||||
@@ -277,7 +285,6 @@ Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
|
||||
LRU_Remove(e);
|
||||
}
|
||||
e->refs++;
|
||||
e->SetHit();
|
||||
}
|
||||
return reinterpret_cast<Cache::Handle*>(e);
|
||||
}
|
||||
@@ -463,31 +470,18 @@ std::string LRUCacheShard::GetPrintableOptions() const {
|
||||
}
|
||||
|
||||
LRUCache::LRUCache(size_t capacity, int num_shard_bits,
|
||||
bool strict_capacity_limit, double high_pri_pool_ratio,
|
||||
std::shared_ptr<MemoryAllocator> allocator,
|
||||
bool use_adaptive_mutex)
|
||||
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
|
||||
std::move(allocator)) {
|
||||
bool strict_capacity_limit, double high_pri_pool_ratio)
|
||||
: ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
|
||||
num_shards_ = 1 << num_shard_bits;
|
||||
shards_ = reinterpret_cast<LRUCacheShard*>(
|
||||
port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
|
||||
size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
|
||||
shards_ = new LRUCacheShard[num_shards_];
|
||||
SetCapacity(capacity);
|
||||
SetStrictCapacityLimit(strict_capacity_limit);
|
||||
for (int i = 0; i < num_shards_; i++) {
|
||||
new (&shards_[i])
|
||||
LRUCacheShard(per_shard, strict_capacity_limit, high_pri_pool_ratio,
|
||||
use_adaptive_mutex);
|
||||
shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
|
||||
}
|
||||
}
|
||||
|
||||
LRUCache::~LRUCache() {
|
||||
if (shards_ != nullptr) {
|
||||
assert(num_shards_ > 0);
|
||||
for (int i = 0; i < num_shards_; i++) {
|
||||
shards_[i].~LRUCacheShard();
|
||||
}
|
||||
port::cacheline_aligned_free(shards_);
|
||||
}
|
||||
}
|
||||
LRUCache::~LRUCache() { delete[] shards_; }
|
||||
|
||||
CacheShard* LRUCache::GetShard(int shard) {
|
||||
return reinterpret_cast<CacheShard*>(&shards_[shard]);
|
||||
@@ -511,17 +505,9 @@ uint32_t LRUCache::GetHash(Handle* handle) const {
|
||||
|
||||
void LRUCache::DisownData() {
|
||||
// Do not drop data if compile with ASAN to suppress leak warning.
|
||||
#if defined(__clang__)
|
||||
#if !defined(__has_feature) || !__has_feature(address_sanitizer)
|
||||
shards_ = nullptr;
|
||||
num_shards_ = 0;
|
||||
#endif
|
||||
#else // __clang__
|
||||
#ifndef __SANITIZE_ADDRESS__
|
||||
shards_ = nullptr;
|
||||
num_shards_ = 0;
|
||||
#endif // !__SANITIZE_ADDRESS__
|
||||
#endif // __clang__
|
||||
}
|
||||
|
||||
size_t LRUCache::TEST_GetLRUSize() {
|
||||
@@ -543,16 +529,12 @@ double LRUCache::GetHighPriPoolRatio() {
|
||||
std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
|
||||
return NewLRUCache(cache_opts.capacity, cache_opts.num_shard_bits,
|
||||
cache_opts.strict_capacity_limit,
|
||||
cache_opts.high_pri_pool_ratio,
|
||||
cache_opts.memory_allocator,
|
||||
cache_opts.use_adaptive_mutex);
|
||||
cache_opts.high_pri_pool_ratio);
|
||||
}
|
||||
|
||||
std::shared_ptr<Cache> NewLRUCache(
|
||||
size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
||||
double high_pri_pool_ratio,
|
||||
std::shared_ptr<MemoryAllocator> memory_allocator,
|
||||
bool use_adaptive_mutex) {
|
||||
std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
|
||||
bool strict_capacity_limit,
|
||||
double high_pri_pool_ratio) {
|
||||
if (num_shard_bits >= 20) {
|
||||
return nullptr; // the cache cannot be sharded into too many fine pieces
|
||||
}
|
||||
@@ -564,9 +546,7 @@ std::shared_ptr<Cache> NewLRUCache(
|
||||
num_shard_bits = GetDefaultCacheShardBits(capacity);
|
||||
}
|
||||
return std::make_shared<LRUCache>(capacity, num_shard_bits,
|
||||
strict_capacity_limit, high_pri_pool_ratio,
|
||||
std::move(memory_allocator),
|
||||
use_adaptive_mutex);
|
||||
strict_capacity_limit, high_pri_pool_ratio);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
Vendored
+27
-36
@@ -55,18 +55,10 @@ struct LRUHandle {
|
||||
// cache itself is counted as 1
|
||||
|
||||
// Include the following flags:
|
||||
// IN_CACHE: whether this entry is referenced by the hash table.
|
||||
// IS_HIGH_PRI: whether this entry is high priority entry.
|
||||
// IN_HIGH_PRI_POOL: whether this entry is in high-pri pool.
|
||||
// HAS_HIT: whether this entry has had any lookups (hits).
|
||||
enum Flags : uint8_t {
|
||||
IN_CACHE = (1 << 0),
|
||||
IS_HIGH_PRI = (1 << 1),
|
||||
IN_HIGH_PRI_POOL = (1 << 2),
|
||||
HAS_HIT = (1 << 3),
|
||||
};
|
||||
|
||||
uint8_t flags;
|
||||
// in_cache: whether this entry is referenced by the hash table.
|
||||
// is_high_pri: whether this entry is high priority entry.
|
||||
// in_high_pro_pool: whether this entry is in high-pri pool.
|
||||
char flags;
|
||||
|
||||
uint32_t hash; // Hash of key(); used for fast sharding and comparisons
|
||||
|
||||
@@ -82,37 +74,34 @@ struct LRUHandle {
|
||||
}
|
||||
}
|
||||
|
||||
bool InCache() const { return flags & IN_CACHE; }
|
||||
bool IsHighPri() const { return flags & IS_HIGH_PRI; }
|
||||
bool InHighPriPool() const { return flags & IN_HIGH_PRI_POOL; }
|
||||
bool HasHit() const { return flags & HAS_HIT; }
|
||||
bool InCache() { return flags & 1; }
|
||||
bool IsHighPri() { return flags & 2; }
|
||||
bool InHighPriPool() { return flags & 4; }
|
||||
|
||||
void SetInCache(bool in_cache) {
|
||||
if (in_cache) {
|
||||
flags |= IN_CACHE;
|
||||
flags |= 1;
|
||||
} else {
|
||||
flags &= ~IN_CACHE;
|
||||
flags &= ~1;
|
||||
}
|
||||
}
|
||||
|
||||
void SetPriority(Cache::Priority priority) {
|
||||
if (priority == Cache::Priority::HIGH) {
|
||||
flags |= IS_HIGH_PRI;
|
||||
flags |= 2;
|
||||
} else {
|
||||
flags &= ~IS_HIGH_PRI;
|
||||
flags &= ~2;
|
||||
}
|
||||
}
|
||||
|
||||
void SetInHighPriPool(bool in_high_pri_pool) {
|
||||
if (in_high_pri_pool) {
|
||||
flags |= IN_HIGH_PRI_POOL;
|
||||
flags |= 4;
|
||||
} else {
|
||||
flags &= ~IN_HIGH_PRI_POOL;
|
||||
flags &= ~4;
|
||||
}
|
||||
}
|
||||
|
||||
void SetHit() { flags |= HAS_HIT; }
|
||||
|
||||
void Free() {
|
||||
assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
|
||||
if (deleter) {
|
||||
@@ -165,10 +154,9 @@ class LRUHandleTable {
|
||||
};
|
||||
|
||||
// A single shard of sharded cache.
|
||||
class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
|
||||
class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard : public CacheShard {
|
||||
public:
|
||||
LRUCacheShard(size_t capacity, bool strict_capacity_limit,
|
||||
double high_pri_pool_ratio, bool use_adaptive_mutex);
|
||||
LRUCacheShard();
|
||||
virtual ~LRUCacheShard();
|
||||
|
||||
// Separate from constructor so caller can easily make an array of LRUCache
|
||||
@@ -217,6 +205,15 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
|
||||
// Retrives high pri pool ratio
|
||||
double GetHighPriPoolRatio();
|
||||
|
||||
// Overloading to aligned it to cache line size
|
||||
void* operator new(size_t);
|
||||
|
||||
void* operator new[](size_t);
|
||||
|
||||
void operator delete(void *);
|
||||
|
||||
void operator delete[](void*);
|
||||
|
||||
private:
|
||||
void LRU_Remove(LRUHandle* e);
|
||||
void LRU_Insert(LRUHandle* e);
|
||||
@@ -284,16 +281,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
|
||||
mutable port::Mutex mutex_;
|
||||
};
|
||||
|
||||
class LRUCache
|
||||
#ifdef NDEBUG
|
||||
final
|
||||
#endif
|
||||
: public ShardedCache {
|
||||
class LRUCache : public ShardedCache {
|
||||
public:
|
||||
LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
||||
double high_pri_pool_ratio,
|
||||
std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
|
||||
bool use_adaptive_mutex = kDefaultToAdaptiveMutex);
|
||||
double high_pri_pool_ratio);
|
||||
virtual ~LRUCache();
|
||||
virtual const char* Name() const override { return "LRUCache"; }
|
||||
virtual CacheShard* GetShard(int shard) override;
|
||||
@@ -309,7 +300,7 @@ class LRUCache
|
||||
double GetHighPriPoolRatio();
|
||||
|
||||
private:
|
||||
LRUCacheShard* shards_ = nullptr;
|
||||
LRUCacheShard* shards_;
|
||||
int num_shards_ = 0;
|
||||
};
|
||||
|
||||
|
||||
Vendored
+22
-47
@@ -7,7 +7,6 @@
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "port/port.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace rocksdb {
|
||||
@@ -15,23 +14,22 @@ namespace rocksdb {
|
||||
class LRUCacheTest : public testing::Test {
|
||||
public:
|
||||
LRUCacheTest() {}
|
||||
~LRUCacheTest() override { DeleteCache(); }
|
||||
~LRUCacheTest() {}
|
||||
|
||||
void DeleteCache() {
|
||||
if (cache_ != nullptr) {
|
||||
cache_->~LRUCacheShard();
|
||||
port::cacheline_aligned_free(cache_);
|
||||
cache_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0,
|
||||
bool use_adaptive_mutex = kDefaultToAdaptiveMutex) {
|
||||
DeleteCache();
|
||||
cache_ = reinterpret_cast<LRUCacheShard*>(
|
||||
port::cacheline_aligned_alloc(sizeof(LRUCacheShard)));
|
||||
new (cache_) LRUCacheShard(capacity, false /*strict_capcity_limit*/,
|
||||
high_pri_pool_ratio, use_adaptive_mutex);
|
||||
void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
|
||||
cache_.reset(
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(push)
|
||||
#pragma warning(disable: 4316) // We've validated the alignment with the new operators
|
||||
#endif
|
||||
new LRUCacheShard()
|
||||
#if defined(_MSC_VER)
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
);
|
||||
cache_->SetCapacity(capacity);
|
||||
cache_->SetStrictCapacityLimit(false);
|
||||
cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
|
||||
}
|
||||
|
||||
void Insert(const std::string& key,
|
||||
@@ -87,7 +85,7 @@ class LRUCacheTest : public testing::Test {
|
||||
}
|
||||
|
||||
private:
|
||||
LRUCacheShard* cache_ = nullptr;
|
||||
std::unique_ptr<LRUCacheShard> cache_;
|
||||
};
|
||||
|
||||
TEST_F(LRUCacheTest, BasicLRU) {
|
||||
@@ -116,30 +114,7 @@ TEST_F(LRUCacheTest, BasicLRU) {
|
||||
ValidateLRUList({"e", "z", "d", "u", "v"});
|
||||
}
|
||||
|
||||
TEST_F(LRUCacheTest, MidpointInsertion) {
|
||||
// Allocate 2 cache entries to high-pri pool.
|
||||
NewCache(5, 0.45);
|
||||
|
||||
Insert("a", Cache::Priority::LOW);
|
||||
Insert("b", Cache::Priority::LOW);
|
||||
Insert("c", Cache::Priority::LOW);
|
||||
Insert("x", Cache::Priority::HIGH);
|
||||
Insert("y", Cache::Priority::HIGH);
|
||||
ValidateLRUList({"a", "b", "c", "x", "y"}, 2);
|
||||
|
||||
// Low-pri entries inserted to the tail of low-pri list (the midpoint).
|
||||
// After lookup, it will move to the tail of the full list.
|
||||
Insert("d", Cache::Priority::LOW);
|
||||
ValidateLRUList({"b", "c", "d", "x", "y"}, 2);
|
||||
ASSERT_TRUE(Lookup("d"));
|
||||
ValidateLRUList({"b", "c", "x", "y", "d"}, 2);
|
||||
|
||||
// High-pri entries will be inserted to the tail of full list.
|
||||
Insert("z", Cache::Priority::HIGH);
|
||||
ValidateLRUList({"c", "x", "y", "d", "z"}, 2);
|
||||
}
|
||||
|
||||
TEST_F(LRUCacheTest, EntriesWithPriority) {
|
||||
TEST_F(LRUCacheTest, MidPointInsertion) {
|
||||
// Allocate 2 cache entries to high-pri pool.
|
||||
NewCache(5, 0.45);
|
||||
|
||||
@@ -165,15 +140,15 @@ TEST_F(LRUCacheTest, EntriesWithPriority) {
|
||||
Insert("a", Cache::Priority::LOW);
|
||||
ValidateLRUList({"v", "X", "a", "Y", "Z"}, 2);
|
||||
|
||||
// Low-pri entries will be inserted to head of high-pri pool after lookup.
|
||||
// Low-pri entries will be inserted to head of low-pri pool after lookup.
|
||||
ASSERT_TRUE(Lookup("v"));
|
||||
ValidateLRUList({"X", "a", "Y", "Z", "v"}, 2);
|
||||
ValidateLRUList({"X", "a", "v", "Y", "Z"}, 2);
|
||||
|
||||
// High-pri entries will be inserted to the head of the list after lookup.
|
||||
ASSERT_TRUE(Lookup("X"));
|
||||
ValidateLRUList({"a", "Y", "Z", "v", "X"}, 2);
|
||||
ValidateLRUList({"a", "v", "Y", "Z", "X"}, 2);
|
||||
ASSERT_TRUE(Lookup("Z"));
|
||||
ValidateLRUList({"a", "Y", "v", "X", "Z"}, 2);
|
||||
ValidateLRUList({"a", "v", "Y", "X", "Z"}, 2);
|
||||
|
||||
Erase("Y");
|
||||
ValidateLRUList({"a", "v", "X", "Z"}, 2);
|
||||
@@ -186,7 +161,7 @@ TEST_F(LRUCacheTest, EntriesWithPriority) {
|
||||
Insert("g", Cache::Priority::LOW);
|
||||
ValidateLRUList({"d", "e", "f", "g", "Z"}, 1);
|
||||
ASSERT_TRUE(Lookup("d"));
|
||||
ValidateLRUList({"e", "f", "g", "Z", "d"}, 2);
|
||||
ValidateLRUList({"e", "f", "g", "d", "Z"}, 1);
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
Vendored
+3
-8
@@ -20,10 +20,8 @@
|
||||
namespace rocksdb {
|
||||
|
||||
ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
|
||||
bool strict_capacity_limit,
|
||||
std::shared_ptr<MemoryAllocator> allocator)
|
||||
: Cache(std::move(allocator)),
|
||||
num_shard_bits_(num_shard_bits),
|
||||
bool strict_capacity_limit)
|
||||
: num_shard_bits_(num_shard_bits),
|
||||
capacity_(capacity),
|
||||
strict_capacity_limit_(strict_capacity_limit),
|
||||
last_id_(1) {}
|
||||
@@ -55,7 +53,7 @@ Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
|
||||
->Insert(key, hash, value, charge, deleter, handle, priority);
|
||||
}
|
||||
|
||||
Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
|
||||
Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
|
||||
uint32_t hash = HashSlice(key);
|
||||
return GetShard(Shard(hash))->Lookup(key, hash);
|
||||
}
|
||||
@@ -144,9 +142,6 @@ std::string ShardedCache::GetPrintableOptions() const {
|
||||
strict_capacity_limit_);
|
||||
ret.append(buffer);
|
||||
}
|
||||
snprintf(buffer, kBufferSize, " memory_allocator : %s\n",
|
||||
memory_allocator() ? memory_allocator()->Name() : "None");
|
||||
ret.append(buffer);
|
||||
ret.append(GetShard(0)->GetPrintableOptions());
|
||||
return ret;
|
||||
}
|
||||
|
||||
Vendored
+2
-3
@@ -47,8 +47,7 @@ class CacheShard {
|
||||
// Keys are sharded by the highest num_shard_bits bits of hash value.
|
||||
class ShardedCache : public Cache {
|
||||
public:
|
||||
ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
|
||||
std::shared_ptr<MemoryAllocator> memory_allocator = nullptr);
|
||||
ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
|
||||
virtual ~ShardedCache() = default;
|
||||
virtual const char* Name() const override = 0;
|
||||
virtual CacheShard* GetShard(int shard) = 0;
|
||||
@@ -83,7 +82,7 @@ class ShardedCache : public Cache {
|
||||
|
||||
private:
|
||||
static inline uint32_t HashSlice(const Slice& s) {
|
||||
return static_cast<uint32_t>(GetSliceNPHash64(s));
|
||||
return Hash(s.data(), s.size(), 0);
|
||||
}
|
||||
|
||||
uint32_t Shard(uint32_t hash) {
|
||||
|
||||
@@ -5,17 +5,13 @@
|
||||
# TBB_LIBRARIES - List of libraries when using TBB.
|
||||
# TBB_FOUND - True if TBB found.
|
||||
|
||||
if(NOT DEFINED TBB_ROOT_DIR)
|
||||
set(TBB_ROOT_DIR "$ENV{TBBROOT}")
|
||||
endif()
|
||||
|
||||
find_path(TBB_INCLUDE_DIR
|
||||
NAMES tbb/tbb.h
|
||||
HINTS ${TBB_ROOT_DIR}/include)
|
||||
|
||||
find_library(TBB_LIBRARIES
|
||||
NAMES tbb
|
||||
HINTS ${TBB_ROOT_DIR}/lib ENV LIBRARY_PATH)
|
||||
HINTS ${TBB_ROOT_DIR}/lib)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARIES TBB_INCLUDE_DIR)
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
# - Find zlib
|
||||
# Find the zlib compression library and includes
|
||||
#
|
||||
# ZLIB_INCLUDE_DIR - where to find zlib.h, etc.
|
||||
# ZLIB_LIBRARIES - List of libraries when using zlib.
|
||||
# ZLIB_FOUND - True if zlib found.
|
||||
|
||||
find_path(ZLIB_INCLUDE_DIR
|
||||
NAMES zlib.h
|
||||
HINTS ${ZLIB_ROOT_DIR}/include)
|
||||
|
||||
find_library(ZLIB_LIBRARIES
|
||||
NAMES z
|
||||
HINTS ${ZLIB_ROOT_DIR}/lib)
|
||||
|
||||
include(FindPackageHandleStandardArgs)
|
||||
find_package_handle_standard_args(zlib DEFAULT_MSG ZLIB_LIBRARIES ZLIB_INCLUDE_DIR)
|
||||
|
||||
mark_as_advanced(
|
||||
ZLIB_LIBRARIES
|
||||
ZLIB_INCLUDE_DIR)
|
||||
@@ -1,3 +1,4 @@
|
||||
import optparse
|
||||
import re
|
||||
import sys
|
||||
|
||||
@@ -71,7 +72,7 @@ def display_file_coverage(per_file_coverage, total_coverage):
|
||||
header_template = \
|
||||
"%" + str(max_file_name_length) + "s\t%s\t%s"
|
||||
separator = "-" * (max_file_name_length + 10 + 20)
|
||||
print header_template % ("Filename", "Coverage", "Lines") # noqa: E999 T25377293 Grandfathered in
|
||||
print header_template % ("Filename", "Coverage", "Lines")
|
||||
print separator
|
||||
|
||||
# -- Print body
|
||||
|
||||
+38
-59
@@ -18,7 +18,6 @@
|
||||
#include "db/event_helpers.h"
|
||||
#include "db/internal_stats.h"
|
||||
#include "db/merge_helper.h"
|
||||
#include "db/range_del_aggregator.h"
|
||||
#include "db/table_cache.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "monitoring/iostats_context_imp.h"
|
||||
@@ -29,7 +28,6 @@
|
||||
#include "rocksdb/options.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "table/block_based_table_builder.h"
|
||||
#include "table/format.h"
|
||||
#include "table/internal_iterator.h"
|
||||
#include "util/file_reader_writer.h"
|
||||
#include "util/filename.h"
|
||||
@@ -41,24 +39,23 @@ namespace rocksdb {
|
||||
class TableFactory;
|
||||
|
||||
TableBuilder* NewTableBuilder(
|
||||
const ImmutableCFOptions& ioptions, const MutableCFOptions& moptions,
|
||||
const ImmutableCFOptions& ioptions,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
WritableFileWriter* file, const CompressionType compression_type,
|
||||
uint64_t sample_for_compression, const CompressionOptions& compression_opts,
|
||||
int level, const bool skip_filters, const uint64_t creation_time,
|
||||
const uint64_t oldest_key_time, const uint64_t target_file_size) {
|
||||
const CompressionOptions& compression_opts, int level,
|
||||
const std::string* compression_dict, const bool skip_filters,
|
||||
const uint64_t creation_time, const uint64_t oldest_key_time) {
|
||||
assert((column_family_id ==
|
||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
|
||||
column_family_name.empty());
|
||||
return ioptions.table_factory->NewTableBuilder(
|
||||
TableBuilderOptions(ioptions, moptions, internal_comparator,
|
||||
int_tbl_prop_collector_factories, compression_type,
|
||||
sample_for_compression, compression_opts,
|
||||
skip_filters, column_family_name, level,
|
||||
creation_time, oldest_key_time, target_file_size),
|
||||
TableBuilderOptions(
|
||||
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
||||
compression_type, compression_opts, compression_dict, skip_filters,
|
||||
column_family_name, level, creation_time, oldest_key_time),
|
||||
column_family_id, file);
|
||||
}
|
||||
|
||||
@@ -66,21 +63,19 @@ Status BuildTable(
|
||||
const std::string& dbname, Env* env, const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
|
||||
TableCache* table_cache, InternalIterator* iter,
|
||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
|
||||
range_del_iters,
|
||||
FileMetaData* meta, const InternalKeyComparator& internal_comparator,
|
||||
std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
std::vector<SequenceNumber> snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
SnapshotChecker* snapshot_checker, const CompressionType compression,
|
||||
uint64_t sample_for_compression, const CompressionOptions& compression_opts,
|
||||
bool paranoid_file_checks, InternalStats* internal_stats,
|
||||
TableFileCreationReason reason, EventLogger* event_logger, int job_id,
|
||||
const Env::IOPriority io_priority, TableProperties* table_properties,
|
||||
int level, const uint64_t creation_time, const uint64_t oldest_key_time,
|
||||
Env::WriteLifeTimeHint write_hint) {
|
||||
const CompressionOptions& compression_opts, bool paranoid_file_checks,
|
||||
InternalStats* internal_stats, TableFileCreationReason reason,
|
||||
EventLogger* event_logger, int job_id, const Env::IOPriority io_priority,
|
||||
TableProperties* table_properties, int level, const uint64_t creation_time,
|
||||
const uint64_t oldest_key_time, Env::WriteLifeTimeHint write_hint) {
|
||||
assert((column_family_id ==
|
||||
TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
|
||||
column_family_name.empty());
|
||||
@@ -89,13 +84,15 @@ Status BuildTable(
|
||||
Status s;
|
||||
meta->fd.file_size = 0;
|
||||
iter->SeekToFirst();
|
||||
std::unique_ptr<CompactionRangeDelAggregator> range_del_agg(
|
||||
new CompactionRangeDelAggregator(&internal_comparator, snapshots));
|
||||
for (auto& range_del_iter : range_del_iters) {
|
||||
range_del_agg->AddTombstones(std::move(range_del_iter));
|
||||
std::unique_ptr<RangeDelAggregator> range_del_agg(
|
||||
new RangeDelAggregator(internal_comparator, snapshots));
|
||||
s = range_del_agg->AddTombstones(std::move(range_del_iter));
|
||||
if (!s.ok()) {
|
||||
// may be non-ok if a range tombstone key is unparsable
|
||||
return s;
|
||||
}
|
||||
|
||||
std::string fname = TableFileName(ioptions.cf_paths, meta->fd.GetNumber(),
|
||||
std::string fname = TableFileName(ioptions.db_paths, meta->fd.GetNumber(),
|
||||
meta->fd.GetPathId());
|
||||
#ifndef ROCKSDB_LITE
|
||||
EventHelpers::NotifyTableFileCreationStarted(
|
||||
@@ -103,16 +100,11 @@ Status BuildTable(
|
||||
#endif // !ROCKSDB_LITE
|
||||
TableProperties tp;
|
||||
|
||||
if (iter->Valid() || !range_del_agg->IsEmpty()) {
|
||||
if (iter->Valid() || range_del_agg->ShouldAddTombstones()) {
|
||||
TableBuilder* builder;
|
||||
std::unique_ptr<WritableFileWriter> file_writer;
|
||||
// Currently we only enable dictionary compression during compaction to the
|
||||
// bottommost level.
|
||||
CompressionOptions compression_opts_for_flush(compression_opts);
|
||||
compression_opts_for_flush.max_dict_bytes = 0;
|
||||
compression_opts_for_flush.zstd_max_train_bytes = 0;
|
||||
unique_ptr<WritableFileWriter> file_writer;
|
||||
{
|
||||
std::unique_ptr<WritableFile> file;
|
||||
unique_ptr<WritableFile> file;
|
||||
#ifndef NDEBUG
|
||||
bool use_direct_writes = env_options.use_direct_writes;
|
||||
TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
|
||||
@@ -127,27 +119,23 @@ Status BuildTable(
|
||||
file->SetIOPriority(io_priority);
|
||||
file->SetWriteLifeTimeHint(write_hint);
|
||||
|
||||
file_writer.reset(
|
||||
new WritableFileWriter(std::move(file), fname, env_options, env,
|
||||
ioptions.statistics, ioptions.listeners));
|
||||
file_writer.reset(new WritableFileWriter(std::move(file), env_options,
|
||||
ioptions.statistics));
|
||||
builder = NewTableBuilder(
|
||||
ioptions, mutable_cf_options, internal_comparator,
|
||||
int_tbl_prop_collector_factories, column_family_id,
|
||||
column_family_name, file_writer.get(), compression,
|
||||
sample_for_compression, compression_opts_for_flush, level,
|
||||
ioptions, internal_comparator, int_tbl_prop_collector_factories,
|
||||
column_family_id, column_family_name, file_writer.get(), compression,
|
||||
compression_opts, level, nullptr /* compression_dict */,
|
||||
false /* skip_filters */, creation_time, oldest_key_time);
|
||||
}
|
||||
|
||||
MergeHelper merge(env, internal_comparator.user_comparator(),
|
||||
ioptions.merge_operator, nullptr, ioptions.info_log,
|
||||
true /* internal key corruption is not ok */,
|
||||
snapshots.empty() ? 0 : snapshots.back(),
|
||||
snapshot_checker);
|
||||
snapshots.empty() ? 0 : snapshots.back());
|
||||
|
||||
CompactionIterator c_iter(
|
||||
iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
|
||||
&snapshots, earliest_write_conflict_snapshot, snapshot_checker, env,
|
||||
ShouldReportDetailedTime(env, ioptions.statistics),
|
||||
true /* internal key corruption is not ok */, range_del_agg.get());
|
||||
c_iter.SeekToFirst();
|
||||
for (; c_iter.Valid(); c_iter.Next()) {
|
||||
@@ -163,20 +151,12 @@ Status BuildTable(
|
||||
ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
|
||||
}
|
||||
}
|
||||
|
||||
auto range_del_it = range_del_agg->NewIterator();
|
||||
for (range_del_it->SeekToFirst(); range_del_it->Valid();
|
||||
range_del_it->Next()) {
|
||||
auto tombstone = range_del_it->Tombstone();
|
||||
auto kv = tombstone.Serialize();
|
||||
builder->Add(kv.first.Encode(), kv.second);
|
||||
meta->UpdateBoundariesForRange(kv.first, tombstone.SerializeEndKey(),
|
||||
tombstone.seq_, internal_comparator);
|
||||
}
|
||||
// nullptr for table_{min,max} so all range tombstones will be flushed
|
||||
range_del_agg->AddToBuilder(builder, nullptr /* lower_bound */,
|
||||
nullptr /* upper_bound */, meta);
|
||||
|
||||
// Finish and check for builder errors
|
||||
tp = builder->GetTableProperties();
|
||||
bool empty = builder->NumEntries() == 0 && tp.num_range_deletions == 0;
|
||||
bool empty = builder->NumEntries() == 0;
|
||||
s = c_iter.status();
|
||||
if (!s.ok() || empty) {
|
||||
builder->Abandon();
|
||||
@@ -189,7 +169,7 @@ Status BuildTable(
|
||||
meta->fd.file_size = file_size;
|
||||
meta->marked_for_compaction = builder->NeedCompact();
|
||||
assert(meta->fd.GetFileSize() > 0);
|
||||
tp = builder->GetTableProperties(); // refresh now that builder is finished
|
||||
tp = builder->GetTableProperties();
|
||||
if (table_properties) {
|
||||
*table_properties = tp;
|
||||
}
|
||||
@@ -213,9 +193,8 @@ Status BuildTable(
|
||||
// we will regrad this verification as user reads since the goal is
|
||||
// to cache it here for further user reads
|
||||
std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
|
||||
ReadOptions(), env_options, internal_comparator, *meta,
|
||||
nullptr /* range_del_agg */,
|
||||
mutable_cf_options.prefix_extractor.get(), nullptr,
|
||||
ReadOptions(), env_options, internal_comparator, meta->fd,
|
||||
nullptr /* range_del_agg */, nullptr,
|
||||
(internal_stats == nullptr) ? nullptr
|
||||
: internal_stats->GetFileReadHist(0),
|
||||
false /* for_compaction */, nullptr /* arena */,
|
||||
|
||||
+8
-8
@@ -9,7 +9,6 @@
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "db/range_tombstone_fragmenter.h"
|
||||
#include "db/table_properties_collector.h"
|
||||
#include "options/cf_options.h"
|
||||
#include "rocksdb/comparator.h"
|
||||
@@ -36,21 +35,24 @@ class VersionEdit;
|
||||
class TableBuilder;
|
||||
class WritableFileWriter;
|
||||
class InternalStats;
|
||||
class InternalIterator;
|
||||
|
||||
// @param column_family_name Name of the column family that is also identified
|
||||
// by column_family_id, or empty string if unknown. It must outlive the
|
||||
// TableBuilder returned by this function.
|
||||
// @param compression_dict Data for presetting the compression library's
|
||||
// dictionary, or nullptr.
|
||||
TableBuilder* NewTableBuilder(
|
||||
const ImmutableCFOptions& options, const MutableCFOptions& moptions,
|
||||
const ImmutableCFOptions& options,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
WritableFileWriter* file, const CompressionType compression_type,
|
||||
const uint64_t sample_for_compression,
|
||||
const CompressionOptions& compression_opts, int level,
|
||||
const std::string* compression_dict = nullptr,
|
||||
const bool skip_filters = false, const uint64_t creation_time = 0,
|
||||
const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0);
|
||||
const uint64_t oldest_key_time = 0);
|
||||
|
||||
// Build a Table file from the contents of *iter. The generated file
|
||||
// will be named according to number specified in meta. On success, the rest of
|
||||
@@ -64,16 +66,14 @@ extern Status BuildTable(
|
||||
const std::string& dbname, Env* env, const ImmutableCFOptions& options,
|
||||
const MutableCFOptions& mutable_cf_options, const EnvOptions& env_options,
|
||||
TableCache* table_cache, InternalIterator* iter,
|
||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
|
||||
range_del_iters,
|
||||
FileMetaData* meta, const InternalKeyComparator& internal_comparator,
|
||||
std::unique_ptr<InternalIterator> range_del_iter, FileMetaData* meta,
|
||||
const InternalKeyComparator& internal_comparator,
|
||||
const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
|
||||
int_tbl_prop_collector_factories,
|
||||
uint32_t column_family_id, const std::string& column_family_name,
|
||||
std::vector<SequenceNumber> snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
SnapshotChecker* snapshot_checker, const CompressionType compression,
|
||||
const uint64_t sample_for_compression,
|
||||
const CompressionOptions& compression_opts, bool paranoid_file_checks,
|
||||
InternalStats* internal_stats, TableFileCreationReason reason,
|
||||
EventLogger* event_logger = nullptr, int job_id = 0,
|
||||
|
||||
+15
-105
@@ -19,8 +19,11 @@
|
||||
|
||||
// Can not use port/port.h macros as this is a c file
|
||||
#ifdef OS_WIN
|
||||
|
||||
#include <windows.h>
|
||||
|
||||
#define snprintf _snprintf
|
||||
|
||||
// Ok for uniqueness
|
||||
int geteuid() {
|
||||
int result = 0;
|
||||
@@ -31,11 +34,6 @@ int geteuid() {
|
||||
return result;
|
||||
}
|
||||
|
||||
// VS < 2015
|
||||
#if defined(_MSC_VER) && (_MSC_VER < 1900)
|
||||
#define snprintf _snprintf
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
const char* phase = "";
|
||||
@@ -49,19 +47,12 @@ static void StartPhase(const char* name) {
|
||||
fprintf(stderr, "=== Test %s\n", name);
|
||||
phase = name;
|
||||
}
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(push)
|
||||
#pragma warning (disable: 4996) // getenv security warning
|
||||
#endif
|
||||
static const char* GetTempDir(void) {
|
||||
const char* ret = getenv("TEST_TMPDIR");
|
||||
if (ret == NULL || ret[0] == '\0')
|
||||
ret = "/tmp";
|
||||
return ret;
|
||||
}
|
||||
#ifdef _MSC_VER
|
||||
#pragma warning(pop)
|
||||
#endif
|
||||
|
||||
#define CheckNoError(err) \
|
||||
if ((err) != NULL) { \
|
||||
@@ -201,11 +192,10 @@ static void CheckDel(void* ptr, const char* k, size_t klen) {
|
||||
(*state)++;
|
||||
}
|
||||
|
||||
static void CmpDestroy(void* arg) { (void)arg; }
|
||||
static void CmpDestroy(void* arg) { }
|
||||
|
||||
static int CmpCompare(void* arg, const char* a, size_t alen,
|
||||
const char* b, size_t blen) {
|
||||
(void)arg;
|
||||
size_t n = (alen < blen) ? alen : blen;
|
||||
int r = memcmp(a, b, n);
|
||||
if (r == 0) {
|
||||
@@ -216,15 +206,13 @@ static int CmpCompare(void* arg, const char* a, size_t alen,
|
||||
}
|
||||
|
||||
static const char* CmpName(void* arg) {
|
||||
(void)arg;
|
||||
return "foo";
|
||||
}
|
||||
|
||||
// Custom filter policy
|
||||
static unsigned char fake_filter_result = 1;
|
||||
static void FilterDestroy(void* arg) { (void)arg; }
|
||||
static void FilterDestroy(void* arg) { }
|
||||
static const char* FilterName(void* arg) {
|
||||
(void)arg;
|
||||
return "TestFilter";
|
||||
}
|
||||
static char* FilterCreate(
|
||||
@@ -232,10 +220,6 @@ static char* FilterCreate(
|
||||
const char* const* key_array, const size_t* key_length_array,
|
||||
int num_keys,
|
||||
size_t* filter_length) {
|
||||
(void)arg;
|
||||
(void)key_array;
|
||||
(void)key_length_array;
|
||||
(void)num_keys;
|
||||
*filter_length = 4;
|
||||
char* result = malloc(4);
|
||||
memcpy(result, "fake", 4);
|
||||
@@ -245,30 +229,20 @@ static unsigned char FilterKeyMatch(
|
||||
void* arg,
|
||||
const char* key, size_t length,
|
||||
const char* filter, size_t filter_length) {
|
||||
(void)arg;
|
||||
(void)key;
|
||||
(void)length;
|
||||
CheckCondition(filter_length == 4);
|
||||
CheckCondition(memcmp(filter, "fake", 4) == 0);
|
||||
return fake_filter_result;
|
||||
}
|
||||
|
||||
// Custom compaction filter
|
||||
static void CFilterDestroy(void* arg) { (void)arg; }
|
||||
static const char* CFilterName(void* arg) {
|
||||
(void)arg;
|
||||
return "foo";
|
||||
}
|
||||
static void CFilterDestroy(void* arg) {}
|
||||
static const char* CFilterName(void* arg) { return "foo"; }
|
||||
static unsigned char CFilterFilter(void* arg, int level, const char* key,
|
||||
size_t key_length,
|
||||
const char* existing_value,
|
||||
size_t value_length, char** new_value,
|
||||
size_t* new_value_length,
|
||||
unsigned char* value_changed) {
|
||||
(void)arg;
|
||||
(void)level;
|
||||
(void)existing_value;
|
||||
(void)value_length;
|
||||
if (key_length == 3) {
|
||||
if (memcmp(key, "bar", key_length) == 0) {
|
||||
return 1;
|
||||
@@ -282,15 +256,10 @@ static unsigned char CFilterFilter(void* arg, int level, const char* key,
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void CFilterFactoryDestroy(void* arg) { (void)arg; }
|
||||
static const char* CFilterFactoryName(void* arg) {
|
||||
(void)arg;
|
||||
return "foo";
|
||||
}
|
||||
static void CFilterFactoryDestroy(void* arg) {}
|
||||
static const char* CFilterFactoryName(void* arg) { return "foo"; }
|
||||
static rocksdb_compactionfilter_t* CFilterCreate(
|
||||
void* arg, rocksdb_compactionfiltercontext_t* context) {
|
||||
(void)arg;
|
||||
(void)context;
|
||||
return rocksdb_compactionfilter_create(NULL, CFilterDestroy, CFilterFilter,
|
||||
CFilterName);
|
||||
}
|
||||
@@ -321,9 +290,8 @@ static rocksdb_t* CheckCompaction(rocksdb_t* db, rocksdb_options_t* options,
|
||||
}
|
||||
|
||||
// Custom merge operator
|
||||
static void MergeOperatorDestroy(void* arg) { (void)arg; }
|
||||
static void MergeOperatorDestroy(void* arg) { }
|
||||
static const char* MergeOperatorName(void* arg) {
|
||||
(void)arg;
|
||||
return "TestMergeOperator";
|
||||
}
|
||||
static char* MergeOperatorFullMerge(
|
||||
@@ -333,14 +301,6 @@ static char* MergeOperatorFullMerge(
|
||||
const char* const* operands_list, const size_t* operands_list_length,
|
||||
int num_operands,
|
||||
unsigned char* success, size_t* new_value_length) {
|
||||
(void)arg;
|
||||
(void)key;
|
||||
(void)key_length;
|
||||
(void)existing_value;
|
||||
(void)existing_value_length;
|
||||
(void)operands_list;
|
||||
(void)operands_list_length;
|
||||
(void)num_operands;
|
||||
*new_value_length = 4;
|
||||
*success = 1;
|
||||
char* result = malloc(4);
|
||||
@@ -353,12 +313,6 @@ static char* MergeOperatorPartialMerge(
|
||||
const char* const* operands_list, const size_t* operands_list_length,
|
||||
int num_operands,
|
||||
unsigned char* success, size_t* new_value_length) {
|
||||
(void)arg;
|
||||
(void)key;
|
||||
(void)key_length;
|
||||
(void)operands_list;
|
||||
(void)operands_list_length;
|
||||
(void)num_operands;
|
||||
*new_value_length = 4;
|
||||
*success = 1;
|
||||
char* result = malloc(4);
|
||||
@@ -423,8 +377,6 @@ static void CheckTxnDBGetCF(rocksdb_transactiondb_t* txn_db,
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
(void)argc;
|
||||
(void)argv;
|
||||
rocksdb_t* db;
|
||||
rocksdb_comparator_t* cmp;
|
||||
rocksdb_cache_t* cache;
|
||||
@@ -652,7 +604,7 @@ int main(int argc, char** argv) {
|
||||
rocksdb_sstfilewriter_t* writer =
|
||||
rocksdb_sstfilewriter_create(env_opt, io_options);
|
||||
|
||||
remove(sstfilename);
|
||||
unlink(sstfilename);
|
||||
rocksdb_sstfilewriter_open(writer, sstfilename, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_put(writer, "sstk1", 5, "v1", 2, &err);
|
||||
@@ -673,7 +625,7 @@ int main(int argc, char** argv) {
|
||||
CheckGet(db, roptions, "sstk2", "v2");
|
||||
CheckGet(db, roptions, "sstk3", "v3");
|
||||
|
||||
remove(sstfilename);
|
||||
unlink(sstfilename);
|
||||
rocksdb_sstfilewriter_open(writer, sstfilename, &err);
|
||||
CheckNoError(err);
|
||||
rocksdb_sstfilewriter_put(writer, "sstk2", 5, "v4", 2, &err);
|
||||
@@ -917,8 +869,7 @@ int main(int argc, char** argv) {
|
||||
rocksdb_writebatch_wi_t* wbi = rocksdb_writebatch_wi_create(0, 1);
|
||||
rocksdb_writebatch_wi_put(wbi, "bar", 3, "b", 1);
|
||||
rocksdb_writebatch_wi_delete(wbi, "foo", 3);
|
||||
rocksdb_iterator_t* iter =
|
||||
rocksdb_writebatch_wi_create_iterator_with_base(wbi, base_iter);
|
||||
rocksdb_iterator_t* iter = rocksdb_writebatch_wi_create_iterator_with_base(wbi, base_iter);
|
||||
CheckCondition(!rocksdb_iter_valid(iter));
|
||||
rocksdb_iter_seek_to_first(iter);
|
||||
CheckCondition(rocksdb_iter_valid(iter));
|
||||
@@ -1344,47 +1295,6 @@ int main(int argc, char** argv) {
|
||||
rocksdb_destroy_db(options, dbname, &err);
|
||||
}
|
||||
|
||||
// Check memory usage stats
|
||||
StartPhase("approximate_memory_usage");
|
||||
{
|
||||
// Create database
|
||||
db = rocksdb_open(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
rocksdb_memory_consumers_t* consumers;
|
||||
consumers = rocksdb_memory_consumers_create();
|
||||
rocksdb_memory_consumers_add_db(consumers, db);
|
||||
rocksdb_memory_consumers_add_cache(consumers, cache);
|
||||
|
||||
// take memory usage report before write-read operation
|
||||
rocksdb_memory_usage_t* mu1;
|
||||
mu1 = rocksdb_approximate_memory_usage_create(consumers, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
// Put data (this should affect memtables)
|
||||
rocksdb_put(db, woptions, "memory", 6, "test", 4, &err);
|
||||
CheckNoError(err);
|
||||
CheckGet(db, roptions, "memory", "test");
|
||||
|
||||
// take memory usage report after write-read operation
|
||||
rocksdb_memory_usage_t* mu2;
|
||||
mu2 = rocksdb_approximate_memory_usage_create(consumers, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
// amount of memory used within memtables should grow
|
||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_total(mu2) >=
|
||||
rocksdb_approximate_memory_usage_get_mem_table_total(mu1));
|
||||
CheckCondition(rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu2) >=
|
||||
rocksdb_approximate_memory_usage_get_mem_table_unflushed(mu1));
|
||||
|
||||
rocksdb_memory_consumers_destroy(consumers);
|
||||
rocksdb_approximate_memory_usage_destroy(mu1);
|
||||
rocksdb_approximate_memory_usage_destroy(mu2);
|
||||
rocksdb_close(db);
|
||||
rocksdb_destroy_db(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
}
|
||||
|
||||
StartPhase("cuckoo_options");
|
||||
{
|
||||
rocksdb_cuckoo_table_options_t* cuckoo_options;
|
||||
@@ -1528,7 +1438,7 @@ int main(int argc, char** argv) {
|
||||
const rocksdb_snapshot_t* snapshot;
|
||||
snapshot = rocksdb_transactiondb_create_snapshot(txn_db);
|
||||
rocksdb_readoptions_set_snapshot(roptions, snapshot);
|
||||
|
||||
|
||||
rocksdb_transactiondb_put(txn_db, woptions, "foo", 3, "hey", 3, &err);
|
||||
CheckNoError(err);
|
||||
|
||||
@@ -1726,7 +1636,7 @@ int main(int argc, char** argv) {
|
||||
db = rocksdb_open(options, dbname, &err);
|
||||
CheckNoError(err);
|
||||
}
|
||||
|
||||
|
||||
StartPhase("cleanup");
|
||||
rocksdb_close(db);
|
||||
rocksdb_options_destroy(options);
|
||||
|
||||
+68
-235
@@ -20,12 +20,10 @@
|
||||
#include <limits>
|
||||
|
||||
#include "db/compaction_picker.h"
|
||||
#include "db/compaction_picker_fifo.h"
|
||||
#include "db/compaction_picker_universal.h"
|
||||
#include "db/db_impl.h"
|
||||
#include "db/internal_stats.h"
|
||||
#include "db/job_context.h"
|
||||
#include "db/range_del_aggregator.h"
|
||||
#include "db/table_properties_collector.h"
|
||||
#include "db/version_set.h"
|
||||
#include "db/write_controller.h"
|
||||
@@ -33,10 +31,8 @@
|
||||
#include "monitoring/thread_status_util.h"
|
||||
#include "options/options_helper.h"
|
||||
#include "table/block_based_table_factory.h"
|
||||
#include "table/merging_iterator.h"
|
||||
#include "util/autovector.h"
|
||||
#include "util/compression.h"
|
||||
#include "util/sst_file_manager_impl.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
@@ -57,9 +53,6 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
|
||||
#endif // ROCKSDB_LITE
|
||||
// Job id == 0 means that this is not our background process, but rather
|
||||
// user thread
|
||||
// Need to hold some shared pointers owned by the initial_cf_options
|
||||
// before final cleaning up finishes.
|
||||
ColumnFamilyOptions initial_cf_options_copy = cfd_->initial_cf_options();
|
||||
JobContext job_context(0);
|
||||
mutex_->Lock();
|
||||
if (cfd_->Unref()) {
|
||||
@@ -68,14 +61,7 @@ ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
|
||||
db_->FindObsoleteFiles(&job_context, false, true);
|
||||
mutex_->Unlock();
|
||||
if (job_context.HaveSomethingToDelete()) {
|
||||
bool defer_purge =
|
||||
db_->immutable_db_options().avoid_unnecessary_blocking_io;
|
||||
db_->PurgeObsoleteFiles(job_context, defer_purge);
|
||||
if (defer_purge) {
|
||||
mutex_->Lock();
|
||||
db_->SchedulePurge();
|
||||
mutex_->Unlock();
|
||||
}
|
||||
db_->PurgeObsoleteFiles(job_context);
|
||||
}
|
||||
job_context.Clean();
|
||||
}
|
||||
@@ -94,7 +80,6 @@ Status ColumnFamilyHandleImpl::GetDescriptor(ColumnFamilyDescriptor* desc) {
|
||||
*desc = ColumnFamilyDescriptor(cfd()->GetName(), cfd()->GetLatestCFOptions());
|
||||
return Status::OK();
|
||||
#else
|
||||
(void)desc;
|
||||
return Status::NotSupported();
|
||||
#endif // !ROCKSDB_LITE
|
||||
}
|
||||
@@ -114,6 +99,9 @@ void GetIntTblPropCollectorFactory(
|
||||
int_tbl_prop_collector_factories->emplace_back(
|
||||
new UserKeyTablePropertiesCollectorFactory(collector_factories[i]));
|
||||
}
|
||||
// Add collector to collect internal key statistics
|
||||
int_tbl_prop_collector_factories->emplace_back(
|
||||
new InternalKeyPropertiesCollectorFactory);
|
||||
}
|
||||
|
||||
Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
|
||||
@@ -136,10 +124,14 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
|
||||
}
|
||||
}
|
||||
if (cf_options.compression_opts.zstd_max_train_bytes > 0) {
|
||||
if (!ZSTD_TrainDictionarySupported()) {
|
||||
if (!CompressionTypeSupported(CompressionType::kZSTD)) {
|
||||
// Dictionary trainer is available since v0.6.1, but ZSTD was marked
|
||||
// stable only since v0.8.0. For now we enable the feature in stable
|
||||
// versions only.
|
||||
return Status::InvalidArgument(
|
||||
"zstd dictionary trainer cannot be used because ZSTD 1.1.3+ "
|
||||
"is not linked with the binary.");
|
||||
"zstd dictionary trainer cannot be used because " +
|
||||
CompressionTypeToString(CompressionType::kZSTD) +
|
||||
" is not linked with the binary.");
|
||||
}
|
||||
if (cf_options.compression_opts.max_dict_bytes == 0) {
|
||||
return Status::InvalidArgument(
|
||||
@@ -163,28 +155,6 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status CheckCFPathsSupported(const DBOptions& db_options,
|
||||
const ColumnFamilyOptions& cf_options) {
|
||||
// More than one cf_paths are supported only in universal
|
||||
// and level compaction styles. This function also checks the case
|
||||
// in which cf_paths is not specified, which results in db_paths
|
||||
// being used.
|
||||
if ((cf_options.compaction_style != kCompactionStyleUniversal) &&
|
||||
(cf_options.compaction_style != kCompactionStyleLevel)) {
|
||||
if (cf_options.cf_paths.size() > 1) {
|
||||
return Status::NotSupported(
|
||||
"More than one CF paths are only supported in "
|
||||
"universal and level compaction styles. ");
|
||||
} else if (cf_options.cf_paths.empty() &&
|
||||
db_options.db_paths.size() > 1) {
|
||||
return Status::NotSupported(
|
||||
"More than one DB paths are only supported in "
|
||||
"universal and level compaction styles. ");
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||
const ColumnFamilyOptions& src) {
|
||||
ColumnFamilyOptions result = src;
|
||||
@@ -303,24 +273,9 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||
result.hard_pending_compaction_bytes_limit;
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
// When the DB is stopped, it's possible that there are some .trash files that
|
||||
// were not deleted yet, when we open the DB we will find these .trash files
|
||||
// and schedule them to be deleted (or delete immediately if SstFileManager
|
||||
// was not used)
|
||||
auto sfm = static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
|
||||
for (size_t i = 0; i < result.cf_paths.size(); i++) {
|
||||
DeleteScheduler::CleanupDirectory(db_options.env, sfm, result.cf_paths[i].path);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (result.cf_paths.empty()) {
|
||||
result.cf_paths = db_options.db_paths;
|
||||
}
|
||||
|
||||
if (result.level_compaction_dynamic_level_bytes) {
|
||||
if (result.compaction_style != kCompactionStyleLevel ||
|
||||
result.cf_paths.size() > 1U) {
|
||||
db_options.db_paths.size() > 1U) {
|
||||
// 1. level_compaction_dynamic_level_bytes only makes sense for
|
||||
// level-based compaction.
|
||||
// 2. we don't yet know how to make both of this feature and multiple
|
||||
@@ -389,13 +344,12 @@ void SuperVersionUnrefHandle(void* ptr) {
|
||||
// When latter happens, we are in ~ColumnFamilyData(), no get should happen as
|
||||
// well.
|
||||
SuperVersion* sv = static_cast<SuperVersion*>(ptr);
|
||||
bool was_last_ref __attribute__((__unused__));
|
||||
was_last_ref = sv->Unref();
|
||||
// Thread-local SuperVersions can't outlive ColumnFamilyData::super_version_.
|
||||
// This is important because we can't do SuperVersion cleanup here.
|
||||
// That would require locking DB mutex, which would deadlock because
|
||||
// SuperVersionUnrefHandle is called with locked ThreadLocalPtr mutex.
|
||||
assert(!was_last_ref);
|
||||
if (sv->Unref()) {
|
||||
sv->db_mutex->Lock();
|
||||
sv->Cleanup();
|
||||
sv->db_mutex->Unlock();
|
||||
delete sv;
|
||||
}
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
@@ -427,10 +381,9 @@ ColumnFamilyData::ColumnFamilyData(
|
||||
next_(nullptr),
|
||||
prev_(nullptr),
|
||||
log_number_(0),
|
||||
flush_reason_(FlushReason::kOthers),
|
||||
column_family_set_(column_family_set),
|
||||
queued_for_flush_(false),
|
||||
queued_for_compaction_(false),
|
||||
pending_flush_(false),
|
||||
pending_compaction_(false),
|
||||
prev_compaction_needed_bytes_(0),
|
||||
allow_2pc_(db_options.allow_2pc),
|
||||
last_memtable_id_(0) {
|
||||
@@ -506,8 +459,8 @@ ColumnFamilyData::~ColumnFamilyData() {
|
||||
|
||||
// It would be wrong if this ColumnFamilyData is in flush_queue_ or
|
||||
// compaction_queue_ and we destroyed it
|
||||
assert(!queued_for_flush_);
|
||||
assert(!queued_for_compaction_);
|
||||
assert(!pending_flush_);
|
||||
assert(!pending_compaction_);
|
||||
|
||||
if (super_version_ != nullptr) {
|
||||
// Release SuperVersion reference kept in ThreadLocalPtr.
|
||||
@@ -516,7 +469,7 @@ ColumnFamilyData::~ColumnFamilyData() {
|
||||
local_sv_.reset();
|
||||
super_version_->db_mutex->Lock();
|
||||
|
||||
bool is_last_reference __attribute__((__unused__));
|
||||
bool is_last_reference __attribute__((unused));
|
||||
is_last_reference = super_version_->Unref();
|
||||
assert(is_last_reference);
|
||||
super_version_->Cleanup();
|
||||
@@ -527,7 +480,7 @@ ColumnFamilyData::~ColumnFamilyData() {
|
||||
if (dummy_versions_ != nullptr) {
|
||||
// List must be empty
|
||||
assert(dummy_versions_->TEST_Next() == dummy_versions_);
|
||||
bool deleted __attribute__((__unused__));
|
||||
bool deleted __attribute__((unused));
|
||||
deleted = dummy_versions_->Unref();
|
||||
assert(deleted);
|
||||
}
|
||||
@@ -560,9 +513,7 @@ uint64_t ColumnFamilyData::OldestLogToKeep() {
|
||||
auto current_log = GetLogNumber();
|
||||
|
||||
if (allow_2pc_) {
|
||||
autovector<MemTable*> empty_list;
|
||||
auto imm_prep_log =
|
||||
imm()->PrecomputeMinLogContainingPrepSection(empty_list);
|
||||
auto imm_prep_log = imm()->GetMinLogContainingPrepSection();
|
||||
auto mem_prep_log = mem()->GetMinLogContainingPrepSection();
|
||||
|
||||
if (imm_prep_log > 0 && imm_prep_log < current_log) {
|
||||
@@ -680,41 +631,6 @@ int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
|
||||
ColumnFamilyData::GetWriteStallConditionAndCause(
|
||||
int num_unflushed_memtables, int num_l0_files,
|
||||
uint64_t num_compaction_needed_bytes,
|
||||
const MutableCFOptions& mutable_cf_options) {
|
||||
if (num_unflushed_memtables >= mutable_cf_options.max_write_buffer_number) {
|
||||
return {WriteStallCondition::kStopped, WriteStallCause::kMemtableLimit};
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
num_l0_files >= mutable_cf_options.level0_stop_writes_trigger) {
|
||||
return {WriteStallCondition::kStopped, WriteStallCause::kL0FileCountLimit};
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
|
||||
num_compaction_needed_bytes >=
|
||||
mutable_cf_options.hard_pending_compaction_bytes_limit) {
|
||||
return {WriteStallCondition::kStopped,
|
||||
WriteStallCause::kPendingCompactionBytes};
|
||||
} else if (mutable_cf_options.max_write_buffer_number > 3 &&
|
||||
num_unflushed_memtables >=
|
||||
mutable_cf_options.max_write_buffer_number - 1) {
|
||||
return {WriteStallCondition::kDelayed, WriteStallCause::kMemtableLimit};
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
|
||||
num_l0_files >=
|
||||
mutable_cf_options.level0_slowdown_writes_trigger) {
|
||||
return {WriteStallCondition::kDelayed, WriteStallCause::kL0FileCountLimit};
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
|
||||
num_compaction_needed_bytes >=
|
||||
mutable_cf_options.soft_pending_compaction_bytes_limit) {
|
||||
return {WriteStallCondition::kDelayed,
|
||||
WriteStallCause::kPendingCompactionBytes};
|
||||
}
|
||||
return {WriteStallCondition::kNormal, WriteStallCause::kNone};
|
||||
}
|
||||
|
||||
WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
const MutableCFOptions& mutable_cf_options) {
|
||||
auto write_stall_condition = WriteStallCondition::kNormal;
|
||||
@@ -724,29 +640,25 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
uint64_t compaction_needed_bytes =
|
||||
vstorage->estimated_compaction_needed_bytes();
|
||||
|
||||
auto write_stall_condition_and_cause = GetWriteStallConditionAndCause(
|
||||
imm()->NumNotFlushed(), vstorage->l0_delay_trigger_count(),
|
||||
vstorage->estimated_compaction_needed_bytes(), mutable_cf_options);
|
||||
write_stall_condition = write_stall_condition_and_cause.first;
|
||||
auto write_stall_cause = write_stall_condition_and_cause.second;
|
||||
|
||||
bool was_stopped = write_controller->IsStopped();
|
||||
bool needed_delay = write_controller->NeedsDelay();
|
||||
|
||||
if (write_stall_condition == WriteStallCondition::kStopped &&
|
||||
write_stall_cause == WriteStallCause::kMemtableLimit) {
|
||||
if (imm()->NumNotFlushed() >= mutable_cf_options.max_write_buffer_number) {
|
||||
write_controller_token_ = write_controller->GetStopToken();
|
||||
internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1);
|
||||
write_stall_condition = WriteStallCondition::kStopped;
|
||||
ROCKS_LOG_WARN(
|
||||
ioptions_.info_log,
|
||||
"[%s] Stopping writes because we have %d immutable memtables "
|
||||
"(waiting for flush), max_write_buffer_number is set to %d",
|
||||
name_.c_str(), imm()->NumNotFlushed(),
|
||||
mutable_cf_options.max_write_buffer_number);
|
||||
} else if (write_stall_condition == WriteStallCondition::kStopped &&
|
||||
write_stall_cause == WriteStallCause::kL0FileCountLimit) {
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
vstorage->l0_delay_trigger_count() >=
|
||||
mutable_cf_options.level0_stop_writes_trigger) {
|
||||
write_controller_token_ = write_controller->GetStopToken();
|
||||
internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_STOPS, 1);
|
||||
write_stall_condition = WriteStallCondition::kStopped;
|
||||
if (compaction_picker_->IsLevel0CompactionInProgress()) {
|
||||
internal_stats_->AddCFStats(
|
||||
InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_STOPS, 1);
|
||||
@@ -754,23 +666,28 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
ROCKS_LOG_WARN(ioptions_.info_log,
|
||||
"[%s] Stopping writes because we have %d level-0 files",
|
||||
name_.c_str(), vstorage->l0_delay_trigger_count());
|
||||
} else if (write_stall_condition == WriteStallCondition::kStopped &&
|
||||
write_stall_cause == WriteStallCause::kPendingCompactionBytes) {
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
mutable_cf_options.hard_pending_compaction_bytes_limit > 0 &&
|
||||
compaction_needed_bytes >=
|
||||
mutable_cf_options.hard_pending_compaction_bytes_limit) {
|
||||
write_controller_token_ = write_controller->GetStopToken();
|
||||
internal_stats_->AddCFStats(
|
||||
InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1);
|
||||
write_stall_condition = WriteStallCondition::kStopped;
|
||||
ROCKS_LOG_WARN(
|
||||
ioptions_.info_log,
|
||||
"[%s] Stopping writes because of estimated pending compaction "
|
||||
"bytes %" PRIu64,
|
||||
name_.c_str(), compaction_needed_bytes);
|
||||
} else if (write_stall_condition == WriteStallCondition::kDelayed &&
|
||||
write_stall_cause == WriteStallCause::kMemtableLimit) {
|
||||
} else if (mutable_cf_options.max_write_buffer_number > 3 &&
|
||||
imm()->NumNotFlushed() >=
|
||||
mutable_cf_options.max_write_buffer_number - 1) {
|
||||
write_controller_token_ =
|
||||
SetupDelay(write_controller, compaction_needed_bytes,
|
||||
prev_compaction_needed_bytes_, was_stopped,
|
||||
mutable_cf_options.disable_auto_compactions);
|
||||
internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_SLOWDOWNS, 1);
|
||||
write_stall_condition = WriteStallCondition::kDelayed;
|
||||
ROCKS_LOG_WARN(
|
||||
ioptions_.info_log,
|
||||
"[%s] Stalling writes because we have %d immutable memtables "
|
||||
@@ -779,8 +696,10 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
name_.c_str(), imm()->NumNotFlushed(),
|
||||
mutable_cf_options.max_write_buffer_number,
|
||||
write_controller->delayed_write_rate());
|
||||
} else if (write_stall_condition == WriteStallCondition::kDelayed &&
|
||||
write_stall_cause == WriteStallCause::kL0FileCountLimit) {
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
|
||||
vstorage->l0_delay_trigger_count() >=
|
||||
mutable_cf_options.level0_slowdown_writes_trigger) {
|
||||
// L0 is the last two files from stopping.
|
||||
bool near_stop = vstorage->l0_delay_trigger_count() >=
|
||||
mutable_cf_options.level0_stop_writes_trigger - 2;
|
||||
@@ -790,6 +709,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
mutable_cf_options.disable_auto_compactions);
|
||||
internal_stats_->AddCFStats(InternalStats::L0_FILE_COUNT_LIMIT_SLOWDOWNS,
|
||||
1);
|
||||
write_stall_condition = WriteStallCondition::kDelayed;
|
||||
if (compaction_picker_->IsLevel0CompactionInProgress()) {
|
||||
internal_stats_->AddCFStats(
|
||||
InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS, 1);
|
||||
@@ -799,8 +719,10 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
"rate %" PRIu64,
|
||||
name_.c_str(), vstorage->l0_delay_trigger_count(),
|
||||
write_controller->delayed_write_rate());
|
||||
} else if (write_stall_condition == WriteStallCondition::kDelayed &&
|
||||
write_stall_cause == WriteStallCause::kPendingCompactionBytes) {
|
||||
} else if (!mutable_cf_options.disable_auto_compactions &&
|
||||
mutable_cf_options.soft_pending_compaction_bytes_limit > 0 &&
|
||||
vstorage->estimated_compaction_needed_bytes() >=
|
||||
mutable_cf_options.soft_pending_compaction_bytes_limit) {
|
||||
// If the distance to hard limit is less than 1/4 of the gap between soft
|
||||
// and
|
||||
// hard bytes limit, we think it is near stop and speed up the slowdown.
|
||||
@@ -818,6 +740,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
mutable_cf_options.disable_auto_compactions);
|
||||
internal_stats_->AddCFStats(
|
||||
InternalStats::PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS, 1);
|
||||
write_stall_condition = WriteStallCondition::kDelayed;
|
||||
ROCKS_LOG_WARN(
|
||||
ioptions_.info_log,
|
||||
"[%s] Stalling writes because of estimated pending compaction "
|
||||
@@ -825,7 +748,6 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
|
||||
name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
|
||||
write_controller->delayed_write_rate());
|
||||
} else {
|
||||
assert(write_stall_condition == WriteStallCondition::kNormal);
|
||||
if (vstorage->l0_delay_trigger_count() >=
|
||||
GetL0ThresholdSpeedupCompaction(
|
||||
mutable_cf_options.level0_file_num_compaction_trigger,
|
||||
@@ -892,10 +814,6 @@ uint64_t ColumnFamilyData::GetTotalSstFilesSize() const {
|
||||
return VersionSet::GetTotalSstFilesSize(dummy_versions_);
|
||||
}
|
||||
|
||||
uint64_t ColumnFamilyData::GetLiveSstFilesSize() const {
|
||||
return current_->GetSstFilesSize();
|
||||
}
|
||||
|
||||
MemTable* ColumnFamilyData::ConstructNewMemtable(
|
||||
const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
|
||||
return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
|
||||
@@ -932,70 +850,16 @@ bool ColumnFamilyData::RangeOverlapWithCompaction(
|
||||
smallest_user_key, largest_user_key, level);
|
||||
}
|
||||
|
||||
Status ColumnFamilyData::RangesOverlapWithMemtables(
|
||||
const autovector<Range>& ranges, SuperVersion* super_version,
|
||||
bool* overlap) {
|
||||
assert(overlap != nullptr);
|
||||
*overlap = false;
|
||||
// Create an InternalIterator over all unflushed memtables
|
||||
Arena arena;
|
||||
ReadOptions read_opts;
|
||||
read_opts.total_order_seek = true;
|
||||
MergeIteratorBuilder merge_iter_builder(&internal_comparator_, &arena);
|
||||
merge_iter_builder.AddIterator(
|
||||
super_version->mem->NewIterator(read_opts, &arena));
|
||||
super_version->imm->AddIterators(read_opts, &merge_iter_builder);
|
||||
ScopedArenaIterator memtable_iter(merge_iter_builder.Finish());
|
||||
|
||||
auto read_seq = super_version->current->version_set()->LastSequence();
|
||||
ReadRangeDelAggregator range_del_agg(&internal_comparator_, read_seq);
|
||||
auto* active_range_del_iter =
|
||||
super_version->mem->NewRangeTombstoneIterator(read_opts, read_seq);
|
||||
range_del_agg.AddTombstones(
|
||||
std::unique_ptr<FragmentedRangeTombstoneIterator>(active_range_del_iter));
|
||||
super_version->imm->AddRangeTombstoneIterators(read_opts, nullptr /* arena */,
|
||||
&range_del_agg);
|
||||
|
||||
Status status;
|
||||
for (size_t i = 0; i < ranges.size() && status.ok() && !*overlap; ++i) {
|
||||
auto* vstorage = super_version->current->storage_info();
|
||||
auto* ucmp = vstorage->InternalComparator()->user_comparator();
|
||||
InternalKey range_start(ranges[i].start, kMaxSequenceNumber,
|
||||
kValueTypeForSeek);
|
||||
memtable_iter->Seek(range_start.Encode());
|
||||
status = memtable_iter->status();
|
||||
ParsedInternalKey seek_result;
|
||||
if (status.ok()) {
|
||||
if (memtable_iter->Valid() &&
|
||||
!ParseInternalKey(memtable_iter->key(), &seek_result)) {
|
||||
status = Status::Corruption("DB have corrupted keys");
|
||||
}
|
||||
}
|
||||
if (status.ok()) {
|
||||
if (memtable_iter->Valid() &&
|
||||
ucmp->Compare(seek_result.user_key, ranges[i].limit) <= 0) {
|
||||
*overlap = true;
|
||||
} else if (range_del_agg.IsRangeOverlapped(ranges[i].start,
|
||||
ranges[i].limit)) {
|
||||
*overlap = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
return status;
|
||||
}
|
||||
|
||||
const int ColumnFamilyData::kCompactAllLevels = -1;
|
||||
const int ColumnFamilyData::kCompactToBaseLevel = -2;
|
||||
|
||||
Compaction* ColumnFamilyData::CompactRange(
|
||||
const MutableCFOptions& mutable_cf_options, int input_level,
|
||||
int output_level, uint32_t output_path_id, uint32_t max_subcompactions,
|
||||
const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* conflict) {
|
||||
int output_level, uint32_t output_path_id, const InternalKey* begin,
|
||||
const InternalKey* end, InternalKey** compaction_end, bool* conflict) {
|
||||
auto* result = compaction_picker_->CompactRange(
|
||||
GetName(), mutable_cf_options, current_->storage_info(), input_level,
|
||||
output_level, output_path_id, max_subcompactions, begin, end,
|
||||
compaction_end, conflict);
|
||||
output_level, output_path_id, begin, end, compaction_end, conflict);
|
||||
if (result != nullptr) {
|
||||
result->SetInputVersion(current_);
|
||||
}
|
||||
@@ -1004,7 +868,8 @@ Compaction* ColumnFamilyData::CompactRange(
|
||||
|
||||
SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
|
||||
InstrumentedMutex* db_mutex) {
|
||||
SuperVersion* sv = GetThreadLocalSuperVersion(db_mutex);
|
||||
SuperVersion* sv = nullptr;
|
||||
sv = GetThreadLocalSuperVersion(db_mutex);
|
||||
sv->Ref();
|
||||
if (!ReturnThreadLocalSuperVersion(sv)) {
|
||||
// This Unref() corresponds to the Ref() in GetThreadLocalSuperVersion()
|
||||
@@ -1018,6 +883,7 @@ SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
|
||||
|
||||
SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
|
||||
InstrumentedMutex* db_mutex) {
|
||||
SuperVersion* sv = nullptr;
|
||||
// The SuperVersion is cached in thread local storage to avoid acquiring
|
||||
// mutex when SuperVersion does not change since the last use. When a new
|
||||
// SuperVersion is installed, the compaction or flush thread cleans up
|
||||
@@ -1036,7 +902,7 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
|
||||
// should only keep kSVInUse before ReturnThreadLocalSuperVersion call
|
||||
// (if no Scrape happens).
|
||||
assert(ptr != SuperVersion::kSVInUse);
|
||||
SuperVersion* sv = static_cast<SuperVersion*>(ptr);
|
||||
sv = static_cast<SuperVersion*>(ptr);
|
||||
if (sv == SuperVersion::kSVObsolete ||
|
||||
sv->version_number != super_version_number_.load()) {
|
||||
RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_ACQUIRES);
|
||||
@@ -1100,12 +966,6 @@ void ColumnFamilyData::InstallSuperVersion(
|
||||
RecalculateWriteStallConditions(mutable_cf_options);
|
||||
|
||||
if (old_superversion != nullptr) {
|
||||
// Reset SuperVersions cached in thread local storage.
|
||||
// This should be done before old_superversion->Unref(). That's to ensure
|
||||
// that local_sv_ never holds the last reference to SuperVersion, since
|
||||
// it has no means to safely do SuperVersion cleanup.
|
||||
ResetThreadLocalSuperVersions();
|
||||
|
||||
if (old_superversion->mutable_cf_options.write_buffer_size !=
|
||||
mutable_cf_options.write_buffer_size) {
|
||||
mem_->UpdateWriteBufferSize(mutable_cf_options.write_buffer_size);
|
||||
@@ -1121,6 +981,9 @@ void ColumnFamilyData::InstallSuperVersion(
|
||||
sv_context->superversions_to_free.push_back(old_superversion);
|
||||
}
|
||||
}
|
||||
|
||||
// Reset SuperVersions cached in thread local storage
|
||||
ResetThreadLocalSuperVersions();
|
||||
}
|
||||
|
||||
void ColumnFamilyData::ResetThreadLocalSuperVersions() {
|
||||
@@ -1132,12 +995,10 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
|
||||
continue;
|
||||
}
|
||||
auto sv = static_cast<SuperVersion*>(ptr);
|
||||
bool was_last_ref __attribute__((__unused__));
|
||||
was_last_ref = sv->Unref();
|
||||
// sv couldn't have been the last reference because
|
||||
// ResetThreadLocalSuperVersions() is called before
|
||||
// unref'ing super_version_.
|
||||
assert(!was_last_ref);
|
||||
if (sv->Unref()) {
|
||||
sv->Cleanup();
|
||||
delete sv;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1145,9 +1006,8 @@ void ColumnFamilyData::ResetThreadLocalSuperVersions() {
|
||||
Status ColumnFamilyData::SetOptions(
|
||||
const std::unordered_map<std::string, std::string>& options_map) {
|
||||
MutableCFOptions new_mutable_cf_options;
|
||||
Status s =
|
||||
GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
|
||||
ioptions_.info_log, &new_mutable_cf_options);
|
||||
Status s = GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
|
||||
&new_mutable_cf_options);
|
||||
if (s.ok()) {
|
||||
mutable_cf_options_ = new_mutable_cf_options;
|
||||
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
||||
@@ -1174,31 +1034,6 @@ Env::WriteLifeTimeHint ColumnFamilyData::CalculateSSTWriteHint(int level) {
|
||||
static_cast<int>(Env::WLTH_MEDIUM));
|
||||
}
|
||||
|
||||
Status ColumnFamilyData::AddDirectories() {
|
||||
Status s;
|
||||
assert(data_dirs_.empty());
|
||||
for (auto& p : ioptions_.cf_paths) {
|
||||
std::unique_ptr<Directory> path_directory;
|
||||
s = DBImpl::CreateAndNewDirectory(ioptions_.env, p.path, &path_directory);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
assert(path_directory != nullptr);
|
||||
data_dirs_.emplace_back(path_directory.release());
|
||||
}
|
||||
assert(data_dirs_.size() == ioptions_.cf_paths.size());
|
||||
return s;
|
||||
}
|
||||
|
||||
Directory* ColumnFamilyData::GetDataDir(size_t path_id) const {
|
||||
if (data_dirs_.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
assert(path_id < data_dirs_.size());
|
||||
return data_dirs_[path_id].get();
|
||||
}
|
||||
|
||||
ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
|
||||
const ImmutableDBOptions* db_options,
|
||||
const EnvOptions& env_options,
|
||||
@@ -1225,13 +1060,11 @@ ColumnFamilySet::~ColumnFamilySet() {
|
||||
while (column_family_data_.size() > 0) {
|
||||
// cfd destructor will delete itself from column_family_data_
|
||||
auto cfd = column_family_data_.begin()->second;
|
||||
bool last_ref __attribute__((__unused__));
|
||||
last_ref = cfd->Unref();
|
||||
bool last_ref __attribute__((__unused__)) = cfd->Unref();
|
||||
assert(last_ref);
|
||||
delete cfd;
|
||||
}
|
||||
bool dummy_last_ref __attribute__((__unused__));
|
||||
dummy_last_ref = dummy_cfd_->Unref();
|
||||
bool dummy_last_ref __attribute__((__unused__)) = dummy_cfd_->Unref();
|
||||
assert(dummy_last_ref);
|
||||
delete dummy_cfd_;
|
||||
}
|
||||
|
||||
+9
-54
@@ -30,7 +30,6 @@ namespace rocksdb {
|
||||
|
||||
class Version;
|
||||
class VersionSet;
|
||||
class VersionStorageInfo;
|
||||
class MemTable;
|
||||
class MemTableListVersion;
|
||||
class CompactionPicker;
|
||||
@@ -139,9 +138,6 @@ extern Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options);
|
||||
extern Status CheckConcurrentWritesSupported(
|
||||
const ColumnFamilyOptions& cf_options);
|
||||
|
||||
extern Status CheckCFPathsSupported(const DBOptions& db_options,
|
||||
const ColumnFamilyOptions& cf_options);
|
||||
|
||||
extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
|
||||
const ColumnFamilyOptions& src);
|
||||
// Wrap user defined table proproties collector factories `from cf_options`
|
||||
@@ -206,10 +202,6 @@ class ColumnFamilyData {
|
||||
void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
|
||||
uint64_t GetLogNumber() const { return log_number_; }
|
||||
|
||||
void SetFlushReason(FlushReason flush_reason) {
|
||||
flush_reason_ = flush_reason;
|
||||
}
|
||||
FlushReason GetFlushReason() const { return flush_reason_; }
|
||||
// thread-safe
|
||||
const EnvOptions* soptions() const;
|
||||
const ImmutableCFOptions* ioptions() const { return &ioptions_; }
|
||||
@@ -247,7 +239,6 @@ class ColumnFamilyData {
|
||||
void SetCurrent(Version* _current);
|
||||
uint64_t GetNumLiveVersions() const; // REQUIRE: DB mutex held
|
||||
uint64_t GetTotalSstFilesSize() const; // REQUIRE: DB mutex held
|
||||
uint64_t GetLiveSstFilesSize() const; // REQUIRE: DB mutex held
|
||||
void SetMemtable(MemTable* new_mem) {
|
||||
uint64_t memtable_id = last_memtable_id_.fetch_add(1) + 1;
|
||||
new_mem->SetID(memtable_id);
|
||||
@@ -278,16 +269,6 @@ class ColumnFamilyData {
|
||||
const Slice& largest_user_key,
|
||||
int level) const;
|
||||
|
||||
// Check if the passed ranges overlap with any unflushed memtables
|
||||
// (immutable or mutable).
|
||||
//
|
||||
// @param super_version A referenced SuperVersion that will be held for the
|
||||
// duration of this function.
|
||||
//
|
||||
// Thread-safe
|
||||
Status RangesOverlapWithMemtables(const autovector<Range>& ranges,
|
||||
SuperVersion* super_version, bool* overlap);
|
||||
|
||||
// A flag to tell a manual compaction is to compact all levels together
|
||||
// instead of a specific level.
|
||||
static const int kCompactAllLevels;
|
||||
@@ -296,9 +277,9 @@ class ColumnFamilyData {
|
||||
// REQUIRES: DB mutex held
|
||||
Compaction* CompactRange(const MutableCFOptions& mutable_cf_options,
|
||||
int input_level, int output_level,
|
||||
uint32_t output_path_id, uint32_t max_subcompactions,
|
||||
const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* manual_conflict);
|
||||
uint32_t output_path_id, const InternalKey* begin,
|
||||
const InternalKey* end, InternalKey** compaction_end,
|
||||
bool* manual_conflict);
|
||||
|
||||
CompactionPicker* compaction_picker() { return compaction_picker_.get(); }
|
||||
// thread-safe
|
||||
@@ -345,21 +326,10 @@ class ColumnFamilyData {
|
||||
void ResetThreadLocalSuperVersions();
|
||||
|
||||
// Protected by DB mutex
|
||||
void set_queued_for_flush(bool value) { queued_for_flush_ = value; }
|
||||
void set_queued_for_compaction(bool value) { queued_for_compaction_ = value; }
|
||||
bool queued_for_flush() { return queued_for_flush_; }
|
||||
bool queued_for_compaction() { return queued_for_compaction_; }
|
||||
|
||||
enum class WriteStallCause {
|
||||
kNone,
|
||||
kMemtableLimit,
|
||||
kL0FileCountLimit,
|
||||
kPendingCompactionBytes,
|
||||
};
|
||||
static std::pair<WriteStallCondition, WriteStallCause>
|
||||
GetWriteStallConditionAndCause(int num_unflushed_memtables, int num_l0_files,
|
||||
uint64_t num_compaction_needed_bytes,
|
||||
const MutableCFOptions& mutable_cf_options);
|
||||
void set_pending_flush(bool value) { pending_flush_ = value; }
|
||||
void set_pending_compaction(bool value) { pending_compaction_ = value; }
|
||||
bool pending_flush() { return pending_flush_; }
|
||||
bool pending_compaction() { return pending_compaction_; }
|
||||
|
||||
// Recalculate some small conditions, which are changed only during
|
||||
// compaction, adding new memtable and/or
|
||||
@@ -373,18 +343,8 @@ class ColumnFamilyData {
|
||||
|
||||
bool initialized() const { return initialized_.load(); }
|
||||
|
||||
const ColumnFamilyOptions& initial_cf_options() {
|
||||
return initial_cf_options_;
|
||||
}
|
||||
|
||||
Env::WriteLifeTimeHint CalculateSSTWriteHint(int level);
|
||||
|
||||
Status AddDirectories();
|
||||
|
||||
Directory* GetDataDir(size_t path_id) const;
|
||||
|
||||
ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); }
|
||||
|
||||
private:
|
||||
friend class ColumnFamilySet;
|
||||
ColumnFamilyData(uint32_t id, const std::string& name,
|
||||
@@ -444,8 +404,6 @@ class ColumnFamilyData {
|
||||
// recovered from
|
||||
uint64_t log_number_;
|
||||
|
||||
std::atomic<FlushReason> flush_reason_;
|
||||
|
||||
// An object that keeps all the compaction stats
|
||||
// and picks the next compaction
|
||||
std::unique_ptr<CompactionPicker> compaction_picker_;
|
||||
@@ -455,11 +413,11 @@ class ColumnFamilyData {
|
||||
std::unique_ptr<WriteControllerToken> write_controller_token_;
|
||||
|
||||
// If true --> this ColumnFamily is currently present in DBImpl::flush_queue_
|
||||
bool queued_for_flush_;
|
||||
bool pending_flush_;
|
||||
|
||||
// If true --> this ColumnFamily is currently present in
|
||||
// DBImpl::compaction_queue_
|
||||
bool queued_for_compaction_;
|
||||
bool pending_compaction_;
|
||||
|
||||
uint64_t prev_compaction_needed_bytes_;
|
||||
|
||||
@@ -468,9 +426,6 @@ class ColumnFamilyData {
|
||||
|
||||
// Memtable id to track flush.
|
||||
std::atomic<uint64_t> last_memtable_id_;
|
||||
|
||||
// Directories corresponding to cf_paths.
|
||||
std::vector<std::unique_ptr<Directory>> data_dirs_;
|
||||
};
|
||||
|
||||
// ColumnFamilySet has interesting thread-safety requirements
|
||||
|
||||
+181
-375
File diff suppressed because it is too large
Load Diff
+9
-102
@@ -24,7 +24,7 @@ class CompactFilesTest : public testing::Test {
|
||||
public:
|
||||
CompactFilesTest() {
|
||||
env_ = Env::Default();
|
||||
db_name_ = test::PerThreadDBPath("compact_files_test");
|
||||
db_name_ = test::TmpDir(env_) + "/compact_files_test";
|
||||
}
|
||||
|
||||
std::string db_name_;
|
||||
@@ -35,9 +35,10 @@ class CompactFilesTest : public testing::Test {
|
||||
class FlushedFileCollector : public EventListener {
|
||||
public:
|
||||
FlushedFileCollector() {}
|
||||
~FlushedFileCollector() override {}
|
||||
~FlushedFileCollector() {}
|
||||
|
||||
void OnFlushCompleted(DB* /*db*/, const FlushJobInfo& info) override {
|
||||
virtual void OnFlushCompleted(
|
||||
DB* db, const FlushJobInfo& info) override {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
flushed_files_.push_back(info.file_path);
|
||||
}
|
||||
@@ -256,9 +257,9 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
|
||||
TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||
class FilterWithGet : public CompactionFilter {
|
||||
public:
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value,
|
||||
bool* value_changed) const override {
|
||||
if (db_ == nullptr) {
|
||||
return true;
|
||||
}
|
||||
@@ -271,7 +272,7 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||
db_ = db;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "FilterWithGet"; }
|
||||
virtual const char* Name() const override { return "FilterWithGet"; }
|
||||
|
||||
private:
|
||||
DB* db_;
|
||||
@@ -308,100 +309,6 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
|
||||
delete db;
|
||||
}
|
||||
|
||||
TEST_F(CompactFilesTest, SentinelCompressionType) {
|
||||
if (!Zlib_Supported()) {
|
||||
fprintf(stderr, "zlib compression not supported, skip this test\n");
|
||||
return;
|
||||
}
|
||||
if (!Snappy_Supported()) {
|
||||
fprintf(stderr, "snappy compression not supported, skip this test\n");
|
||||
return;
|
||||
}
|
||||
// Check that passing `CompressionType::kDisableCompressionOption` to
|
||||
// `CompactFiles` causes it to use the column family compression options.
|
||||
for (auto compaction_style :
|
||||
{CompactionStyle::kCompactionStyleLevel,
|
||||
CompactionStyle::kCompactionStyleUniversal,
|
||||
CompactionStyle::kCompactionStyleNone}) {
|
||||
DestroyDB(db_name_, Options());
|
||||
Options options;
|
||||
options.compaction_style = compaction_style;
|
||||
// L0: Snappy, L1: ZSTD, L2: Snappy
|
||||
options.compression_per_level = {CompressionType::kSnappyCompression,
|
||||
CompressionType::kZlibCompression,
|
||||
CompressionType::kSnappyCompression};
|
||||
options.create_if_missing = true;
|
||||
FlushedFileCollector* collector = new FlushedFileCollector();
|
||||
options.listeners.emplace_back(collector);
|
||||
DB* db = nullptr;
|
||||
ASSERT_OK(DB::Open(options, db_name_, &db));
|
||||
|
||||
db->Put(WriteOptions(), "key", "val");
|
||||
db->Flush(FlushOptions());
|
||||
|
||||
auto l0_files = collector->GetFlushedFiles();
|
||||
ASSERT_EQ(1, l0_files.size());
|
||||
|
||||
// L0->L1 compaction, so output should be ZSTD-compressed
|
||||
CompactionOptions compaction_opts;
|
||||
compaction_opts.compression = CompressionType::kDisableCompressionOption;
|
||||
ASSERT_OK(db->CompactFiles(compaction_opts, l0_files, 1));
|
||||
|
||||
rocksdb::TablePropertiesCollection all_tables_props;
|
||||
ASSERT_OK(db->GetPropertiesOfAllTables(&all_tables_props));
|
||||
for (const auto& name_and_table_props : all_tables_props) {
|
||||
ASSERT_EQ(CompressionTypeToString(CompressionType::kZlibCompression),
|
||||
name_and_table_props.second->compression_name);
|
||||
}
|
||||
delete db;
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(CompactFilesTest, GetCompactionJobInfo) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
// Disable RocksDB background compaction.
|
||||
options.compaction_style = kCompactionStyleNone;
|
||||
options.level0_slowdown_writes_trigger = 1000;
|
||||
options.level0_stop_writes_trigger = 1000;
|
||||
options.write_buffer_size = 65536;
|
||||
options.max_write_buffer_number = 2;
|
||||
options.compression = kNoCompression;
|
||||
options.max_compaction_bytes = 5000;
|
||||
|
||||
// Add listener
|
||||
FlushedFileCollector* collector = new FlushedFileCollector();
|
||||
options.listeners.emplace_back(collector);
|
||||
|
||||
DB* db = nullptr;
|
||||
DestroyDB(db_name_, options);
|
||||
Status s = DB::Open(options, db_name_, &db);
|
||||
assert(s.ok());
|
||||
assert(db);
|
||||
|
||||
// create couple files
|
||||
for (int i = 0; i < 500; ++i) {
|
||||
db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
|
||||
}
|
||||
reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
|
||||
auto l0_files_1 = collector->GetFlushedFiles();
|
||||
CompactionOptions co;
|
||||
co.compression = CompressionType::kLZ4Compression;
|
||||
CompactionJobInfo compaction_job_info;
|
||||
ASSERT_OK(
|
||||
db->CompactFiles(co, l0_files_1, 0, -1, nullptr, &compaction_job_info));
|
||||
ASSERT_EQ(compaction_job_info.base_input_level, 0);
|
||||
ASSERT_EQ(compaction_job_info.cf_id, db->DefaultColumnFamily()->GetID());
|
||||
ASSERT_EQ(compaction_job_info.cf_name, db->DefaultColumnFamily()->GetName());
|
||||
ASSERT_EQ(compaction_job_info.compaction_reason,
|
||||
CompactionReason::kManualCompaction);
|
||||
ASSERT_EQ(compaction_job_info.compression, CompressionType::kLZ4Compression);
|
||||
ASSERT_EQ(compaction_job_info.output_level, 0);
|
||||
ASSERT_OK(compaction_job_info.status);
|
||||
// no assertion failure
|
||||
delete db;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
@@ -412,7 +319,7 @@ int main(int argc, char** argv) {
|
||||
#else
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) {
|
||||
int main(int argc, char** argv) {
|
||||
fprintf(stderr,
|
||||
"SKIPPED as DBImpl::CompactFiles is not supported in ROCKSDB_LITE\n");
|
||||
return 0;
|
||||
|
||||
+18
-9
@@ -25,12 +25,22 @@ CompactedDBImpl::~CompactedDBImpl() {
|
||||
}
|
||||
|
||||
size_t CompactedDBImpl::FindFile(const Slice& key) {
|
||||
size_t left = 0;
|
||||
size_t right = files_.num_files - 1;
|
||||
auto cmp = [&](const FdWithKeyRange& f, const Slice& k) -> bool {
|
||||
return user_comparator_->Compare(ExtractUserKey(f.largest_key), k) < 0;
|
||||
};
|
||||
return static_cast<size_t>(std::lower_bound(files_.files,
|
||||
files_.files + right, key, cmp) - files_.files);
|
||||
while (left < right) {
|
||||
size_t mid = (left + right) >> 1;
|
||||
const FdWithKeyRange& f = files_.files[mid];
|
||||
if (user_comparator_->Compare(ExtractUserKey(f.largest_key), key) < 0) {
|
||||
// Key at "mid.largest" is < "target". Therefore all
|
||||
// files at or before "mid" are uninteresting.
|
||||
left = mid + 1;
|
||||
} else {
|
||||
// Key at "mid.largest" is >= "target". Therefore all files
|
||||
// after "mid" are uninteresting.
|
||||
right = mid;
|
||||
}
|
||||
}
|
||||
return right;
|
||||
}
|
||||
|
||||
Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
|
||||
@@ -39,8 +49,8 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
|
||||
GetContext::kNotFound, key, value, nullptr, nullptr,
|
||||
nullptr, nullptr);
|
||||
LookupKey lkey(key, kMaxSequenceNumber);
|
||||
files_.files[FindFile(key)].fd.table_reader->Get(options, lkey.internal_key(),
|
||||
&get_context, nullptr);
|
||||
files_.files[FindFile(key)].fd.table_reader->Get(
|
||||
options, lkey.internal_key(), &get_context);
|
||||
if (get_context.State() == GetContext::kFound) {
|
||||
return Status::OK();
|
||||
}
|
||||
@@ -72,7 +82,7 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
|
||||
GetContext::kNotFound, keys[idx], &pinnable_val,
|
||||
nullptr, nullptr, nullptr, nullptr);
|
||||
LookupKey lkey(keys[idx], kMaxSequenceNumber);
|
||||
r->Get(options, lkey.internal_key(), &get_context, nullptr);
|
||||
r->Get(options, lkey.internal_key(), &get_context);
|
||||
value.assign(pinnable_val.data(), pinnable_val.size());
|
||||
if (get_context.State() == GetContext::kFound) {
|
||||
statuses[idx] = Status::OK();
|
||||
@@ -147,7 +157,6 @@ Status CompactedDBImpl::Open(const Options& options,
|
||||
std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
|
||||
Status s = db->Init(options);
|
||||
if (s.ok()) {
|
||||
db->StartTimedTasks();
|
||||
ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
|
||||
"Opened the db as fully compacted mode");
|
||||
LogFlush(db->immutable_db_options_.info_log);
|
||||
|
||||
+23
-25
@@ -32,57 +32,55 @@ class CompactedDBImpl : public DBImpl {
|
||||
override;
|
||||
|
||||
using DBImpl::Put;
|
||||
virtual Status Put(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/, const Slice& /*value*/) override {
|
||||
virtual Status Put(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family, const Slice& key,
|
||||
const Slice& value) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
using DBImpl::Merge;
|
||||
virtual Status Merge(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/, const Slice& /*value*/) override {
|
||||
virtual Status Merge(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family, const Slice& key,
|
||||
const Slice& value) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
using DBImpl::Delete;
|
||||
virtual Status Delete(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/) override {
|
||||
virtual Status Delete(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const Slice& key) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
virtual Status Write(const WriteOptions& /*options*/,
|
||||
WriteBatch* /*updates*/) override {
|
||||
virtual Status Write(const WriteOptions& options,
|
||||
WriteBatch* updates) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
using DBImpl::CompactRange;
|
||||
virtual Status CompactRange(const CompactRangeOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice* /*begin*/,
|
||||
const Slice* /*end*/) override {
|
||||
virtual Status CompactRange(const CompactRangeOptions& options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const Slice* begin, const Slice* end) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
|
||||
virtual Status DisableFileDeletions() override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
virtual Status EnableFileDeletions(bool /*force*/) override {
|
||||
virtual Status EnableFileDeletions(bool force) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
virtual Status GetLiveFiles(std::vector<std::string>& ret,
|
||||
virtual Status GetLiveFiles(std::vector<std::string>&,
|
||||
uint64_t* manifest_file_size,
|
||||
bool /*flush_memtable*/) override {
|
||||
return DBImpl::GetLiveFiles(ret, manifest_file_size,
|
||||
false /* flush_memtable */);
|
||||
bool flush_memtable = true) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
using DBImpl::Flush;
|
||||
virtual Status Flush(const FlushOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/) override {
|
||||
virtual Status Flush(const FlushOptions& options,
|
||||
ColumnFamilyHandle* column_family) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
using DB::IngestExternalFile;
|
||||
virtual Status IngestExternalFile(
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const std::vector<std::string>& /*external_files*/,
|
||||
const IngestExternalFileOptions& /*ingestion_options*/) override {
|
||||
ColumnFamilyHandle* column_family,
|
||||
const std::vector<std::string>& external_files,
|
||||
const IngestExternalFileOptions& ingestion_options) override {
|
||||
return Status::NotSupported("Not supported in compacted db mode.");
|
||||
}
|
||||
|
||||
|
||||
+5
-105
@@ -23,43 +23,6 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
const uint64_t kRangeTombstoneSentinel =
|
||||
PackSequenceAndType(kMaxSequenceNumber, kTypeRangeDeletion);
|
||||
|
||||
int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
|
||||
const InternalKey& b) {
|
||||
auto c = user_cmp->Compare(a.user_key(), b.user_key());
|
||||
if (c != 0) {
|
||||
return c;
|
||||
}
|
||||
auto a_footer = ExtractInternalKeyFooter(a.Encode());
|
||||
auto b_footer = ExtractInternalKeyFooter(b.Encode());
|
||||
if (a_footer == kRangeTombstoneSentinel) {
|
||||
if (b_footer != kRangeTombstoneSentinel) {
|
||||
return -1;
|
||||
}
|
||||
} else if (b_footer == kRangeTombstoneSentinel) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a,
|
||||
const InternalKey& b) {
|
||||
if (a == nullptr) {
|
||||
return -1;
|
||||
}
|
||||
return sstableKeyCompare(user_cmp, *a, b);
|
||||
}
|
||||
|
||||
int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
|
||||
const InternalKey* b) {
|
||||
if (b == nullptr) {
|
||||
return -1;
|
||||
}
|
||||
return sstableKeyCompare(user_cmp, a, *b);
|
||||
}
|
||||
|
||||
uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
|
||||
uint64_t sum = 0;
|
||||
for (size_t i = 0; i < files.size() && files[i]; i++) {
|
||||
@@ -118,49 +81,6 @@ void Compaction::GetBoundaryKeys(
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<CompactionInputFiles> Compaction::PopulateWithAtomicBoundaries(
|
||||
VersionStorageInfo* vstorage, std::vector<CompactionInputFiles> inputs) {
|
||||
const Comparator* ucmp = vstorage->InternalComparator()->user_comparator();
|
||||
for (size_t i = 0; i < inputs.size(); i++) {
|
||||
if (inputs[i].level == 0 || inputs[i].files.empty()) {
|
||||
continue;
|
||||
}
|
||||
inputs[i].atomic_compaction_unit_boundaries.reserve(inputs[i].files.size());
|
||||
AtomicCompactionUnitBoundary cur_boundary;
|
||||
size_t first_atomic_idx = 0;
|
||||
auto add_unit_boundary = [&](size_t to) {
|
||||
if (first_atomic_idx == to) return;
|
||||
for (size_t k = first_atomic_idx; k < to; k++) {
|
||||
inputs[i].atomic_compaction_unit_boundaries.push_back(cur_boundary);
|
||||
}
|
||||
first_atomic_idx = to;
|
||||
};
|
||||
for (size_t j = 0; j < inputs[i].files.size(); j++) {
|
||||
const auto* f = inputs[i].files[j];
|
||||
if (j == 0) {
|
||||
// First file in a level.
|
||||
cur_boundary.smallest = &f->smallest;
|
||||
cur_boundary.largest = &f->largest;
|
||||
} else if (sstableKeyCompare(ucmp, *cur_boundary.largest, f->smallest) ==
|
||||
0) {
|
||||
// SSTs overlap but the end key of the previous file was not
|
||||
// artificially extended by a range tombstone. Extend the current
|
||||
// boundary.
|
||||
cur_boundary.largest = &f->largest;
|
||||
} else {
|
||||
// Atomic compaction unit has ended.
|
||||
add_unit_boundary(j);
|
||||
cur_boundary.smallest = &f->smallest;
|
||||
cur_boundary.largest = &f->largest;
|
||||
}
|
||||
}
|
||||
add_unit_boundary(inputs[i].files.size());
|
||||
assert(inputs[i].files.size() ==
|
||||
inputs[i].atomic_compaction_unit_boundaries.size());
|
||||
}
|
||||
return inputs;
|
||||
}
|
||||
|
||||
// helper function to determine if compaction is creating files at the
|
||||
// bottommost level
|
||||
bool Compaction::IsBottommostLevel(
|
||||
@@ -214,8 +134,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
int _output_level, uint64_t _target_file_size,
|
||||
uint64_t _max_compaction_bytes, uint32_t _output_path_id,
|
||||
CompressionType _compression,
|
||||
CompressionOptions _compression_opts,
|
||||
uint32_t _max_subcompactions,
|
||||
std::vector<FileMetaData*> _grandparents,
|
||||
bool _manual_compaction, double _score,
|
||||
bool _deletion_compaction,
|
||||
@@ -225,7 +143,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
output_level_(_output_level),
|
||||
max_output_file_size_(_target_file_size),
|
||||
max_compaction_bytes_(_max_compaction_bytes),
|
||||
max_subcompactions_(_max_subcompactions),
|
||||
immutable_cf_options_(_immutable_cf_options),
|
||||
mutable_cf_options_(_mutable_cf_options),
|
||||
input_version_(nullptr),
|
||||
@@ -233,9 +150,8 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
cfd_(nullptr),
|
||||
output_path_id_(_output_path_id),
|
||||
output_compression_(_compression),
|
||||
output_compression_opts_(_compression_opts),
|
||||
deletion_compaction_(_deletion_compaction),
|
||||
inputs_(PopulateWithAtomicBoundaries(vstorage, std::move(_inputs))),
|
||||
inputs_(std::move(_inputs)),
|
||||
grandparents_(std::move(_grandparents)),
|
||||
score_(_score),
|
||||
bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
|
||||
@@ -247,15 +163,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
|
||||
if (is_manual_compaction_) {
|
||||
compaction_reason_ = CompactionReason::kManualCompaction;
|
||||
}
|
||||
if (max_subcompactions_ == 0) {
|
||||
max_subcompactions_ = immutable_cf_options_.max_subcompactions;
|
||||
}
|
||||
if (!bottommost_level_) {
|
||||
// Currently we only enable dictionary compression during compaction to the
|
||||
// bottommost level.
|
||||
output_compression_opts_.max_dict_bytes = 0;
|
||||
output_compression_opts_.zstd_max_train_bytes = 0;
|
||||
}
|
||||
|
||||
#ifndef NDEBUG
|
||||
for (size_t i = 1; i < inputs_.size(); ++i) {
|
||||
@@ -379,8 +286,8 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
|
||||
if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
|
||||
// We've advanced far enough
|
||||
if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
|
||||
// Key falls in this file's range, so it may
|
||||
// exist beyond output level
|
||||
// Key falls in this file's range, so definitely
|
||||
// exists beyond output level
|
||||
return false;
|
||||
}
|
||||
break;
|
||||
@@ -417,14 +324,12 @@ const char* Compaction::InputLevelSummary(
|
||||
if (!is_first) {
|
||||
len +=
|
||||
snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, " + ");
|
||||
len = std::min(len, static_cast<int>(sizeof(scratch->buffer)));
|
||||
} else {
|
||||
is_first = false;
|
||||
}
|
||||
len += snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
|
||||
"%" ROCKSDB_PRIszt "@%d", input_level.size(),
|
||||
input_level.level);
|
||||
len = std::min(len, static_cast<int>(sizeof(scratch->buffer)));
|
||||
}
|
||||
snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len,
|
||||
" files to L%d", output_level());
|
||||
@@ -537,12 +442,11 @@ bool Compaction::IsOutputLevelEmpty() const {
|
||||
}
|
||||
|
||||
bool Compaction::ShouldFormSubcompactions() const {
|
||||
if (max_subcompactions_ <= 1 || cfd_ == nullptr) {
|
||||
if (immutable_cf_options_.max_subcompactions <= 1 || cfd_ == nullptr) {
|
||||
return false;
|
||||
}
|
||||
if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
|
||||
return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0 &&
|
||||
!IsOutputLevelEmpty();
|
||||
return start_level_ == 0 && output_level_ > 0 && !IsOutputLevelEmpty();
|
||||
} else if (cfd_->ioptions()->compaction_style == kCompactionStyleUniversal) {
|
||||
return number_levels_ > 1 && output_level_ > 0;
|
||||
} else {
|
||||
@@ -563,8 +467,4 @@ uint64_t Compaction::MaxInputFileCreationTime() const {
|
||||
return max_creation_time;
|
||||
}
|
||||
|
||||
int Compaction::GetInputBaseLevel() const {
|
||||
return input_vstorage_->base_level();
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@@ -15,43 +15,11 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
// Utility for comparing sstable boundary keys. Returns -1 if either a or b is
|
||||
// null which provides the property that a==null indicates a key that is less
|
||||
// than any key and b==null indicates a key that is greater than any key. Note
|
||||
// that the comparison is performed primarily on the user-key portion of the
|
||||
// key. If the user-keys compare equal, an additional test is made to sort
|
||||
// range tombstone sentinel keys before other keys with the same user-key. The
|
||||
// result is that 2 user-keys will compare equal if they differ purely on
|
||||
// their sequence number and value, but the range tombstone sentinel for that
|
||||
// user-key will compare not equal. This is necessary because the range
|
||||
// tombstone sentinel key is set as the largest key for an sstable even though
|
||||
// that key never appears in the database. We don't want adjacent sstables to
|
||||
// be considered overlapping if they are separated by the range tombstone
|
||||
// sentinel.
|
||||
int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
|
||||
const InternalKey& b);
|
||||
int sstableKeyCompare(const Comparator* user_cmp, const InternalKey* a,
|
||||
const InternalKey& b);
|
||||
int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
|
||||
const InternalKey* b);
|
||||
|
||||
// An AtomicCompactionUnitBoundary represents a range of keys [smallest,
|
||||
// largest] that exactly spans one ore more neighbouring SSTs on the same
|
||||
// level. Every pair of SSTs in this range "overlap" (i.e., the largest
|
||||
// user key of one file is the smallest user key of the next file). These
|
||||
// boundaries are propagated down to RangeDelAggregator during compaction
|
||||
// to provide safe truncation boundaries for range tombstones.
|
||||
struct AtomicCompactionUnitBoundary {
|
||||
const InternalKey* smallest = nullptr;
|
||||
const InternalKey* largest = nullptr;
|
||||
};
|
||||
|
||||
// The structure that manages compaction input files associated
|
||||
// with the same physical level.
|
||||
struct CompactionInputFiles {
|
||||
int level;
|
||||
std::vector<FileMetaData*> files;
|
||||
std::vector<AtomicCompactionUnitBoundary> atomic_compaction_unit_boundaries;
|
||||
inline bool empty() const { return files.empty(); }
|
||||
inline size_t size() const { return files.size(); }
|
||||
inline void clear() { files.clear(); }
|
||||
@@ -72,7 +40,6 @@ class Compaction {
|
||||
std::vector<CompactionInputFiles> inputs, int output_level,
|
||||
uint64_t target_file_size, uint64_t max_compaction_bytes,
|
||||
uint32_t output_path_id, CompressionType compression,
|
||||
CompressionOptions compression_opts, uint32_t max_subcompactions,
|
||||
std::vector<FileMetaData*> grandparents,
|
||||
bool manual_compaction = false, double score = -1,
|
||||
bool deletion_compaction = false,
|
||||
@@ -128,12 +95,6 @@ class Compaction {
|
||||
return inputs_[compaction_input_level][i];
|
||||
}
|
||||
|
||||
const std::vector<AtomicCompactionUnitBoundary>* boundaries(
|
||||
size_t compaction_input_level) const {
|
||||
assert(compaction_input_level < inputs_.size());
|
||||
return &inputs_[compaction_input_level].atomic_compaction_unit_boundaries;
|
||||
}
|
||||
|
||||
// Returns the list of file meta data of the specified compaction
|
||||
// input level.
|
||||
// REQUIREMENT: "compaction_input_level" must be >= 0 and
|
||||
@@ -157,11 +118,6 @@ class Compaction {
|
||||
// What compression for output
|
||||
CompressionType output_compression() const { return output_compression_; }
|
||||
|
||||
// What compression options for output
|
||||
CompressionOptions output_compression_opts() const {
|
||||
return output_compression_opts_;
|
||||
}
|
||||
|
||||
// Whether need to write output file to second DB path.
|
||||
uint32_t output_path_id() const { return output_path_id_; }
|
||||
|
||||
@@ -277,8 +233,6 @@ class Compaction {
|
||||
|
||||
Slice GetLargestUserKey() const { return largest_user_key_; }
|
||||
|
||||
int GetInputBaseLevel() const;
|
||||
|
||||
CompactionReason compaction_reason() { return compaction_reason_; }
|
||||
|
||||
const std::vector<FileMetaData*>& grandparents() const {
|
||||
@@ -287,8 +241,6 @@ class Compaction {
|
||||
|
||||
uint64_t max_compaction_bytes() const { return max_compaction_bytes_; }
|
||||
|
||||
uint32_t max_subcompactions() const { return max_subcompactions_; }
|
||||
|
||||
uint64_t MaxInputFileCreationTime() const;
|
||||
|
||||
private:
|
||||
@@ -300,13 +252,6 @@ class Compaction {
|
||||
const std::vector<CompactionInputFiles>& inputs,
|
||||
Slice* smallest_key, Slice* largest_key);
|
||||
|
||||
// Get the atomic file boundaries for all files in the compaction. Necessary
|
||||
// in order to avoid the scenario described in
|
||||
// https://github.com/facebook/rocksdb/pull/4432#discussion_r221072219 and plumb
|
||||
// down appropriate key boundaries to RangeDelAggregator during compaction.
|
||||
static std::vector<CompactionInputFiles> PopulateWithAtomicBoundaries(
|
||||
VersionStorageInfo* vstorage, std::vector<CompactionInputFiles> inputs);
|
||||
|
||||
// helper function to determine if compaction with inputs and storage is
|
||||
// bottommost
|
||||
static bool IsBottommostLevel(
|
||||
@@ -322,7 +267,6 @@ class Compaction {
|
||||
const int output_level_; // levels to which output files are stored
|
||||
uint64_t max_output_file_size_;
|
||||
uint64_t max_compaction_bytes_;
|
||||
uint32_t max_subcompactions_;
|
||||
const ImmutableCFOptions immutable_cf_options_;
|
||||
const MutableCFOptions mutable_cf_options_;
|
||||
Version* input_version_;
|
||||
@@ -333,7 +277,6 @@ class Compaction {
|
||||
|
||||
const uint32_t output_path_id_;
|
||||
CompressionType output_compression_;
|
||||
CompressionOptions output_compression_opts_;
|
||||
// If true, then the comaction can be done by simply deleting input files.
|
||||
const bool deletion_compaction_;
|
||||
|
||||
|
||||
+120
-158
@@ -9,55 +9,65 @@
|
||||
#include "port/likely.h"
|
||||
#include "rocksdb/listener.h"
|
||||
#include "table/internal_iterator.h"
|
||||
#include "util/sync_point.h"
|
||||
|
||||
#define DEFINITELY_IN_SNAPSHOT(seq, snapshot) \
|
||||
((seq) <= (snapshot) && \
|
||||
(snapshot_checker_ == nullptr || \
|
||||
LIKELY(snapshot_checker_->CheckInSnapshot((seq), (snapshot)) == \
|
||||
SnapshotCheckerResult::kInSnapshot)))
|
||||
|
||||
#define DEFINITELY_NOT_IN_SNAPSHOT(seq, snapshot) \
|
||||
((seq) > (snapshot) || \
|
||||
(snapshot_checker_ != nullptr && \
|
||||
UNLIKELY(snapshot_checker_->CheckInSnapshot((seq), (snapshot)) == \
|
||||
SnapshotCheckerResult::kNotInSnapshot)))
|
||||
|
||||
#define IN_EARLIEST_SNAPSHOT(seq) \
|
||||
((seq) <= earliest_snapshot_ && \
|
||||
(snapshot_checker_ == nullptr || LIKELY(IsInEarliestSnapshot(seq))))
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
CompactionEventListener::CompactionListenerValueType fromInternalValueType(
|
||||
ValueType vt) {
|
||||
switch (vt) {
|
||||
case kTypeDeletion:
|
||||
return CompactionEventListener::CompactionListenerValueType::kDelete;
|
||||
case kTypeValue:
|
||||
return CompactionEventListener::CompactionListenerValueType::kValue;
|
||||
case kTypeMerge:
|
||||
return CompactionEventListener::CompactionListenerValueType::
|
||||
kMergeOperand;
|
||||
case kTypeSingleDeletion:
|
||||
return CompactionEventListener::CompactionListenerValueType::
|
||||
kSingleDelete;
|
||||
case kTypeRangeDeletion:
|
||||
return CompactionEventListener::CompactionListenerValueType::kRangeDelete;
|
||||
case kTypeBlobIndex:
|
||||
return CompactionEventListener::CompactionListenerValueType::kBlobIndex;
|
||||
default:
|
||||
assert(false);
|
||||
return CompactionEventListener::CompactionListenerValueType::kInvalid;
|
||||
}
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
CompactionIterator::CompactionIterator(
|
||||
InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
|
||||
SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const SnapshotChecker* snapshot_checker, Env* env,
|
||||
bool report_detailed_time, bool expect_valid_internal_key,
|
||||
CompactionRangeDelAggregator* range_del_agg, const Compaction* compaction,
|
||||
const CompactionFilter* compaction_filter,
|
||||
bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
|
||||
const Compaction* compaction, const CompactionFilter* compaction_filter,
|
||||
CompactionEventListener* compaction_listener,
|
||||
const std::atomic<bool>* shutting_down,
|
||||
const SequenceNumber preserve_deletes_seqnum)
|
||||
: CompactionIterator(
|
||||
input, cmp, merge_helper, last_sequence, snapshots,
|
||||
earliest_write_conflict_snapshot, snapshot_checker, env,
|
||||
report_detailed_time, expect_valid_internal_key, range_del_agg,
|
||||
expect_valid_internal_key, range_del_agg,
|
||||
std::unique_ptr<CompactionProxy>(
|
||||
compaction ? new CompactionProxy(compaction) : nullptr),
|
||||
compaction_filter, shutting_down, preserve_deletes_seqnum) {}
|
||||
compaction_filter, compaction_listener, shutting_down,
|
||||
preserve_deletes_seqnum) {}
|
||||
|
||||
CompactionIterator::CompactionIterator(
|
||||
InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
|
||||
SequenceNumber /*last_sequence*/, std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const SnapshotChecker* snapshot_checker, Env* env,
|
||||
bool report_detailed_time, bool expect_valid_internal_key,
|
||||
CompactionRangeDelAggregator* range_del_agg,
|
||||
bool expect_valid_internal_key, RangeDelAggregator* range_del_agg,
|
||||
std::unique_ptr<CompactionProxy> compaction,
|
||||
const CompactionFilter* compaction_filter,
|
||||
CompactionEventListener* compaction_listener,
|
||||
const std::atomic<bool>* shutting_down,
|
||||
const SequenceNumber preserve_deletes_seqnum)
|
||||
const SequenceNumber preserve_deletes_seqnum
|
||||
)
|
||||
: input_(input),
|
||||
cmp_(cmp),
|
||||
merge_helper_(merge_helper),
|
||||
@@ -65,44 +75,45 @@ CompactionIterator::CompactionIterator(
|
||||
earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
|
||||
snapshot_checker_(snapshot_checker),
|
||||
env_(env),
|
||||
report_detailed_time_(report_detailed_time),
|
||||
expect_valid_internal_key_(expect_valid_internal_key),
|
||||
range_del_agg_(range_del_agg),
|
||||
compaction_(std::move(compaction)),
|
||||
compaction_filter_(compaction_filter),
|
||||
#ifndef ROCKSDB_LITE
|
||||
compaction_listener_(compaction_listener),
|
||||
#endif // ROCKSDB_LITE
|
||||
shutting_down_(shutting_down),
|
||||
preserve_deletes_seqnum_(preserve_deletes_seqnum),
|
||||
ignore_snapshots_(false),
|
||||
current_user_key_sequence_(0),
|
||||
current_user_key_snapshot_(0),
|
||||
merge_out_iter_(merge_helper_),
|
||||
current_key_committed_(false) {
|
||||
assert(compaction_filter_ == nullptr || compaction_ != nullptr);
|
||||
assert(snapshots_ != nullptr);
|
||||
bottommost_level_ =
|
||||
compaction_ == nullptr ? false : compaction_->bottommost_level();
|
||||
if (compaction_ != nullptr) {
|
||||
level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
|
||||
}
|
||||
|
||||
if (snapshots_->size() == 0) {
|
||||
// optimize for fast path if there are no snapshots
|
||||
visible_at_tip_ = true;
|
||||
earliest_snapshot_iter_ = snapshots_->end();
|
||||
earliest_snapshot_ = kMaxSequenceNumber;
|
||||
latest_snapshot_ = 0;
|
||||
} else {
|
||||
visible_at_tip_ = false;
|
||||
earliest_snapshot_iter_ = snapshots_->begin();
|
||||
earliest_snapshot_ = snapshots_->at(0);
|
||||
latest_snapshot_ = snapshots_->back();
|
||||
}
|
||||
#ifndef NDEBUG
|
||||
// findEarliestVisibleSnapshot assumes this ordering.
|
||||
for (size_t i = 1; i < snapshots_->size(); ++i) {
|
||||
assert(snapshots_->at(i - 1) < snapshots_->at(i));
|
||||
if (compaction_filter_ != nullptr) {
|
||||
if (compaction_filter_->IgnoreSnapshots()) {
|
||||
ignore_snapshots_ = true;
|
||||
}
|
||||
} else {
|
||||
ignore_snapshots_ = false;
|
||||
}
|
||||
#endif
|
||||
input_->SetPinnedItersMgr(&pinned_iters_mgr_);
|
||||
TEST_SYNC_POINT_CALLBACK("CompactionIterator:AfterInit", compaction_.get());
|
||||
}
|
||||
|
||||
CompactionIterator::~CompactionIterator() {
|
||||
@@ -172,7 +183,12 @@ void CompactionIterator::Next() {
|
||||
void CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
|
||||
Slice* skip_until) {
|
||||
if (compaction_filter_ != nullptr &&
|
||||
(ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex)) {
|
||||
(ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex) &&
|
||||
(visible_at_tip_ || ignore_snapshots_ ||
|
||||
ikey_.sequence > latest_snapshot_ ||
|
||||
(snapshot_checker_ != nullptr &&
|
||||
UNLIKELY(!snapshot_checker_->IsInSnapshot(ikey_.sequence,
|
||||
latest_snapshot_))))) {
|
||||
// If the user has specified a compaction filter and the sequence
|
||||
// number is greater than any external snapshot, then invoke the
|
||||
// filter. If the return value of the compaction filter is true,
|
||||
@@ -183,16 +199,13 @@ void CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
|
||||
CompactionFilter::ValueType value_type =
|
||||
ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
|
||||
: CompactionFilter::ValueType::kBlobIndex;
|
||||
// Hack: pass internal key to BlobIndexCompactionFilter since it needs
|
||||
// to get sequence number.
|
||||
Slice& filter_key = ikey_.type == kTypeValue ? ikey_.user_key : key_;
|
||||
{
|
||||
StopWatchNano timer(env_, report_detailed_time_);
|
||||
StopWatchNano timer(env_, true);
|
||||
filter = compaction_filter_->FilterV2(
|
||||
compaction_->level(), filter_key, value_type, value_,
|
||||
compaction_->level(), ikey_.user_key, value_type, value_,
|
||||
&compaction_filter_value_, compaction_filter_skip_until_.rep());
|
||||
iter_stats_.total_filter_time +=
|
||||
env_ != nullptr && report_detailed_time_ ? timer.ElapsedNanos() : 0;
|
||||
env_ != nullptr ? timer.ElapsedNanos() : 0;
|
||||
}
|
||||
|
||||
if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
|
||||
@@ -248,7 +261,6 @@ void CompactionIterator::NextFromInput() {
|
||||
valid_ = true;
|
||||
break;
|
||||
}
|
||||
TEST_SYNC_POINT_CALLBACK("CompactionIterator:ProcessKV", &ikey_);
|
||||
|
||||
// Update input statistics
|
||||
if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
|
||||
@@ -277,7 +289,17 @@ void CompactionIterator::NextFromInput() {
|
||||
has_outputted_key_ = false;
|
||||
current_user_key_sequence_ = kMaxSequenceNumber;
|
||||
current_user_key_snapshot_ = 0;
|
||||
current_key_committed_ = KeyCommitted(ikey_.sequence);
|
||||
current_key_committed_ =
|
||||
(snapshot_checker_ == nullptr ||
|
||||
snapshot_checker_->IsInSnapshot(ikey_.sequence, kMaxSequenceNumber));
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
if (compaction_listener_) {
|
||||
compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
|
||||
fromInternalValueType(ikey_.type),
|
||||
value_, ikey_.sequence, true);
|
||||
}
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
// Apply the compaction filter to the first committed version of the user
|
||||
// key.
|
||||
@@ -285,6 +307,14 @@ void CompactionIterator::NextFromInput() {
|
||||
InvokeFilterIfNeeded(&need_skip, &skip_until);
|
||||
}
|
||||
} else {
|
||||
#ifndef ROCKSDB_LITE
|
||||
if (compaction_listener_) {
|
||||
compaction_listener_->OnCompaction(compaction_->level(), ikey_.user_key,
|
||||
fromInternalValueType(ikey_.type),
|
||||
value_, ikey_.sequence, false);
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
// Update the current key to reflect the new sequence number/type without
|
||||
// copying the user key.
|
||||
// TODO(rven): Compaction filter does not process keys in this path
|
||||
@@ -299,7 +329,8 @@ void CompactionIterator::NextFromInput() {
|
||||
// to query snapshot_checker_ in that case.
|
||||
if (UNLIKELY(!current_key_committed_)) {
|
||||
assert(snapshot_checker_ != nullptr);
|
||||
current_key_committed_ = KeyCommitted(ikey_.sequence);
|
||||
current_key_committed_ =
|
||||
snapshot_checker_->IsInSnapshot(ikey_.sequence, kMaxSequenceNumber);
|
||||
// Apply the compaction filter to the first committed version of the
|
||||
// user key.
|
||||
if (current_key_committed_) {
|
||||
@@ -383,8 +414,10 @@ void CompactionIterator::NextFromInput() {
|
||||
cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
|
||||
// Check whether the next key belongs to the same snapshot as the
|
||||
// SingleDelete.
|
||||
if (prev_snapshot == 0 ||
|
||||
DEFINITELY_NOT_IN_SNAPSHOT(next_ikey.sequence, prev_snapshot)) {
|
||||
if (prev_snapshot == 0 || next_ikey.sequence > prev_snapshot ||
|
||||
(snapshot_checker_ != nullptr &&
|
||||
UNLIKELY(!snapshot_checker_->IsInSnapshot(next_ikey.sequence,
|
||||
prev_snapshot)))) {
|
||||
if (next_ikey.type == kTypeSingleDeletion) {
|
||||
// We encountered two SingleDeletes in a row. This could be due to
|
||||
// unexpected user input.
|
||||
@@ -396,8 +429,11 @@ void CompactionIterator::NextFromInput() {
|
||||
++iter_stats_.num_record_drop_obsolete;
|
||||
++iter_stats_.num_single_del_mismatch;
|
||||
} else if (has_outputted_key_ ||
|
||||
DEFINITELY_IN_SNAPSHOT(
|
||||
ikey_.sequence, earliest_write_conflict_snapshot_)) {
|
||||
(ikey_.sequence <= earliest_write_conflict_snapshot_ &&
|
||||
(snapshot_checker_ == nullptr ||
|
||||
LIKELY(snapshot_checker_->IsInSnapshot(
|
||||
ikey_.sequence,
|
||||
earliest_write_conflict_snapshot_))))) {
|
||||
// Found a matching value, we can drop the single delete and the
|
||||
// value. It is safe to drop both records since we've already
|
||||
// outputted a key in this snapshot, or there is no earlier
|
||||
@@ -407,8 +443,7 @@ void CompactionIterator::NextFromInput() {
|
||||
// is an unexpected Merge or Delete. We will compact it out
|
||||
// either way. We will maintain counts of how many mismatches
|
||||
// happened
|
||||
if (next_ikey.type != kTypeValue &&
|
||||
next_ikey.type != kTypeBlobIndex) {
|
||||
if (next_ikey.type != kTypeValue) {
|
||||
++iter_stats_.num_single_del_mismatch;
|
||||
}
|
||||
|
||||
@@ -445,7 +480,10 @@ void CompactionIterator::NextFromInput() {
|
||||
// iteration. If the next key is corrupt, we return before the
|
||||
// comparison, so the value of has_current_user_key does not matter.
|
||||
has_current_user_key_ = false;
|
||||
if (compaction_ != nullptr && IN_EARLIEST_SNAPSHOT(ikey_.sequence) &&
|
||||
if (compaction_ != nullptr && ikey_.sequence <= earliest_snapshot_ &&
|
||||
(snapshot_checker_ == nullptr ||
|
||||
LIKELY(snapshot_checker_->IsInSnapshot(ikey_.sequence,
|
||||
earliest_snapshot_))) &&
|
||||
compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
|
||||
&level_ptrs_)) {
|
||||
// Key doesn't exist outside of this range.
|
||||
@@ -464,37 +502,24 @@ void CompactionIterator::NextFromInput() {
|
||||
if (valid_) {
|
||||
at_next_ = true;
|
||||
}
|
||||
} else if (last_snapshot == current_user_key_snapshot_ ||
|
||||
(last_snapshot > 0 &&
|
||||
last_snapshot < current_user_key_snapshot_)) {
|
||||
} else if (last_snapshot == current_user_key_snapshot_) {
|
||||
// If the earliest snapshot is which this key is visible in
|
||||
// is the same as the visibility of a previous instance of the
|
||||
// same key, then this kv is not visible in any snapshot.
|
||||
// Hidden by an newer entry for same user key
|
||||
// TODO(noetzli): why not > ?
|
||||
//
|
||||
// Note: Dropping this key will not affect TransactionDB write-conflict
|
||||
// checking since there has already been a record returned for this key
|
||||
// in this snapshot.
|
||||
assert(last_sequence >= current_user_key_sequence_);
|
||||
|
||||
// Note2: if last_snapshot < current_user_key_snapshot, it can only
|
||||
// mean last_snapshot is released between we process last value and
|
||||
// this value, and findEarliestVisibleSnapshot returns the next snapshot
|
||||
// as current_user_key_snapshot. In this case last value and current
|
||||
// value are both in current_user_key_snapshot currently.
|
||||
// Although last_snapshot is released we might still get a definitive
|
||||
// response when key sequence number changes, e.g., when seq is determined
|
||||
// too old and visible in all snapshots.
|
||||
assert(last_snapshot == current_user_key_snapshot_ ||
|
||||
(snapshot_checker_ != nullptr &&
|
||||
snapshot_checker_->CheckInSnapshot(current_user_key_sequence_,
|
||||
last_snapshot) !=
|
||||
SnapshotCheckerResult::kNotInSnapshot));
|
||||
|
||||
++iter_stats_.num_record_drop_hidden; // (A)
|
||||
input_->Next();
|
||||
} else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
|
||||
IN_EARLIEST_SNAPSHOT(ikey_.sequence) &&
|
||||
ikey_.sequence <= earliest_snapshot_ &&
|
||||
(snapshot_checker_ == nullptr ||
|
||||
LIKELY(snapshot_checker_->IsInSnapshot(ikey_.sequence,
|
||||
earliest_snapshot_))) &&
|
||||
ikeyNotNeededForIncrementalSnapshot() &&
|
||||
compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
|
||||
&level_ptrs_)) {
|
||||
@@ -512,38 +537,11 @@ void CompactionIterator::NextFromInput() {
|
||||
//
|
||||
// Note: Dropping this Delete will not affect TransactionDB
|
||||
// write-conflict checking since it is earlier than any snapshot.
|
||||
//
|
||||
// It seems that we can also drop deletion later than earliest snapshot
|
||||
// given that:
|
||||
// (1) The deletion is earlier than earliest_write_conflict_snapshot, and
|
||||
// (2) No value exist earlier than the deletion.
|
||||
++iter_stats_.num_record_drop_obsolete;
|
||||
if (!bottommost_level_) {
|
||||
++iter_stats_.num_optimized_del_drop_obsolete;
|
||||
}
|
||||
input_->Next();
|
||||
} else if ((ikey_.type == kTypeDeletion) && bottommost_level_ &&
|
||||
ikeyNotNeededForIncrementalSnapshot()) {
|
||||
// Handle the case where we have a delete key at the bottom most level
|
||||
// We can skip outputting the key iff there are no subsequent puts for this
|
||||
// key
|
||||
ParsedInternalKey next_ikey;
|
||||
input_->Next();
|
||||
// Skip over all versions of this key that happen to occur in the same snapshot
|
||||
// range as the delete
|
||||
while (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
|
||||
cmp_->Equal(ikey_.user_key, next_ikey.user_key) &&
|
||||
(prev_snapshot == 0 ||
|
||||
DEFINITELY_NOT_IN_SNAPSHOT(next_ikey.sequence, prev_snapshot))) {
|
||||
input_->Next();
|
||||
}
|
||||
// If you find you still need to output a row with this key, we need to output the
|
||||
// delete too
|
||||
if (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
|
||||
cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
|
||||
valid_ = true;
|
||||
at_next_ = true;
|
||||
}
|
||||
} else if (ikey_.type == kTypeMerge) {
|
||||
if (!merge_helper_->HasOperator()) {
|
||||
status_ = Status::InvalidArgument(
|
||||
@@ -556,6 +554,10 @@ void CompactionIterator::NextFromInput() {
|
||||
// have hit (A)
|
||||
// We encapsulate the merge related state machine in a different
|
||||
// object to minimize change to the existing flow.
|
||||
// In case snapshot_checker is present, we can probably merge further
|
||||
// beyond prev_snapshot, since there could be more keys with sequence
|
||||
// smaller than prev_snapshot, but reported by snapshot_checker as not
|
||||
// visible by prev_snapshot. But it will make the logic more complicated.
|
||||
Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
|
||||
prev_snapshot, bottommost_level_);
|
||||
merge_out_iter_.SeekToFirst();
|
||||
@@ -593,7 +595,7 @@ void CompactionIterator::NextFromInput() {
|
||||
// 1. new user key -OR-
|
||||
// 2. different snapshot stripe
|
||||
bool should_delete = range_del_agg_->ShouldDelete(
|
||||
key_, RangeDelPositioningMode::kForwardTraversal);
|
||||
key_, RangeDelAggregator::RangePositioningMode::kForwardTraversal);
|
||||
if (should_delete) {
|
||||
++iter_stats_.num_record_drop_hidden;
|
||||
++iter_stats_.num_record_drop_range_del;
|
||||
@@ -619,15 +621,17 @@ void CompactionIterator::PrepareOutput() {
|
||||
// and the earliest snapshot is larger than this seqno
|
||||
// and the userkey differs from the last userkey in compaction
|
||||
// then we can squash the seqno to zero.
|
||||
//
|
||||
|
||||
// This is safe for TransactionDB write-conflict checking since transactions
|
||||
// only care about sequence number larger than any active snapshots.
|
||||
//
|
||||
// Can we do the same for levels above bottom level as long as
|
||||
// KeyNotExistsBeyondOutputLevel() return true?
|
||||
if ((compaction_ != nullptr && !compaction_->allow_ingest_behind()) &&
|
||||
ikeyNotNeededForIncrementalSnapshot() && bottommost_level_ && valid_ &&
|
||||
IN_EARLIEST_SNAPSHOT(ikey_.sequence) && ikey_.type != kTypeMerge) {
|
||||
if ((compaction_ != nullptr &&
|
||||
!compaction_->allow_ingest_behind()) &&
|
||||
ikeyNotNeededForIncrementalSnapshot() &&
|
||||
bottommost_level_ && valid_ && ikey_.sequence <= earliest_snapshot_ &&
|
||||
(snapshot_checker_ == nullptr || LIKELY(snapshot_checker_->IsInSnapshot(
|
||||
ikey_.sequence, earliest_snapshot_))) &&
|
||||
ikey_.type != kTypeMerge &&
|
||||
!cmp_->Equal(compaction_->GetLargestUserKey(), ikey_.user_key)) {
|
||||
assert(ikey_.type != kTypeDeletion && ikey_.type != kTypeSingleDeletion);
|
||||
ikey_.sequence = 0;
|
||||
current_key_.UpdateInternalKey(0, ikey_.type);
|
||||
@@ -637,34 +641,18 @@ void CompactionIterator::PrepareOutput() {
|
||||
inline SequenceNumber CompactionIterator::findEarliestVisibleSnapshot(
|
||||
SequenceNumber in, SequenceNumber* prev_snapshot) {
|
||||
assert(snapshots_->size());
|
||||
auto snapshots_iter = std::lower_bound(
|
||||
snapshots_->begin(), snapshots_->end(), in);
|
||||
if (snapshots_iter == snapshots_->begin()) {
|
||||
*prev_snapshot = 0;
|
||||
} else {
|
||||
*prev_snapshot = *std::prev(snapshots_iter);
|
||||
assert(*prev_snapshot < in);
|
||||
}
|
||||
if (snapshot_checker_ == nullptr) {
|
||||
return snapshots_iter != snapshots_->end()
|
||||
? *snapshots_iter : kMaxSequenceNumber;
|
||||
}
|
||||
bool has_released_snapshot = !released_snapshots_.empty();
|
||||
for (; snapshots_iter != snapshots_->end(); ++snapshots_iter) {
|
||||
auto cur = *snapshots_iter;
|
||||
assert(in <= cur);
|
||||
// Skip if cur is in released_snapshots.
|
||||
if (has_released_snapshot && released_snapshots_.count(cur) > 0) {
|
||||
continue;
|
||||
}
|
||||
auto res = snapshot_checker_->CheckInSnapshot(in, cur);
|
||||
if (res == SnapshotCheckerResult::kInSnapshot) {
|
||||
SequenceNumber prev = kMaxSequenceNumber;
|
||||
for (const auto cur : *snapshots_) {
|
||||
assert(prev == kMaxSequenceNumber || prev <= cur);
|
||||
if (cur >= in && (snapshot_checker_ == nullptr ||
|
||||
snapshot_checker_->IsInSnapshot(in, cur))) {
|
||||
*prev_snapshot = prev == kMaxSequenceNumber ? 0 : prev;
|
||||
return cur;
|
||||
} else if (res == SnapshotCheckerResult::kSnapshotReleased) {
|
||||
released_snapshots_.insert(cur);
|
||||
}
|
||||
*prev_snapshot = cur;
|
||||
prev = cur;
|
||||
assert(prev < kMaxSequenceNumber);
|
||||
}
|
||||
*prev_snapshot = prev;
|
||||
return kMaxSequenceNumber;
|
||||
}
|
||||
|
||||
@@ -675,30 +663,4 @@ inline bool CompactionIterator::ikeyNotNeededForIncrementalSnapshot() {
|
||||
(ikey_.sequence < preserve_deletes_seqnum_);
|
||||
}
|
||||
|
||||
bool CompactionIterator::IsInEarliestSnapshot(SequenceNumber sequence) {
|
||||
assert(snapshot_checker_ != nullptr);
|
||||
assert(earliest_snapshot_ == kMaxSequenceNumber ||
|
||||
(earliest_snapshot_iter_ != snapshots_->end() &&
|
||||
*earliest_snapshot_iter_ == earliest_snapshot_));
|
||||
auto in_snapshot =
|
||||
snapshot_checker_->CheckInSnapshot(sequence, earliest_snapshot_);
|
||||
while (UNLIKELY(in_snapshot == SnapshotCheckerResult::kSnapshotReleased)) {
|
||||
// Avoid the the current earliest_snapshot_ being return as
|
||||
// earliest visible snapshot for the next value. So if a value's sequence
|
||||
// is zero-ed out by PrepareOutput(), the next value will be compact out.
|
||||
released_snapshots_.insert(earliest_snapshot_);
|
||||
earliest_snapshot_iter_++;
|
||||
|
||||
if (earliest_snapshot_iter_ == snapshots_->end()) {
|
||||
earliest_snapshot_ = kMaxSequenceNumber;
|
||||
} else {
|
||||
earliest_snapshot_ = *earliest_snapshot_iter_;
|
||||
}
|
||||
in_snapshot =
|
||||
snapshot_checker_->CheckInSnapshot(sequence, earliest_snapshot_);
|
||||
}
|
||||
assert(in_snapshot != SnapshotCheckerResult::kSnapshotReleased);
|
||||
return in_snapshot == SnapshotCheckerResult::kInSnapshot;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
+14
-23
@@ -7,7 +7,6 @@
|
||||
#include <algorithm>
|
||||
#include <deque>
|
||||
#include <string>
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
#include "db/compaction.h"
|
||||
@@ -21,6 +20,8 @@
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class CompactionEventListener;
|
||||
|
||||
class CompactionIterator {
|
||||
public:
|
||||
// A wrapper around Compaction. Has a much smaller interface, only what
|
||||
@@ -31,7 +32,7 @@ class CompactionIterator {
|
||||
: compaction_(compaction) {}
|
||||
|
||||
virtual ~CompactionProxy() = default;
|
||||
virtual int level(size_t /*compaction_input_level*/ = 0) const {
|
||||
virtual int level(size_t compaction_input_level = 0) const {
|
||||
return compaction_->level();
|
||||
}
|
||||
virtual bool KeyNotExistsBeyondOutputLevel(
|
||||
@@ -64,10 +65,11 @@ class CompactionIterator {
|
||||
std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const SnapshotChecker* snapshot_checker, Env* env,
|
||||
bool report_detailed_time, bool expect_valid_internal_key,
|
||||
CompactionRangeDelAggregator* range_del_agg,
|
||||
bool expect_valid_internal_key,
|
||||
RangeDelAggregator* range_del_agg,
|
||||
const Compaction* compaction = nullptr,
|
||||
const CompactionFilter* compaction_filter = nullptr,
|
||||
CompactionEventListener* compaction_listener = nullptr,
|
||||
const std::atomic<bool>* shutting_down = nullptr,
|
||||
const SequenceNumber preserve_deletes_seqnum = 0);
|
||||
|
||||
@@ -77,10 +79,11 @@ class CompactionIterator {
|
||||
std::vector<SequenceNumber>* snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const SnapshotChecker* snapshot_checker, Env* env,
|
||||
bool report_detailed_time, bool expect_valid_internal_key,
|
||||
CompactionRangeDelAggregator* range_del_agg,
|
||||
bool expect_valid_internal_key,
|
||||
RangeDelAggregator* range_del_agg,
|
||||
std::unique_ptr<CompactionProxy> compaction,
|
||||
const CompactionFilter* compaction_filter = nullptr,
|
||||
CompactionEventListener* compaction_listener = nullptr,
|
||||
const std::atomic<bool>* shutting_down = nullptr,
|
||||
const SequenceNumber preserve_deletes_seqnum = 0);
|
||||
|
||||
@@ -133,33 +136,20 @@ class CompactionIterator {
|
||||
// or seqnum be zero-ed out even if all other conditions for it are met.
|
||||
inline bool ikeyNotNeededForIncrementalSnapshot();
|
||||
|
||||
inline bool KeyCommitted(SequenceNumber sequence) {
|
||||
return snapshot_checker_ == nullptr ||
|
||||
snapshot_checker_->CheckInSnapshot(sequence, kMaxSequenceNumber) ==
|
||||
SnapshotCheckerResult::kInSnapshot;
|
||||
}
|
||||
|
||||
bool IsInEarliestSnapshot(SequenceNumber sequence);
|
||||
|
||||
InternalIterator* input_;
|
||||
const Comparator* cmp_;
|
||||
MergeHelper* merge_helper_;
|
||||
const std::vector<SequenceNumber>* snapshots_;
|
||||
// List of snapshots released during compaction.
|
||||
// findEarliestVisibleSnapshot() find them out from return of
|
||||
// snapshot_checker, and make sure they will not be returned as
|
||||
// earliest visible snapshot of an older value.
|
||||
// See WritePreparedTransactionTest::ReleaseSnapshotDuringCompaction3.
|
||||
std::unordered_set<SequenceNumber> released_snapshots_;
|
||||
std::vector<SequenceNumber>::const_iterator earliest_snapshot_iter_;
|
||||
const SequenceNumber earliest_write_conflict_snapshot_;
|
||||
const SnapshotChecker* const snapshot_checker_;
|
||||
Env* env_;
|
||||
bool report_detailed_time_;
|
||||
bool expect_valid_internal_key_;
|
||||
CompactionRangeDelAggregator* range_del_agg_;
|
||||
RangeDelAggregator* range_del_agg_;
|
||||
std::unique_ptr<CompactionProxy> compaction_;
|
||||
const CompactionFilter* compaction_filter_;
|
||||
#ifndef ROCKSDB_LITE
|
||||
CompactionEventListener* compaction_listener_;
|
||||
#endif // !ROCKSDB_LITE
|
||||
const std::atomic<bool>* shutting_down_;
|
||||
const SequenceNumber preserve_deletes_seqnum_;
|
||||
bool bottommost_level_;
|
||||
@@ -167,6 +157,7 @@ class CompactionIterator {
|
||||
bool visible_at_tip_;
|
||||
SequenceNumber earliest_snapshot_;
|
||||
SequenceNumber latest_snapshot_;
|
||||
bool ignore_snapshots_;
|
||||
|
||||
// State
|
||||
//
|
||||
|
||||
+71
-474
@@ -9,25 +9,23 @@
|
||||
#include <vector>
|
||||
|
||||
#include "port/port.h"
|
||||
#include "util/string_util.h"
|
||||
#include "util/testharness.h"
|
||||
#include "util/testutil.h"
|
||||
#include "utilities/merge_operators.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
// Expects no merging attempts.
|
||||
class NoMergingMergeOp : public MergeOperator {
|
||||
public:
|
||||
bool FullMergeV2(const MergeOperationInput& /*merge_in*/,
|
||||
MergeOperationOutput* /*merge_out*/) const override {
|
||||
bool FullMergeV2(const MergeOperationInput& merge_in,
|
||||
MergeOperationOutput* merge_out) const override {
|
||||
ADD_FAILURE();
|
||||
return false;
|
||||
}
|
||||
bool PartialMergeMulti(const Slice& /*key*/,
|
||||
const std::deque<Slice>& /*operand_list*/,
|
||||
std::string* /*new_value*/,
|
||||
Logger* /*logger*/) const override {
|
||||
bool PartialMergeMulti(const Slice& key,
|
||||
const std::deque<Slice>& operand_list,
|
||||
std::string* new_value,
|
||||
Logger* logger) const override {
|
||||
ADD_FAILURE();
|
||||
return false;
|
||||
}
|
||||
@@ -41,9 +39,9 @@ class NoMergingMergeOp : public MergeOperator {
|
||||
// Always returns Decition::kRemove.
|
||||
class StallingFilter : public CompactionFilter {
|
||||
public:
|
||||
Decision FilterV2(int /*level*/, const Slice& key, ValueType /*type*/,
|
||||
const Slice& /*existing_value*/, std::string* /*new_value*/,
|
||||
std::string* /*skip_until*/) const override {
|
||||
virtual Decision FilterV2(int level, const Slice& key, ValueType t,
|
||||
const Slice& existing_value, std::string* new_value,
|
||||
std::string* skip_until) const override {
|
||||
int k = std::atoi(key.ToString().c_str());
|
||||
last_seen.store(k);
|
||||
while (k >= stall_at.load()) {
|
||||
@@ -74,18 +72,6 @@ class StallingFilter : public CompactionFilter {
|
||||
mutable std::atomic<int> last_seen{0};
|
||||
};
|
||||
|
||||
// Compaction filter that filter out all keys.
|
||||
class FilterAllKeysCompactionFilter : public CompactionFilter {
|
||||
public:
|
||||
Decision FilterV2(int /*level*/, const Slice& /*key*/, ValueType /*type*/,
|
||||
const Slice& /*existing_value*/, std::string* /*new_value*/,
|
||||
std::string* /*skip_until*/) const override {
|
||||
return Decision::kRemove;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "AllKeysCompactionFilter"; }
|
||||
};
|
||||
|
||||
class LoggingForwardVectorIterator : public InternalIterator {
|
||||
public:
|
||||
struct Action {
|
||||
@@ -112,39 +98,39 @@ class LoggingForwardVectorIterator : public InternalIterator {
|
||||
assert(keys_.size() == values_.size());
|
||||
}
|
||||
|
||||
bool Valid() const override { return current_ < keys_.size(); }
|
||||
virtual bool Valid() const override { return current_ < keys_.size(); }
|
||||
|
||||
void SeekToFirst() override {
|
||||
virtual void SeekToFirst() override {
|
||||
log.emplace_back(Action::Type::SEEK_TO_FIRST);
|
||||
current_ = 0;
|
||||
}
|
||||
void SeekToLast() override { assert(false); }
|
||||
virtual void SeekToLast() override { assert(false); }
|
||||
|
||||
void Seek(const Slice& target) override {
|
||||
virtual void Seek(const Slice& target) override {
|
||||
log.emplace_back(Action::Type::SEEK, target.ToString());
|
||||
current_ = std::lower_bound(keys_.begin(), keys_.end(), target.ToString()) -
|
||||
keys_.begin();
|
||||
}
|
||||
|
||||
void SeekForPrev(const Slice& /*target*/) override { assert(false); }
|
||||
virtual void SeekForPrev(const Slice& target) override { assert(false); }
|
||||
|
||||
void Next() override {
|
||||
virtual void Next() override {
|
||||
assert(Valid());
|
||||
log.emplace_back(Action::Type::NEXT);
|
||||
current_++;
|
||||
}
|
||||
void Prev() override { assert(false); }
|
||||
virtual void Prev() override { assert(false); }
|
||||
|
||||
Slice key() const override {
|
||||
virtual Slice key() const override {
|
||||
assert(Valid());
|
||||
return Slice(keys_[current_]);
|
||||
}
|
||||
Slice value() const override {
|
||||
virtual Slice value() const override {
|
||||
assert(Valid());
|
||||
return Slice(values_[current_]);
|
||||
}
|
||||
|
||||
Status status() const override { return Status::OK(); }
|
||||
virtual Status status() const override { return Status::OK(); }
|
||||
|
||||
std::vector<Action> log;
|
||||
|
||||
@@ -158,156 +144,76 @@ class FakeCompaction : public CompactionIterator::CompactionProxy {
|
||||
public:
|
||||
FakeCompaction() = default;
|
||||
|
||||
int level(size_t /*compaction_input_level*/) const override { return 0; }
|
||||
bool KeyNotExistsBeyondOutputLevel(
|
||||
const Slice& /*user_key*/,
|
||||
std::vector<size_t>* /*level_ptrs*/) const override {
|
||||
return is_bottommost_level || key_not_exists_beyond_output_level;
|
||||
virtual int level(size_t compaction_input_level) const { return 0; }
|
||||
virtual bool KeyNotExistsBeyondOutputLevel(
|
||||
const Slice& user_key, std::vector<size_t>* level_ptrs) const {
|
||||
return key_not_exists_beyond_output_level;
|
||||
}
|
||||
bool bottommost_level() const override { return is_bottommost_level; }
|
||||
int number_levels() const override { return 1; }
|
||||
Slice GetLargestUserKey() const override {
|
||||
virtual bool bottommost_level() const { return false; }
|
||||
virtual int number_levels() const { return 1; }
|
||||
virtual Slice GetLargestUserKey() const {
|
||||
return "\xff\xff\xff\xff\xff\xff\xff\xff\xff";
|
||||
}
|
||||
bool allow_ingest_behind() const override { return false; }
|
||||
virtual bool allow_ingest_behind() const { return false; }
|
||||
|
||||
bool preserve_deletes() const override { return false; }
|
||||
virtual bool preserve_deletes() const {return false; }
|
||||
|
||||
bool key_not_exists_beyond_output_level = false;
|
||||
|
||||
bool is_bottommost_level = false;
|
||||
};
|
||||
|
||||
// A simplifed snapshot checker which assumes each snapshot has a global
|
||||
// last visible sequence.
|
||||
class TestSnapshotChecker : public SnapshotChecker {
|
||||
public:
|
||||
explicit TestSnapshotChecker(
|
||||
SequenceNumber last_committed_sequence,
|
||||
const std::unordered_map<SequenceNumber, SequenceNumber>& snapshots = {})
|
||||
: last_committed_sequence_(last_committed_sequence),
|
||||
snapshots_(snapshots) {}
|
||||
|
||||
SnapshotCheckerResult CheckInSnapshot(
|
||||
SequenceNumber seq, SequenceNumber snapshot_seq) const override {
|
||||
if (snapshot_seq == kMaxSequenceNumber) {
|
||||
return seq <= last_committed_sequence_
|
||||
? SnapshotCheckerResult::kInSnapshot
|
||||
: SnapshotCheckerResult::kNotInSnapshot;
|
||||
}
|
||||
assert(snapshots_.count(snapshot_seq) > 0);
|
||||
return seq <= snapshots_.at(snapshot_seq)
|
||||
? SnapshotCheckerResult::kInSnapshot
|
||||
: SnapshotCheckerResult::kNotInSnapshot;
|
||||
}
|
||||
|
||||
private:
|
||||
SequenceNumber last_committed_sequence_;
|
||||
// A map of valid snapshot to last visible sequence to the snapshot.
|
||||
std::unordered_map<SequenceNumber, SequenceNumber> snapshots_;
|
||||
};
|
||||
|
||||
// Test param:
|
||||
// bool: whether to pass snapshot_checker to compaction iterator.
|
||||
class CompactionIteratorTest : public testing::TestWithParam<bool> {
|
||||
class CompactionIteratorTest : public testing::Test {
|
||||
public:
|
||||
CompactionIteratorTest()
|
||||
: cmp_(BytewiseComparator()), icmp_(cmp_), snapshots_({}) {}
|
||||
|
||||
void InitIterators(
|
||||
const std::vector<std::string>& ks, const std::vector<std::string>& vs,
|
||||
const std::vector<std::string>& range_del_ks,
|
||||
const std::vector<std::string>& range_del_vs,
|
||||
SequenceNumber last_sequence,
|
||||
SequenceNumber last_committed_sequence = kMaxSequenceNumber,
|
||||
MergeOperator* merge_op = nullptr, CompactionFilter* filter = nullptr,
|
||||
bool bottommost_level = false,
|
||||
SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber) {
|
||||
std::unique_ptr<InternalIterator> unfragmented_range_del_iter(
|
||||
void InitIterators(const std::vector<std::string>& ks,
|
||||
const std::vector<std::string>& vs,
|
||||
const std::vector<std::string>& range_del_ks,
|
||||
const std::vector<std::string>& range_del_vs,
|
||||
SequenceNumber last_sequence,
|
||||
MergeOperator* merge_op = nullptr,
|
||||
CompactionFilter* filter = nullptr) {
|
||||
std::unique_ptr<InternalIterator> range_del_iter(
|
||||
new test::VectorIterator(range_del_ks, range_del_vs));
|
||||
auto tombstone_list = std::make_shared<FragmentedRangeTombstoneList>(
|
||||
std::move(unfragmented_range_del_iter), icmp_);
|
||||
std::unique_ptr<FragmentedRangeTombstoneIterator> range_del_iter(
|
||||
new FragmentedRangeTombstoneIterator(tombstone_list, icmp_,
|
||||
kMaxSequenceNumber));
|
||||
range_del_agg_.reset(new CompactionRangeDelAggregator(&icmp_, snapshots_));
|
||||
range_del_agg_->AddTombstones(std::move(range_del_iter));
|
||||
range_del_agg_.reset(new RangeDelAggregator(icmp_, snapshots_));
|
||||
ASSERT_OK(range_del_agg_->AddTombstones(std::move(range_del_iter)));
|
||||
|
||||
std::unique_ptr<CompactionIterator::CompactionProxy> compaction;
|
||||
if (filter || bottommost_level) {
|
||||
if (filter) {
|
||||
compaction_proxy_ = new FakeCompaction();
|
||||
compaction_proxy_->is_bottommost_level = bottommost_level;
|
||||
compaction.reset(compaction_proxy_);
|
||||
}
|
||||
bool use_snapshot_checker = UseSnapshotChecker() || GetParam();
|
||||
if (use_snapshot_checker || last_committed_sequence < kMaxSequenceNumber) {
|
||||
snapshot_checker_.reset(
|
||||
new TestSnapshotChecker(last_committed_sequence, snapshot_map_));
|
||||
}
|
||||
merge_helper_.reset(
|
||||
new MergeHelper(Env::Default(), cmp_, merge_op, filter, nullptr, false,
|
||||
0 /*latest_snapshot*/, snapshot_checker_.get(),
|
||||
0 /*level*/, nullptr /*statistics*/, &shutting_down_));
|
||||
// TODO(yiwu) add a mock snapshot checker and add test for it.
|
||||
SnapshotChecker* snapshot_checker = nullptr;
|
||||
|
||||
merge_helper_.reset(new MergeHelper(Env::Default(), cmp_, merge_op, filter,
|
||||
nullptr, false, 0, 0, nullptr,
|
||||
&shutting_down_));
|
||||
iter_.reset(new LoggingForwardVectorIterator(ks, vs));
|
||||
iter_->SeekToFirst();
|
||||
c_iter_.reset(new CompactionIterator(
|
||||
iter_.get(), cmp_, merge_helper_.get(), last_sequence, &snapshots_,
|
||||
earliest_write_conflict_snapshot, snapshot_checker_.get(),
|
||||
Env::Default(), false /* report_detailed_time */, false,
|
||||
range_del_agg_.get(), std::move(compaction), filter, &shutting_down_));
|
||||
kMaxSequenceNumber, snapshot_checker, Env::Default(), false,
|
||||
range_del_agg_.get(), std::move(compaction), filter, nullptr,
|
||||
&shutting_down_));
|
||||
}
|
||||
|
||||
void AddSnapshot(SequenceNumber snapshot,
|
||||
SequenceNumber last_visible_seq = kMaxSequenceNumber) {
|
||||
snapshots_.push_back(snapshot);
|
||||
snapshot_map_[snapshot] = last_visible_seq;
|
||||
}
|
||||
|
||||
virtual bool UseSnapshotChecker() const { return false; }
|
||||
|
||||
void RunTest(
|
||||
const std::vector<std::string>& input_keys,
|
||||
const std::vector<std::string>& input_values,
|
||||
const std::vector<std::string>& expected_keys,
|
||||
const std::vector<std::string>& expected_values,
|
||||
SequenceNumber last_committed_seq = kMaxSequenceNumber,
|
||||
MergeOperator* merge_operator = nullptr,
|
||||
CompactionFilter* compaction_filter = nullptr,
|
||||
bool bottommost_level = false,
|
||||
SequenceNumber earliest_write_conflict_snapshot = kMaxSequenceNumber) {
|
||||
InitIterators(input_keys, input_values, {}, {}, kMaxSequenceNumber,
|
||||
last_committed_seq, merge_operator, compaction_filter,
|
||||
bottommost_level, earliest_write_conflict_snapshot);
|
||||
c_iter_->SeekToFirst();
|
||||
for (size_t i = 0; i < expected_keys.size(); i++) {
|
||||
std::string info = "i = " + ToString(i);
|
||||
ASSERT_TRUE(c_iter_->Valid()) << info;
|
||||
ASSERT_OK(c_iter_->status()) << info;
|
||||
ASSERT_EQ(expected_keys[i], c_iter_->key().ToString()) << info;
|
||||
ASSERT_EQ(expected_values[i], c_iter_->value().ToString()) << info;
|
||||
c_iter_->Next();
|
||||
}
|
||||
ASSERT_FALSE(c_iter_->Valid());
|
||||
}
|
||||
void AddSnapshot(SequenceNumber snapshot) { snapshots_.push_back(snapshot); }
|
||||
|
||||
const Comparator* cmp_;
|
||||
const InternalKeyComparator icmp_;
|
||||
std::vector<SequenceNumber> snapshots_;
|
||||
// A map of valid snapshot to last visible sequence to the snapshot.
|
||||
std::unordered_map<SequenceNumber, SequenceNumber> snapshot_map_;
|
||||
std::unique_ptr<MergeHelper> merge_helper_;
|
||||
std::unique_ptr<LoggingForwardVectorIterator> iter_;
|
||||
std::unique_ptr<CompactionIterator> c_iter_;
|
||||
std::unique_ptr<CompactionRangeDelAggregator> range_del_agg_;
|
||||
std::unique_ptr<SnapshotChecker> snapshot_checker_;
|
||||
std::unique_ptr<RangeDelAggregator> range_del_agg_;
|
||||
std::atomic<bool> shutting_down_{false};
|
||||
FakeCompaction* compaction_proxy_;
|
||||
};
|
||||
|
||||
// It is possible that the output of the compaction iterator is empty even if
|
||||
// the input is not.
|
||||
TEST_P(CompactionIteratorTest, EmptyResult) {
|
||||
TEST_F(CompactionIteratorTest, EmptyResult) {
|
||||
InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
|
||||
test::KeyStr("a", 3, kTypeValue)},
|
||||
{"", "val"}, {}, {}, 5);
|
||||
@@ -317,7 +223,7 @@ TEST_P(CompactionIteratorTest, EmptyResult) {
|
||||
|
||||
// If there is a corruption after a single deletion, the corrupted key should
|
||||
// be preserved.
|
||||
TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
|
||||
TEST_F(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
|
||||
InitIterators({test::KeyStr("a", 5, kTypeSingleDeletion),
|
||||
test::KeyStr("a", 3, kTypeValue, true),
|
||||
test::KeyStr("b", 10, kTypeValue)},
|
||||
@@ -336,7 +242,7 @@ TEST_P(CompactionIteratorTest, CorruptionAfterSingleDeletion) {
|
||||
ASSERT_FALSE(c_iter_->Valid());
|
||||
}
|
||||
|
||||
TEST_P(CompactionIteratorTest, SimpleRangeDeletion) {
|
||||
TEST_F(CompactionIteratorTest, SimpleRangeDeletion) {
|
||||
InitIterators({test::KeyStr("morning", 5, kTypeValue),
|
||||
test::KeyStr("morning", 2, kTypeValue),
|
||||
test::KeyStr("night", 3, kTypeValue)},
|
||||
@@ -352,7 +258,7 @@ TEST_P(CompactionIteratorTest, SimpleRangeDeletion) {
|
||||
ASSERT_FALSE(c_iter_->Valid());
|
||||
}
|
||||
|
||||
TEST_P(CompactionIteratorTest, RangeDeletionWithSnapshots) {
|
||||
TEST_F(CompactionIteratorTest, RangeDeletionWithSnapshots) {
|
||||
AddSnapshot(10);
|
||||
std::vector<std::string> ks1;
|
||||
ks1.push_back(test::KeyStr("ma", 28, kTypeRangeDeletion));
|
||||
@@ -373,11 +279,12 @@ TEST_P(CompactionIteratorTest, RangeDeletionWithSnapshots) {
|
||||
ASSERT_FALSE(c_iter_->Valid());
|
||||
}
|
||||
|
||||
TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
|
||||
TEST_F(CompactionIteratorTest, CompactionFilterSkipUntil) {
|
||||
class Filter : public CompactionFilter {
|
||||
Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
|
||||
const Slice& existing_value, std::string* /*new_value*/,
|
||||
std::string* skip_until) const override {
|
||||
virtual Decision FilterV2(int level, const Slice& key, ValueType t,
|
||||
const Slice& existing_value,
|
||||
std::string* new_value,
|
||||
std::string* skip_until) const override {
|
||||
std::string k = key.ToString();
|
||||
std::string v = existing_value.ToString();
|
||||
// See InitIterators() call below for the sequence of keys and their
|
||||
@@ -447,7 +354,7 @@ TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
|
||||
test::KeyStr("j", 99, kTypeValue)},
|
||||
{"av50", "am45", "bv60", "bv40", "cv35", "dm70", "em71", "fm65", "fm30",
|
||||
"fv25", "gv90", "hv91", "im95", "jv99"},
|
||||
{}, {}, kMaxSequenceNumber, kMaxSequenceNumber, &merge_op, &filter);
|
||||
{}, {}, kMaxSequenceNumber, &merge_op, &filter);
|
||||
|
||||
// Compaction should output just "a", "e" and "h" keys.
|
||||
c_iter_->SeekToFirst();
|
||||
@@ -482,14 +389,13 @@ TEST_P(CompactionIteratorTest, CompactionFilterSkipUntil) {
|
||||
ASSERT_EQ(expected_actions, iter_->log);
|
||||
}
|
||||
|
||||
TEST_P(CompactionIteratorTest, ShuttingDownInFilter) {
|
||||
TEST_F(CompactionIteratorTest, ShuttingDownInFilter) {
|
||||
NoMergingMergeOp merge_op;
|
||||
StallingFilter filter;
|
||||
InitIterators(
|
||||
{test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeValue),
|
||||
test::KeyStr("3", 3, kTypeValue), test::KeyStr("4", 4, kTypeValue)},
|
||||
{"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
|
||||
&merge_op, &filter);
|
||||
{"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
|
||||
// Don't leave tombstones (kTypeDeletion) for filtered keys.
|
||||
compaction_proxy_->key_not_exists_beyond_output_level = true;
|
||||
|
||||
@@ -520,14 +426,13 @@ TEST_P(CompactionIteratorTest, ShuttingDownInFilter) {
|
||||
|
||||
// Same as ShuttingDownInFilter, but shutdown happens during filter call for
|
||||
// a merge operand, not for a value.
|
||||
TEST_P(CompactionIteratorTest, ShuttingDownInMerge) {
|
||||
TEST_F(CompactionIteratorTest, ShuttingDownInMerge) {
|
||||
NoMergingMergeOp merge_op;
|
||||
StallingFilter filter;
|
||||
InitIterators(
|
||||
{test::KeyStr("1", 1, kTypeValue), test::KeyStr("2", 2, kTypeMerge),
|
||||
test::KeyStr("3", 3, kTypeMerge), test::KeyStr("4", 4, kTypeValue)},
|
||||
{"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, kMaxSequenceNumber,
|
||||
&merge_op, &filter);
|
||||
{"v1", "v2", "v3", "v4"}, {}, {}, kMaxSequenceNumber, &merge_op, &filter);
|
||||
compaction_proxy_->key_not_exists_beyond_output_level = true;
|
||||
|
||||
std::atomic<bool> seek_done{false};
|
||||
@@ -555,11 +460,12 @@ TEST_P(CompactionIteratorTest, ShuttingDownInMerge) {
|
||||
EXPECT_EQ(2, filter.last_seen.load());
|
||||
}
|
||||
|
||||
TEST_P(CompactionIteratorTest, SingleMergeOperand) {
|
||||
TEST_F(CompactionIteratorTest, SingleMergeOperand) {
|
||||
class Filter : public CompactionFilter {
|
||||
Decision FilterV2(int /*level*/, const Slice& key, ValueType t,
|
||||
const Slice& existing_value, std::string* /*new_value*/,
|
||||
std::string* /*skip_until*/) const override {
|
||||
virtual Decision FilterV2(int level, const Slice& key, ValueType t,
|
||||
const Slice& existing_value,
|
||||
std::string* new_value,
|
||||
std::string* skip_until) const override {
|
||||
std::string k = key.ToString();
|
||||
std::string v = existing_value.ToString();
|
||||
|
||||
@@ -610,7 +516,7 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
|
||||
bool PartialMergeMulti(const Slice& key,
|
||||
const std::deque<Slice>& operand_list,
|
||||
std::string* new_value,
|
||||
Logger* /*logger*/) const override {
|
||||
Logger* logger) const override {
|
||||
std::string string_key = key.ToString();
|
||||
EXPECT_TRUE(string_key == "a" || string_key == "b");
|
||||
|
||||
@@ -646,7 +552,7 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
|
||||
// c should invoke FullMerge due to kTypeValue at the beginning.
|
||||
test::KeyStr("c", 90, kTypeMerge), test::KeyStr("c", 80, kTypeValue)},
|
||||
{"av1", "bv2", "bv1", "cv2", "cv1"}, {}, {}, kMaxSequenceNumber,
|
||||
kMaxSequenceNumber, &merge_op, &filter);
|
||||
&merge_op, &filter);
|
||||
|
||||
c_iter_->SeekToFirst();
|
||||
ASSERT_TRUE(c_iter_->Valid());
|
||||
@@ -659,315 +565,6 @@ TEST_P(CompactionIteratorTest, SingleMergeOperand) {
|
||||
ASSERT_EQ("cv1cv2", c_iter_->value().ToString());
|
||||
}
|
||||
|
||||
// In bottommost level, values earlier than earliest snapshot can be output
|
||||
// with sequence = 0.
|
||||
TEST_P(CompactionIteratorTest, ZeroOutSequenceAtBottomLevel) {
|
||||
AddSnapshot(1);
|
||||
RunTest({test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 2, kTypeValue)},
|
||||
{"v1", "v2"},
|
||||
{test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 2, kTypeValue)},
|
||||
{"v1", "v2"}, kMaxSequenceNumber /*last_commited_seq*/,
|
||||
nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
|
||||
true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
// In bottommost level, deletions earlier than earliest snapshot can be removed
|
||||
// permanently.
|
||||
TEST_P(CompactionIteratorTest, RemoveDeletionAtBottomLevel) {
|
||||
AddSnapshot(1);
|
||||
RunTest({test::KeyStr("a", 1, kTypeDeletion),
|
||||
test::KeyStr("b", 3, kTypeDeletion),
|
||||
test::KeyStr("b", 1, kTypeValue)},
|
||||
{"", "", ""},
|
||||
{test::KeyStr("b", 3, kTypeDeletion),
|
||||
test::KeyStr("b", 0, kTypeValue)},
|
||||
{"", ""},
|
||||
kMaxSequenceNumber /*last_commited_seq*/, nullptr /*merge_operator*/,
|
||||
nullptr /*compaction_filter*/, true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
// In bottommost level, single deletions earlier than earliest snapshot can be
|
||||
// removed permanently.
|
||||
TEST_P(CompactionIteratorTest, RemoveSingleDeletionAtBottomLevel) {
|
||||
AddSnapshot(1);
|
||||
RunTest({test::KeyStr("a", 1, kTypeSingleDeletion),
|
||||
test::KeyStr("b", 2, kTypeSingleDeletion)},
|
||||
{"", ""}, {test::KeyStr("b", 2, kTypeSingleDeletion)}, {""},
|
||||
kMaxSequenceNumber /*last_commited_seq*/, nullptr /*merge_operator*/,
|
||||
nullptr /*compaction_filter*/, true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(CompactionIteratorTestInstance, CompactionIteratorTest,
|
||||
testing::Values(true, false));
|
||||
|
||||
// Tests how CompactionIterator work together with SnapshotChecker.
|
||||
class CompactionIteratorWithSnapshotCheckerTest
|
||||
: public CompactionIteratorTest {
|
||||
public:
|
||||
bool UseSnapshotChecker() const override { return true; }
|
||||
};
|
||||
|
||||
// Uncommitted keys (keys with seq > last_committed_seq) should be output as-is
|
||||
// while committed version of these keys should get compacted as usual.
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
PreserveUncommittedKeys_Value) {
|
||||
RunTest(
|
||||
{test::KeyStr("foo", 3, kTypeValue), test::KeyStr("foo", 2, kTypeValue),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v3", "v2", "v1"},
|
||||
{test::KeyStr("foo", 3, kTypeValue), test::KeyStr("foo", 2, kTypeValue)},
|
||||
{"v3", "v2"}, 2 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
PreserveUncommittedKeys_Deletion) {
|
||||
RunTest({test::KeyStr("foo", 2, kTypeDeletion),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"", "v1"},
|
||||
{test::KeyStr("foo", 2, kTypeDeletion),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"", "v1"}, 1 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
PreserveUncommittedKeys_Merge) {
|
||||
auto merge_op = MergeOperators::CreateStringAppendOperator();
|
||||
RunTest(
|
||||
{test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeMerge),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v3", "v2", "v1"},
|
||||
{test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeValue)},
|
||||
{"v3", "v1,v2"}, 2 /*last_committed_seq*/, merge_op.get());
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
PreserveUncommittedKeys_SingleDelete) {
|
||||
RunTest({test::KeyStr("foo", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"", "v1"},
|
||||
{test::KeyStr("foo", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"", "v1"}, 1 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
PreserveUncommittedKeys_BlobIndex) {
|
||||
RunTest({test::KeyStr("foo", 3, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 2, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 1, kTypeBlobIndex)},
|
||||
{"v3", "v2", "v1"},
|
||||
{test::KeyStr("foo", 3, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 2, kTypeBlobIndex)},
|
||||
{"v3", "v2"}, 2 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
// Test compaction iterator dedup keys visible to the same snapshot.
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Value) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest(
|
||||
{test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 3, kTypeValue),
|
||||
test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v4", "v3", "v2", "v1"},
|
||||
{test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 3, kTypeValue),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v4", "v3", "v1"}, 3 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Deletion) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest(
|
||||
{test::KeyStr("foo", 4, kTypeValue),
|
||||
test::KeyStr("foo", 3, kTypeDeletion),
|
||||
test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v4", "", "v2", "v1"},
|
||||
{test::KeyStr("foo", 4, kTypeValue),
|
||||
test::KeyStr("foo", 3, kTypeDeletion),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v4", "", "v1"}, 3 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_Merge) {
|
||||
AddSnapshot(2, 1);
|
||||
AddSnapshot(4, 3);
|
||||
auto merge_op = MergeOperators::CreateStringAppendOperator();
|
||||
RunTest(
|
||||
{test::KeyStr("foo", 5, kTypeMerge), test::KeyStr("foo", 4, kTypeMerge),
|
||||
test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 2, kTypeMerge),
|
||||
test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v5", "v4", "v3", "v2", "v1"},
|
||||
{test::KeyStr("foo", 5, kTypeMerge), test::KeyStr("foo", 4, kTypeMerge),
|
||||
test::KeyStr("foo", 3, kTypeMerge), test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v5", "v4", "v2,v3", "v1"}, 4 /*last_committed_seq*/, merge_op.get());
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
DedupSameSnapshot_SingleDeletion) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest(
|
||||
{test::KeyStr("foo", 4, kTypeValue),
|
||||
test::KeyStr("foo", 3, kTypeSingleDeletion),
|
||||
test::KeyStr("foo", 2, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v4", "", "v2", "v1"},
|
||||
{test::KeyStr("foo", 4, kTypeValue), test::KeyStr("foo", 1, kTypeValue)},
|
||||
{"v4", "v1"}, 3 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, DedupSameSnapshot_BlobIndex) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest({test::KeyStr("foo", 4, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 3, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 2, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 1, kTypeBlobIndex)},
|
||||
{"v4", "v3", "v2", "v1"},
|
||||
{test::KeyStr("foo", 4, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 3, kTypeBlobIndex),
|
||||
test::KeyStr("foo", 1, kTypeBlobIndex)},
|
||||
{"v4", "v3", "v1"}, 3 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
// At bottom level, sequence numbers can be zero out, and deletions can be
|
||||
// removed, but only when they are visible to earliest snapshot.
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
NotZeroOutSequenceIfNotVisibleToEarliestSnapshot) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest({test::KeyStr("a", 1, kTypeValue), test::KeyStr("b", 2, kTypeValue),
|
||||
test::KeyStr("c", 3, kTypeValue)},
|
||||
{"v1", "v2", "v3"},
|
||||
{test::KeyStr("a", 0, kTypeValue), test::KeyStr("b", 2, kTypeValue),
|
||||
test::KeyStr("c", 3, kTypeValue)},
|
||||
{"v1", "v2", "v3"}, kMaxSequenceNumber /*last_commited_seq*/,
|
||||
nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
|
||||
true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
NotRemoveDeletionIfNotVisibleToEarliestSnapshot) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest(
|
||||
{test::KeyStr("a", 1, kTypeDeletion), test::KeyStr("b", 2, kTypeDeletion),
|
||||
test::KeyStr("c", 3, kTypeDeletion)},
|
||||
{"", "", ""},
|
||||
{},
|
||||
{"", ""}, kMaxSequenceNumber /*last_commited_seq*/,
|
||||
nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
|
||||
true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
NotRemoveDeletionIfValuePresentToEarlierSnapshot) {
|
||||
AddSnapshot(2,1);
|
||||
RunTest(
|
||||
{test::KeyStr("a", 4, kTypeDeletion), test::KeyStr("a", 1, kTypeValue),
|
||||
test::KeyStr("b", 3, kTypeValue)},
|
||||
{"", "", ""},
|
||||
{test::KeyStr("a", 4, kTypeDeletion), test::KeyStr("a", 0, kTypeValue),
|
||||
test::KeyStr("b", 3, kTypeValue)},
|
||||
{"", "", ""}, kMaxSequenceNumber /*last_commited_seq*/,
|
||||
nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
|
||||
true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
NotRemoveSingleDeletionIfNotVisibleToEarliestSnapshot) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest({test::KeyStr("a", 1, kTypeSingleDeletion),
|
||||
test::KeyStr("b", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("c", 3, kTypeSingleDeletion)},
|
||||
{"", "", ""},
|
||||
{test::KeyStr("b", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("c", 3, kTypeSingleDeletion)},
|
||||
{"", ""}, kMaxSequenceNumber /*last_commited_seq*/,
|
||||
nullptr /*merge_operator*/, nullptr /*compaction_filter*/,
|
||||
true /*bottommost_level*/);
|
||||
}
|
||||
|
||||
// Single delete should not cancel out values that not visible to the
|
||||
// same set of snapshots
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
SingleDeleteAcrossSnapshotBoundary) {
|
||||
AddSnapshot(2, 1);
|
||||
RunTest({test::KeyStr("a", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("a", 1, kTypeValue)},
|
||||
{"", "v1"},
|
||||
{test::KeyStr("a", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("a", 1, kTypeValue)},
|
||||
{"", "v1"}, 2 /*last_committed_seq*/);
|
||||
}
|
||||
|
||||
// Single delete should be kept in case it is not visible to the
|
||||
// earliest write conflict snapshot. If a single delete is kept for this reason,
|
||||
// corresponding value can be trimmed to save space.
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
KeepSingleDeletionForWriteConflictChecking) {
|
||||
AddSnapshot(2, 0);
|
||||
RunTest({test::KeyStr("a", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("a", 1, kTypeValue)},
|
||||
{"", "v1"},
|
||||
{test::KeyStr("a", 2, kTypeSingleDeletion),
|
||||
test::KeyStr("a", 1, kTypeValue)},
|
||||
{"", ""}, 2 /*last_committed_seq*/, nullptr /*merge_operator*/,
|
||||
nullptr /*compaction_filter*/, false /*bottommost_level*/,
|
||||
2 /*earliest_write_conflict_snapshot*/);
|
||||
}
|
||||
|
||||
// Compaction filter should keep uncommitted key as-is, and
|
||||
// * Convert the latest velue to deletion, and/or
|
||||
// * if latest value is a merge, apply filter to all suequent merges.
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Value) {
|
||||
std::unique_ptr<CompactionFilter> compaction_filter(
|
||||
new FilterAllKeysCompactionFilter());
|
||||
RunTest(
|
||||
{test::KeyStr("a", 2, kTypeValue), test::KeyStr("a", 1, kTypeValue),
|
||||
test::KeyStr("b", 3, kTypeValue), test::KeyStr("c", 1, kTypeValue)},
|
||||
{"v2", "v1", "v3", "v4"},
|
||||
{test::KeyStr("a", 2, kTypeValue), test::KeyStr("a", 1, kTypeDeletion),
|
||||
test::KeyStr("b", 3, kTypeValue), test::KeyStr("c", 1, kTypeDeletion)},
|
||||
{"v2", "", "v3", ""}, 1 /*last_committed_seq*/,
|
||||
nullptr /*merge_operator*/, compaction_filter.get());
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_Deletion) {
|
||||
std::unique_ptr<CompactionFilter> compaction_filter(
|
||||
new FilterAllKeysCompactionFilter());
|
||||
RunTest(
|
||||
{test::KeyStr("a", 2, kTypeDeletion), test::KeyStr("a", 1, kTypeValue)},
|
||||
{"", "v1"},
|
||||
{test::KeyStr("a", 2, kTypeDeletion),
|
||||
test::KeyStr("a", 1, kTypeDeletion)},
|
||||
{"", ""}, 1 /*last_committed_seq*/, nullptr /*merge_operator*/,
|
||||
compaction_filter.get());
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest,
|
||||
CompactionFilter_PartialMerge) {
|
||||
std::shared_ptr<MergeOperator> merge_op =
|
||||
MergeOperators::CreateStringAppendOperator();
|
||||
std::unique_ptr<CompactionFilter> compaction_filter(
|
||||
new FilterAllKeysCompactionFilter());
|
||||
RunTest({test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 2, kTypeMerge),
|
||||
test::KeyStr("a", 1, kTypeMerge)},
|
||||
{"v3", "v2", "v1"}, {test::KeyStr("a", 3, kTypeMerge)}, {"v3"},
|
||||
2 /*last_committed_seq*/, merge_op.get(), compaction_filter.get());
|
||||
}
|
||||
|
||||
TEST_F(CompactionIteratorWithSnapshotCheckerTest, CompactionFilter_FullMerge) {
|
||||
std::shared_ptr<MergeOperator> merge_op =
|
||||
MergeOperators::CreateStringAppendOperator();
|
||||
std::unique_ptr<CompactionFilter> compaction_filter(
|
||||
new FilterAllKeysCompactionFilter());
|
||||
RunTest(
|
||||
{test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 2, kTypeMerge),
|
||||
test::KeyStr("a", 1, kTypeValue)},
|
||||
{"v3", "v2", "v1"},
|
||||
{test::KeyStr("a", 3, kTypeMerge), test::KeyStr("a", 1, kTypeDeletion)},
|
||||
{"v3", ""}, 2 /*last_committed_seq*/, merge_op.get(),
|
||||
compaction_filter.get());
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
+283
-437
File diff suppressed because it is too large
Load Diff
+7
-10
@@ -29,7 +29,6 @@
|
||||
#include "db/version_edit.h"
|
||||
#include "db/write_controller.h"
|
||||
#include "db/write_thread.h"
|
||||
#include "options/cf_options.h"
|
||||
#include "options/db_options.h"
|
||||
#include "port/port.h"
|
||||
#include "rocksdb/compaction_filter.h"
|
||||
@@ -47,7 +46,6 @@
|
||||
namespace rocksdb {
|
||||
|
||||
class Arena;
|
||||
class ErrorHandler;
|
||||
class MemTable;
|
||||
class SnapshotChecker;
|
||||
class TableCache;
|
||||
@@ -62,17 +60,17 @@ class CompactionJob {
|
||||
const EnvOptions env_options, VersionSet* versions,
|
||||
const std::atomic<bool>* shutting_down,
|
||||
const SequenceNumber preserve_deletes_seqnum,
|
||||
LogBuffer* log_buffer, Directory* db_directory,
|
||||
Directory* output_directory, Statistics* stats,
|
||||
InstrumentedMutex* db_mutex, ErrorHandler* db_error_handler,
|
||||
LogBuffer* log_buffer,
|
||||
Directory* db_directory, Directory* output_directory,
|
||||
Statistics* stats, InstrumentedMutex* db_mutex,
|
||||
Status* db_bg_error,
|
||||
std::vector<SequenceNumber> existing_snapshots,
|
||||
SequenceNumber earliest_write_conflict_snapshot,
|
||||
const SnapshotChecker* snapshot_checker,
|
||||
std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
|
||||
bool paranoid_file_checks, bool measure_io_stats,
|
||||
const std::string& dbname,
|
||||
CompactionJobStats* compaction_job_stats,
|
||||
Env::Priority thread_pri);
|
||||
CompactionJobStats* compaction_job_stats);
|
||||
|
||||
~CompactionJob();
|
||||
|
||||
@@ -104,7 +102,7 @@ class CompactionJob {
|
||||
|
||||
Status FinishCompactionOutputFile(
|
||||
const Status& input_status, SubcompactionState* sub_compact,
|
||||
CompactionRangeDelAggregator* range_del_agg,
|
||||
RangeDelAggregator* range_del_agg,
|
||||
CompactionIterationStats* range_del_out_stats,
|
||||
const Slice* next_table_min_key = nullptr);
|
||||
Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
|
||||
@@ -146,7 +144,7 @@ class CompactionJob {
|
||||
Directory* output_directory_;
|
||||
Statistics* stats_;
|
||||
InstrumentedMutex* db_mutex_;
|
||||
ErrorHandler* db_error_handler_;
|
||||
Status* db_bg_error_;
|
||||
// If there were two snapshots with seq numbers s1 and
|
||||
// s2 and s1 < s2, and if we find two instances of a key k1 then lies
|
||||
// entirely within s1 and s2, then the earlier version of k1 can be safely
|
||||
@@ -172,7 +170,6 @@ class CompactionJob {
|
||||
// Stores the approx size of keys covered in the range of each subcompaction
|
||||
std::vector<uint64_t> sizes_;
|
||||
Env::WriteLifeTimeHint write_hint_;
|
||||
Env::Priority thread_pri_;
|
||||
};
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@@ -98,7 +98,7 @@ class CompactionJobStatsTest : public testing::Test,
|
||||
CompactionJobStatsTest() : env_(Env::Default()) {
|
||||
env_->SetBackgroundThreads(1, Env::LOW);
|
||||
env_->SetBackgroundThreads(1, Env::HIGH);
|
||||
dbname_ = test::PerThreadDBPath("compaction_job_stats_test");
|
||||
dbname_ = test::TmpDir(env_) + "/compaction_job_stats_test";
|
||||
alternative_wal_dir_ = dbname_ + "/wal";
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
@@ -113,7 +113,7 @@ class CompactionJobStatsTest : public testing::Test,
|
||||
Reopen(options);
|
||||
}
|
||||
|
||||
~CompactionJobStatsTest() override {
|
||||
~CompactionJobStatsTest() {
|
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||
rocksdb::SyncPoint::GetInstance()->LoadDependency({});
|
||||
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
|
||||
@@ -426,7 +426,7 @@ class CompactionJobStatsChecker : public EventListener {
|
||||
// Once a compaction completed, this function will verify the returned
|
||||
// CompactionJobInfo with the oldest CompactionJobInfo added earlier
|
||||
// in "expected_stats_" which has not yet being used for verification.
|
||||
void OnCompactionCompleted(DB* /*db*/, const CompactionJobInfo& ci) override {
|
||||
virtual void OnCompactionCompleted(DB *db, const CompactionJobInfo& ci) {
|
||||
if (verify_next_comp_io_stats_) {
|
||||
ASSERT_GT(ci.stats.file_write_nanos, 0);
|
||||
ASSERT_GT(ci.stats.file_range_sync_nanos, 0);
|
||||
@@ -523,7 +523,7 @@ class CompactionJobDeletionStatsChecker : public CompactionJobStatsChecker {
|
||||
public:
|
||||
// Verifies whether two CompactionJobStats match.
|
||||
void Verify(const CompactionJobStats& current_stats,
|
||||
const CompactionJobStats& stats) override {
|
||||
const CompactionJobStats& stats) {
|
||||
ASSERT_EQ(
|
||||
current_stats.num_input_deletion_records,
|
||||
stats.num_input_deletion_records);
|
||||
@@ -806,7 +806,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
||||
stats_checker->set_verify_next_comp_io_stats(true);
|
||||
std::atomic<bool> first_prepare_write(true);
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"WritableFileWriter::Append:BeforePrepareWrite", [&](void* /*arg*/) {
|
||||
"WritableFileWriter::Append:BeforePrepareWrite", [&](void* arg) {
|
||||
if (first_prepare_write.load()) {
|
||||
options.env->SleepForMicroseconds(3);
|
||||
first_prepare_write.store(false);
|
||||
@@ -815,7 +815,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
||||
|
||||
std::atomic<bool> first_flush(true);
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"WritableFileWriter::Flush:BeforeAppend", [&](void* /*arg*/) {
|
||||
"WritableFileWriter::Flush:BeforeAppend", [&](void* arg) {
|
||||
if (first_flush.load()) {
|
||||
options.env->SleepForMicroseconds(3);
|
||||
first_flush.store(false);
|
||||
@@ -824,7 +824,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
||||
|
||||
std::atomic<bool> first_sync(true);
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"WritableFileWriter::SyncInternal:0", [&](void* /*arg*/) {
|
||||
"WritableFileWriter::SyncInternal:0", [&](void* arg) {
|
||||
if (first_sync.load()) {
|
||||
options.env->SleepForMicroseconds(3);
|
||||
first_sync.store(false);
|
||||
@@ -833,7 +833,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
|
||||
|
||||
std::atomic<bool> first_range_sync(true);
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"WritableFileWriter::RangeSync:0", [&](void* /*arg*/) {
|
||||
"WritableFileWriter::RangeSync:0", [&](void* arg) {
|
||||
if (first_range_sync.load()) {
|
||||
options.env->SleepForMicroseconds(3);
|
||||
first_range_sync.store(false);
|
||||
@@ -1034,7 +1034,7 @@ int main(int argc, char** argv) {
|
||||
#else
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) {
|
||||
int main(int argc, char** argv) {
|
||||
fprintf(stderr, "SKIPPED, not supported in ROCKSDB_LITE\n");
|
||||
return 0;
|
||||
}
|
||||
@@ -1043,5 +1043,5 @@ int main(int /*argc*/, char** /*argv*/) {
|
||||
|
||||
#else
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) { return 0; }
|
||||
int main(int argc, char** argv) { return 0; }
|
||||
#endif // !defined(IOS_CROSS_COMPILE)
|
||||
|
||||
+23
-25
@@ -12,7 +12,6 @@
|
||||
|
||||
#include "db/column_family.h"
|
||||
#include "db/compaction_job.h"
|
||||
#include "db/error_handler.h"
|
||||
#include "db/version_set.h"
|
||||
#include "rocksdb/cache.h"
|
||||
#include "rocksdb/db.h"
|
||||
@@ -68,7 +67,7 @@ class CompactionJobTest : public testing::Test {
|
||||
public:
|
||||
CompactionJobTest()
|
||||
: env_(Env::Default()),
|
||||
dbname_(test::PerThreadDBPath("compaction_job_test")),
|
||||
dbname_(test::TmpDir() + "/compaction_job_test"),
|
||||
db_options_(),
|
||||
mutable_cf_options_(cf_options_),
|
||||
table_cache_(NewLRUCache(50000, 16)),
|
||||
@@ -78,8 +77,7 @@ class CompactionJobTest : public testing::Test {
|
||||
&write_controller_)),
|
||||
shutting_down_(false),
|
||||
preserve_deletes_seqnum_(0),
|
||||
mock_table_factory_(new mock::MockTableFactory()),
|
||||
error_handler_(nullptr, db_options_, &mutex_) {
|
||||
mock_table_factory_(new mock::MockTableFactory()) {
|
||||
EXPECT_OK(env_->CreateDirIfMissing(dbname_));
|
||||
db_options_.db_paths.emplace_back(dbname_,
|
||||
std::numeric_limits<uint64_t>::max());
|
||||
@@ -172,7 +170,7 @@ class CompactionJobTest : public testing::Test {
|
||||
// This is how the key will look like once it's written in bottommost
|
||||
// file
|
||||
InternalKey bottommost_internal_key(
|
||||
key, 0, kTypeValue);
|
||||
key, (key == "9999") ? sequence_number : 0, kTypeValue);
|
||||
|
||||
if (corrupt_id(k)) {
|
||||
test::CorruptKeyType(&internal_key);
|
||||
@@ -200,12 +198,12 @@ class CompactionJobTest : public testing::Test {
|
||||
new_db.SetLastSequence(0);
|
||||
|
||||
const std::string manifest = DescriptorFileName(dbname_, 1);
|
||||
std::unique_ptr<WritableFile> file;
|
||||
unique_ptr<WritableFile> file;
|
||||
Status s = env_->NewWritableFile(
|
||||
manifest, &file, env_->OptimizeForManifestWrite(env_options_));
|
||||
ASSERT_OK(s);
|
||||
std::unique_ptr<WritableFileWriter> file_writer(
|
||||
new WritableFileWriter(std::move(file), manifest, env_options_));
|
||||
unique_ptr<WritableFileWriter> file_writer(
|
||||
new WritableFileWriter(std::move(file), env_options_));
|
||||
{
|
||||
log::Writer log(std::move(file_writer), 0, false);
|
||||
std::string record;
|
||||
@@ -248,8 +246,7 @@ class CompactionJobTest : public testing::Test {
|
||||
Compaction compaction(cfd->current()->storage_info(), *cfd->ioptions(),
|
||||
*cfd->GetLatestMutableCFOptions(),
|
||||
compaction_input_files, 1, 1024 * 1024,
|
||||
10 * 1024 * 1024, 0, kNoCompression,
|
||||
cfd->ioptions()->compression_opts, 0, {}, true);
|
||||
10 * 1024 * 1024, 0, kNoCompression, {}, true);
|
||||
compaction.SetInputVersion(cfd->current());
|
||||
|
||||
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, db_options_.info_log.get());
|
||||
@@ -257,13 +254,13 @@ class CompactionJobTest : public testing::Test {
|
||||
EventLogger event_logger(db_options_.info_log.get());
|
||||
// TODO(yiwu) add a mock snapshot checker and add test for it.
|
||||
SnapshotChecker* snapshot_checker = nullptr;
|
||||
CompactionJob compaction_job(
|
||||
0, &compaction, db_options_, env_options_, versions_.get(),
|
||||
&shutting_down_, preserve_deletes_seqnum_, &log_buffer, nullptr,
|
||||
nullptr, nullptr, &mutex_, &error_handler_, snapshots,
|
||||
earliest_write_conflict_snapshot, snapshot_checker, table_cache_,
|
||||
&event_logger, false, false, dbname_, &compaction_job_stats_,
|
||||
Env::Priority::USER);
|
||||
CompactionJob compaction_job(0, &compaction, db_options_, env_options_,
|
||||
versions_.get(), &shutting_down_,
|
||||
preserve_deletes_seqnum_, &log_buffer,
|
||||
nullptr, nullptr, nullptr, &mutex_, &bg_error_,
|
||||
snapshots, earliest_write_conflict_snapshot,
|
||||
snapshot_checker, table_cache_, &event_logger,
|
||||
false, false, dbname_, &compaction_job_stats_);
|
||||
VerifyInitializationOfCompactionJobStats(compaction_job_stats_);
|
||||
|
||||
compaction_job.Prepare();
|
||||
@@ -305,7 +302,7 @@ class CompactionJobTest : public testing::Test {
|
||||
ColumnFamilyData* cfd_;
|
||||
std::unique_ptr<CompactionFilter> compaction_filter_;
|
||||
std::shared_ptr<MergeOperator> merge_op_;
|
||||
ErrorHandler error_handler_;
|
||||
Status bg_error_;
|
||||
};
|
||||
|
||||
TEST_F(CompactionJobTest, Simple) {
|
||||
@@ -380,7 +377,7 @@ TEST_F(CompactionJobTest, SimpleOverwrite) {
|
||||
|
||||
auto expected_results =
|
||||
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "val2"},
|
||||
{KeyStr("b", 0U, kTypeValue), "val3"}});
|
||||
{KeyStr("b", 4U, kTypeValue), "val3"}});
|
||||
|
||||
SetLastSequence(4U);
|
||||
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
||||
@@ -433,7 +430,7 @@ TEST_F(CompactionJobTest, SimpleMerge) {
|
||||
|
||||
auto expected_results =
|
||||
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
|
||||
{KeyStr("b", 0U, kTypeValue), "1,2"}});
|
||||
{KeyStr("b", 2U, kTypeValue), "1,2"}});
|
||||
|
||||
SetLastSequence(5U);
|
||||
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
||||
@@ -457,7 +454,8 @@ TEST_F(CompactionJobTest, NonAssocMerge) {
|
||||
|
||||
auto expected_results =
|
||||
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), "3,4,5"},
|
||||
{KeyStr("b", 0U, kTypeValue), "1,2"}});
|
||||
{KeyStr("b", 2U, kTypeMerge), "2"},
|
||||
{KeyStr("b", 1U, kTypeMerge), "1"}});
|
||||
|
||||
SetLastSequence(5U);
|
||||
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
||||
@@ -484,7 +482,7 @@ TEST_F(CompactionJobTest, MergeOperandFilter) {
|
||||
|
||||
auto expected_results =
|
||||
mock::MakeMockFile({{KeyStr("a", 0U, kTypeValue), test::EncodeInt(8U)},
|
||||
{KeyStr("b", 0U, kTypeValue), test::EncodeInt(2U)}});
|
||||
{KeyStr("b", 2U, kTypeMerge), test::EncodeInt(2U)}});
|
||||
|
||||
SetLastSequence(5U);
|
||||
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
||||
@@ -747,7 +745,7 @@ TEST_F(CompactionJobTest, SingleDeleteZeroSeq) {
|
||||
AddMockFile(file2);
|
||||
|
||||
auto expected_results = mock::MakeMockFile({
|
||||
{KeyStr("dummy", 0U, kTypeValue), "val2"},
|
||||
{KeyStr("dummy", 5U, kTypeValue), "val2"},
|
||||
});
|
||||
|
||||
SetLastSequence(22U);
|
||||
@@ -931,7 +929,7 @@ TEST_F(CompactionJobTest, CorruptionAfterDeletion) {
|
||||
mock::MakeMockFile({{test::KeyStr("A", 0U, kTypeValue), "val3"},
|
||||
{test::KeyStr("a", 0U, kTypeValue, true), "val"},
|
||||
{test::KeyStr("b", 0U, kTypeValue, true), "val"},
|
||||
{test::KeyStr("c", 0U, kTypeValue), "val2"}});
|
||||
{test::KeyStr("c", 1U, kTypeValue), "val2"}});
|
||||
|
||||
SetLastSequence(6U);
|
||||
auto files = cfd_->current()->storage_info()->LevelFiles(0);
|
||||
@@ -948,7 +946,7 @@ int main(int argc, char** argv) {
|
||||
#else
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) {
|
||||
int main(int argc, char** argv) {
|
||||
fprintf(stderr,
|
||||
"SKIPPED as CompactionJobStats is not supported in ROCKSDB_LITE\n");
|
||||
return 0;
|
||||
|
||||
+290
-212
@@ -18,7 +18,6 @@
|
||||
#include <queue>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
#include "db/column_family.h"
|
||||
#include "monitoring/statistics.h"
|
||||
#include "util/filename.h"
|
||||
@@ -37,13 +36,12 @@ uint64_t TotalCompensatedFileSize(const std::vector<FileMetaData*>& files) {
|
||||
}
|
||||
return sum;
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
||||
size_t min_files_to_compact,
|
||||
uint64_t max_compact_bytes_per_del_file,
|
||||
CompactionInputFiles* comp_inputs) {
|
||||
size_t compact_bytes = static_cast<size_t>(level_files[0]->fd.file_size);
|
||||
size_t compact_bytes = level_files[0]->fd.file_size;
|
||||
size_t compact_bytes_per_del_file = port::kMaxSizet;
|
||||
// compaction range will be [0, span_len).
|
||||
size_t span_len;
|
||||
@@ -51,7 +49,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
||||
// increasing.
|
||||
size_t new_compact_bytes_per_del_file = 0;
|
||||
for (span_len = 1; span_len < level_files.size(); ++span_len) {
|
||||
compact_bytes += static_cast<size_t>(level_files[span_len]->fd.file_size);
|
||||
compact_bytes += level_files[span_len]->fd.file_size;
|
||||
new_compact_bytes_per_del_file = compact_bytes / span_len;
|
||||
if (level_files[span_len]->being_compacted ||
|
||||
new_compact_bytes_per_del_file > compact_bytes_per_del_file) {
|
||||
@@ -61,7 +59,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
||||
}
|
||||
|
||||
if (span_len >= min_files_to_compact &&
|
||||
compact_bytes_per_del_file < max_compact_bytes_per_del_file) {
|
||||
new_compact_bytes_per_del_file < max_compact_bytes_per_del_file) {
|
||||
assert(comp_inputs != nullptr);
|
||||
comp_inputs->level = 0;
|
||||
for (size_t i = 0; i < span_len; ++i) {
|
||||
@@ -71,6 +69,7 @@ bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
||||
}
|
||||
return false;
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
// Determine compression type, based on user options, level of the output
|
||||
// file and whether compression is disabled.
|
||||
@@ -111,24 +110,6 @@ CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
|
||||
}
|
||||
}
|
||||
|
||||
CompressionOptions GetCompressionOptions(const ImmutableCFOptions& ioptions,
|
||||
const VersionStorageInfo* vstorage,
|
||||
int level,
|
||||
const bool enable_compression) {
|
||||
if (!enable_compression) {
|
||||
return ioptions.compression_opts;
|
||||
}
|
||||
// If bottommost_compression is set and we are compacting to the
|
||||
// bottommost level then we should use the specified compression options
|
||||
// for the bottmomost_compression.
|
||||
if (ioptions.bottommost_compression != kDisableCompressionOption &&
|
||||
level >= (vstorage->num_non_empty_levels() - 1) &&
|
||||
ioptions.bottommost_compression_opts.enabled) {
|
||||
return ioptions.bottommost_compression_opts;
|
||||
}
|
||||
return ioptions.compression_opts;
|
||||
}
|
||||
|
||||
CompactionPicker::CompactionPicker(const ImmutableCFOptions& ioptions,
|
||||
const InternalKeyComparator* icmp)
|
||||
: ioptions_(ioptions), icmp_(icmp) {}
|
||||
@@ -218,10 +199,9 @@ void CompactionPicker::GetRange(const std::vector<CompactionInputFiles>& inputs,
|
||||
assert(initialized);
|
||||
}
|
||||
|
||||
bool CompactionPicker::ExpandInputsToCleanCut(const std::string& /*cf_name*/,
|
||||
bool CompactionPicker::ExpandInputsToCleanCut(const std::string& cf_name,
|
||||
VersionStorageInfo* vstorage,
|
||||
CompactionInputFiles* inputs,
|
||||
InternalKey** next_smallest) {
|
||||
CompactionInputFiles* inputs) {
|
||||
// This isn't good compaction
|
||||
assert(!inputs->empty());
|
||||
|
||||
@@ -244,8 +224,7 @@ bool CompactionPicker::ExpandInputsToCleanCut(const std::string& /*cf_name*/,
|
||||
GetRange(*inputs, &smallest, &largest);
|
||||
inputs->clear();
|
||||
vstorage->GetOverlappingInputs(level, &smallest, &largest, &inputs->files,
|
||||
hint_index, &hint_index, true,
|
||||
next_smallest);
|
||||
hint_index, &hint_index);
|
||||
} while (inputs->size() > old_size);
|
||||
|
||||
// we started off with inputs non-empty and the previous loop only grew
|
||||
@@ -318,29 +297,11 @@ Compaction* CompactionPicker::CompactFiles(
|
||||
// shouldn't have been released since.
|
||||
assert(!FilesRangeOverlapWithCompaction(input_files, output_level));
|
||||
|
||||
CompressionType compression_type;
|
||||
if (compact_options.compression == kDisableCompressionOption) {
|
||||
int base_level;
|
||||
if (ioptions_.compaction_style == kCompactionStyleLevel) {
|
||||
base_level = vstorage->base_level();
|
||||
} else {
|
||||
base_level = 1;
|
||||
}
|
||||
compression_type =
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
|
||||
output_level, base_level);
|
||||
} else {
|
||||
// TODO(ajkr): `CompactionOptions` offers configurable `CompressionType`
|
||||
// without configurable `CompressionOptions`, which is inconsistent.
|
||||
compression_type = compact_options.compression;
|
||||
}
|
||||
auto c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, input_files, output_level,
|
||||
compact_options.output_file_size_limit,
|
||||
mutable_cf_options.max_compaction_bytes, output_path_id, compression_type,
|
||||
GetCompressionOptions(ioptions_, vstorage, output_level),
|
||||
compact_options.max_subcompactions,
|
||||
/* grandparents */ {}, true);
|
||||
auto c =
|
||||
new Compaction(vstorage, ioptions_, mutable_cf_options, input_files,
|
||||
output_level, compact_options.output_file_size_limit,
|
||||
mutable_cf_options.max_compaction_bytes, output_path_id,
|
||||
compact_options.compression, /* grandparents */ {}, true);
|
||||
RegisterCompaction(c);
|
||||
return c;
|
||||
}
|
||||
@@ -348,7 +309,7 @@ Compaction* CompactionPicker::CompactFiles(
|
||||
Status CompactionPicker::GetCompactionInputsFromFileNumbers(
|
||||
std::vector<CompactionInputFiles>* input_files,
|
||||
std::unordered_set<uint64_t>* input_set, const VersionStorageInfo* vstorage,
|
||||
const CompactionOptions& /*compact_options*/) const {
|
||||
const CompactionOptions& compact_options) const {
|
||||
if (input_set->size() == 0U) {
|
||||
return Status::InvalidArgument(
|
||||
"Compaction must include at least one file.");
|
||||
@@ -403,7 +364,7 @@ bool CompactionPicker::IsRangeInCompaction(VersionStorageInfo* vstorage,
|
||||
assert(level < NumberLevels());
|
||||
|
||||
vstorage->GetOverlappingInputs(level, smallest, largest, &inputs,
|
||||
level_index ? *level_index : 0, level_index);
|
||||
*level_index, level_index);
|
||||
return AreFilesInCompaction(inputs);
|
||||
}
|
||||
|
||||
@@ -514,7 +475,7 @@ bool CompactionPicker::SetupOtherInputs(
|
||||
ROCKS_LOG_INFO(ioptions_.info_log,
|
||||
"[%s] Expanding@%d %" ROCKSDB_PRIszt "+%" ROCKSDB_PRIszt
|
||||
"(%" PRIu64 "+%" PRIu64 " bytes) to %" ROCKSDB_PRIszt
|
||||
"+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 " bytes)\n",
|
||||
"+%" ROCKSDB_PRIszt " (%" PRIu64 "+%" PRIu64 "bytes)\n",
|
||||
cf_name.c_str(), input_level, inputs->size(),
|
||||
output_level_inputs->size(), inputs_size,
|
||||
output_level_inputs_size, expanded_inputs.size(),
|
||||
@@ -543,8 +504,7 @@ void CompactionPicker::GetGrandparents(
|
||||
Compaction* CompactionPicker::CompactRange(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level, int output_level,
|
||||
uint32_t output_path_id, uint32_t max_subcompactions,
|
||||
const InternalKey* begin, const InternalKey* end,
|
||||
uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* manual_conflict) {
|
||||
// CompactionPickerFIFO has its own implementation of compact range
|
||||
assert(ioptions_.compaction_style != kCompactionStyleFIFO);
|
||||
@@ -606,14 +566,11 @@ Compaction* CompactionPicker::CompactRange(
|
||||
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
|
||||
output_level,
|
||||
MaxFileSizeForLevel(mutable_cf_options, output_level,
|
||||
ioptions_.compaction_style),
|
||||
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
/* max_compaction_bytes */ LLONG_MAX, output_path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
|
||||
output_level, 1),
|
||||
GetCompressionOptions(ioptions_, vstorage, output_level),
|
||||
max_subcompactions, /* grandparents */ {}, /* is manual */ true);
|
||||
/* grandparents */ {}, /* is manual */ true);
|
||||
RegisterCompaction(c);
|
||||
return c;
|
||||
}
|
||||
@@ -652,18 +609,16 @@ Compaction* CompactionPicker::CompactRange(
|
||||
uint64_t s = inputs[i]->compensated_file_size;
|
||||
total += s;
|
||||
if (total >= limit) {
|
||||
**compaction_end = inputs[i + 1]->smallest;
|
||||
covering_the_whole_range = false;
|
||||
inputs.files.resize(i + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
assert(output_path_id < static_cast<uint32_t>(ioptions_.cf_paths.size()));
|
||||
assert(output_path_id < static_cast<uint32_t>(ioptions_.db_paths.size()));
|
||||
|
||||
InternalKey key_storage;
|
||||
InternalKey* next_smallest = &key_storage;
|
||||
if (ExpandInputsToCleanCut(cf_name, vstorage, &inputs, &next_smallest) ==
|
||||
false) {
|
||||
if (ExpandInputsToCleanCut(cf_name, vstorage, &inputs) == false) {
|
||||
// manual compaction is now multi-threaded, so it can
|
||||
// happen that ExpandWhileOverlapping fails
|
||||
// we handle it higher in RunManualCompaction
|
||||
@@ -671,10 +626,8 @@ Compaction* CompactionPicker::CompactRange(
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (covering_the_whole_range || !next_smallest) {
|
||||
if (covering_the_whole_range) {
|
||||
*compaction_end = nullptr;
|
||||
} else {
|
||||
**compaction_end = *next_smallest;
|
||||
}
|
||||
|
||||
CompactionInputFiles output_level_inputs;
|
||||
@@ -720,16 +673,11 @@ Compaction* CompactionPicker::CompactRange(
|
||||
GetGrandparents(vstorage, inputs, output_level_inputs, &grandparents);
|
||||
Compaction* compaction = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(compaction_inputs),
|
||||
output_level,
|
||||
MaxFileSizeForLevel(mutable_cf_options, output_level,
|
||||
ioptions_.compaction_style, vstorage->base_level(),
|
||||
ioptions_.level_compaction_dynamic_level_bytes),
|
||||
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
mutable_cf_options.max_compaction_bytes, output_path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
|
||||
vstorage->base_level()),
|
||||
GetCompressionOptions(ioptions_, vstorage, output_level),
|
||||
max_subcompactions, std::move(grandparents),
|
||||
/* is manual compaction */ true);
|
||||
std::move(grandparents), /* is manual compaction */ true);
|
||||
|
||||
TEST_SYNC_POINT_CALLBACK("CompactionPicker::CompactRange:Return", compaction);
|
||||
RegisterCompaction(compaction);
|
||||
@@ -897,11 +845,6 @@ Status CompactionPicker::SanitizeCompactionInputFilesForAllLevels(
|
||||
}
|
||||
}
|
||||
}
|
||||
if (RangeOverlapWithCompaction(smallestkey, largestkey, output_level)) {
|
||||
return Status::Aborted(
|
||||
"A running compaction is writing to the same output level in an "
|
||||
"overlapping key range");
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
@@ -944,8 +887,8 @@ Status CompactionPicker::SanitizeCompactionInputFiles(
|
||||
// any currently-existing files.
|
||||
for (auto file_num : *input_files) {
|
||||
bool found = false;
|
||||
for (const auto& level_meta : cf_meta.levels) {
|
||||
for (const auto& file_meta : level_meta.files) {
|
||||
for (auto level_meta : cf_meta.levels) {
|
||||
for (auto file_meta : level_meta.files) {
|
||||
if (file_num == TableFileNameToNumber(file_meta.name)) {
|
||||
if (file_meta.being_compacted) {
|
||||
return Status::Aborted("Specified compaction input file " +
|
||||
@@ -996,83 +939,8 @@ void CompactionPicker::UnregisterCompaction(Compaction* c) {
|
||||
compactions_in_progress_.erase(c);
|
||||
}
|
||||
|
||||
void CompactionPicker::PickFilesMarkedForCompaction(
|
||||
const std::string& cf_name, VersionStorageInfo* vstorage, int* start_level,
|
||||
int* output_level, CompactionInputFiles* start_level_inputs) {
|
||||
if (vstorage->FilesMarkedForCompaction().empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto continuation = [&, cf_name](std::pair<int, FileMetaData*> level_file) {
|
||||
// If it's being compacted it has nothing to do here.
|
||||
// If this assert() fails that means that some function marked some
|
||||
// files as being_compacted, but didn't call ComputeCompactionScore()
|
||||
assert(!level_file.second->being_compacted);
|
||||
*start_level = level_file.first;
|
||||
*output_level =
|
||||
(*start_level == 0) ? vstorage->base_level() : *start_level + 1;
|
||||
|
||||
if (*start_level == 0 && !level0_compactions_in_progress()->empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
start_level_inputs->files = {level_file.second};
|
||||
start_level_inputs->level = *start_level;
|
||||
return ExpandInputsToCleanCut(cf_name, vstorage, start_level_inputs);
|
||||
};
|
||||
|
||||
// take a chance on a random file first
|
||||
Random64 rnd(/* seed */ reinterpret_cast<uint64_t>(vstorage));
|
||||
size_t random_file_index = static_cast<size_t>(rnd.Uniform(
|
||||
static_cast<uint64_t>(vstorage->FilesMarkedForCompaction().size())));
|
||||
|
||||
if (continuation(vstorage->FilesMarkedForCompaction()[random_file_index])) {
|
||||
// found the compaction!
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto& level_file : vstorage->FilesMarkedForCompaction()) {
|
||||
if (continuation(level_file)) {
|
||||
// found the compaction!
|
||||
return;
|
||||
}
|
||||
}
|
||||
start_level_inputs->files.clear();
|
||||
}
|
||||
|
||||
bool CompactionPicker::GetOverlappingL0Files(
|
||||
VersionStorageInfo* vstorage, CompactionInputFiles* start_level_inputs,
|
||||
int output_level, int* parent_index) {
|
||||
// Two level 0 compaction won't run at the same time, so don't need to worry
|
||||
// about files on level 0 being compacted.
|
||||
assert(level0_compactions_in_progress()->empty());
|
||||
InternalKey smallest, largest;
|
||||
GetRange(*start_level_inputs, &smallest, &largest);
|
||||
// Note that the next call will discard the file we placed in
|
||||
// c->inputs_[0] earlier and replace it with an overlapping set
|
||||
// which will include the picked file.
|
||||
start_level_inputs->files.clear();
|
||||
vstorage->GetOverlappingInputs(0, &smallest, &largest,
|
||||
&(start_level_inputs->files));
|
||||
|
||||
// If we include more L0 files in the same compaction run it can
|
||||
// cause the 'smallest' and 'largest' key to get extended to a
|
||||
// larger range. So, re-invoke GetRange to get the new key range
|
||||
GetRange(*start_level_inputs, &smallest, &largest);
|
||||
if (IsRangeInCompaction(vstorage, &smallest, &largest, output_level,
|
||||
parent_index)) {
|
||||
return false;
|
||||
}
|
||||
assert(!start_level_inputs->files.empty());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool LevelCompactionPicker::NeedsCompaction(
|
||||
const VersionStorageInfo* vstorage) const {
|
||||
if (!vstorage->ExpiredTtlFiles().empty()) {
|
||||
return true;
|
||||
}
|
||||
if (!vstorage->BottommostFilesMarkedForCompaction().empty()) {
|
||||
return true;
|
||||
}
|
||||
@@ -1139,7 +1007,8 @@ class LevelCompactionBuilder {
|
||||
// otherwise, returns false.
|
||||
bool PickIntraL0Compaction();
|
||||
|
||||
void PickExpiredTtlFiles();
|
||||
// If there is any file marked for compaction, put put it into inputs.
|
||||
void PickFilesMarkedForCompaction();
|
||||
|
||||
const std::string& cf_name_;
|
||||
VersionStorageInfo* vstorage_;
|
||||
@@ -1167,8 +1036,8 @@ class LevelCompactionBuilder {
|
||||
static const int kMinFilesForIntraL0Compaction = 4;
|
||||
};
|
||||
|
||||
void LevelCompactionBuilder::PickExpiredTtlFiles() {
|
||||
if (vstorage_->ExpiredTtlFiles().empty()) {
|
||||
void LevelCompactionBuilder::PickFilesMarkedForCompaction() {
|
||||
if (vstorage_->FilesMarkedForCompaction().empty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1181,9 +1050,8 @@ void LevelCompactionBuilder::PickExpiredTtlFiles() {
|
||||
output_level_ =
|
||||
(start_level_ == 0) ? vstorage_->base_level() : start_level_ + 1;
|
||||
|
||||
if ((start_level_ == vstorage_->num_non_empty_levels() - 1) ||
|
||||
(start_level_ == 0 &&
|
||||
!compaction_picker_->level0_compactions_in_progress()->empty())) {
|
||||
if (start_level_ == 0 &&
|
||||
!compaction_picker_->level0_compactions_in_progress()->empty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -1193,13 +1061,22 @@ void LevelCompactionBuilder::PickExpiredTtlFiles() {
|
||||
&start_level_inputs_);
|
||||
};
|
||||
|
||||
for (auto& level_file : vstorage_->ExpiredTtlFiles()) {
|
||||
// take a chance on a random file first
|
||||
Random64 rnd(/* seed */ reinterpret_cast<uint64_t>(vstorage_));
|
||||
size_t random_file_index = static_cast<size_t>(rnd.Uniform(
|
||||
static_cast<uint64_t>(vstorage_->FilesMarkedForCompaction().size())));
|
||||
|
||||
if (continuation(vstorage_->FilesMarkedForCompaction()[random_file_index])) {
|
||||
// found the compaction!
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto& level_file : vstorage_->FilesMarkedForCompaction()) {
|
||||
if (continuation(level_file)) {
|
||||
// found the compaction!
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
start_level_inputs_.files.clear();
|
||||
}
|
||||
|
||||
@@ -1254,51 +1131,61 @@ void LevelCompactionBuilder::SetupInitialFiles() {
|
||||
// if we didn't find a compaction, check if there are any files marked for
|
||||
// compaction
|
||||
if (start_level_inputs_.empty()) {
|
||||
is_manual_ = true;
|
||||
parent_index_ = base_index_ = -1;
|
||||
|
||||
// PickFilesMarkedForCompaction();
|
||||
compaction_picker_->PickFilesMarkedForCompaction(
|
||||
cf_name_, vstorage_, &start_level_, &output_level_, &start_level_inputs_);
|
||||
if (!start_level_inputs_.empty()) {
|
||||
is_manual_ = true;
|
||||
compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
|
||||
return;
|
||||
}
|
||||
|
||||
size_t i;
|
||||
for (i = 0; i < vstorage_->BottommostFilesMarkedForCompaction().size();
|
||||
++i) {
|
||||
auto& level_and_file = vstorage_->BottommostFilesMarkedForCompaction()[i];
|
||||
assert(!level_and_file.second->being_compacted);
|
||||
start_level_inputs_.level = output_level_ = start_level_ =
|
||||
level_and_file.first;
|
||||
start_level_inputs_.files = {level_and_file.second};
|
||||
if (compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
|
||||
&start_level_inputs_)) {
|
||||
break;
|
||||
PickFilesMarkedForCompaction();
|
||||
if (start_level_inputs_.empty()) {
|
||||
size_t i;
|
||||
for (i = 0; i < vstorage_->BottommostFilesMarkedForCompaction().size();
|
||||
++i) {
|
||||
auto& level_and_file =
|
||||
vstorage_->BottommostFilesMarkedForCompaction()[i];
|
||||
assert(!level_and_file.second->being_compacted);
|
||||
start_level_inputs_.level = output_level_ = start_level_ =
|
||||
level_and_file.first;
|
||||
start_level_inputs_.files = {level_and_file.second};
|
||||
if (compaction_picker_->ExpandInputsToCleanCut(cf_name_, vstorage_,
|
||||
&start_level_inputs_)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (i == vstorage_->BottommostFilesMarkedForCompaction().size()) {
|
||||
start_level_inputs_.clear();
|
||||
} else {
|
||||
assert(!start_level_inputs_.empty());
|
||||
compaction_reason_ = CompactionReason::kBottommostFiles;
|
||||
}
|
||||
}
|
||||
if (i == vstorage_->BottommostFilesMarkedForCompaction().size()) {
|
||||
start_level_inputs_.clear();
|
||||
} else {
|
||||
assert(!start_level_inputs_.empty());
|
||||
compaction_reason_ = CompactionReason::kBottommostFiles;
|
||||
return;
|
||||
}
|
||||
|
||||
assert(start_level_inputs_.empty());
|
||||
PickExpiredTtlFiles();
|
||||
if (!start_level_inputs_.empty()) {
|
||||
compaction_reason_ = CompactionReason::kTtl;
|
||||
compaction_reason_ = CompactionReason::kFilesMarkedForCompaction;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool LevelCompactionBuilder::SetupOtherL0FilesIfNeeded() {
|
||||
if (start_level_ == 0 && output_level_ != 0) {
|
||||
return compaction_picker_->GetOverlappingL0Files(
|
||||
vstorage_, &start_level_inputs_, output_level_, &parent_index_);
|
||||
// Two level 0 compaction won't run at the same time, so don't need to worry
|
||||
// about files on level 0 being compacted.
|
||||
assert(compaction_picker_->level0_compactions_in_progress()->empty());
|
||||
InternalKey smallest, largest;
|
||||
compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
|
||||
// Note that the next call will discard the file we placed in
|
||||
// c->inputs_[0] earlier and replace it with an overlapping set
|
||||
// which will include the picked file.
|
||||
start_level_inputs_.files.clear();
|
||||
vstorage_->GetOverlappingInputs(0, &smallest, &largest,
|
||||
&start_level_inputs_.files);
|
||||
|
||||
// If we include more L0 files in the same compaction run it can
|
||||
// cause the 'smallest' and 'largest' key to get extended to a
|
||||
// larger range. So, re-invoke GetRange to get the new key range
|
||||
compaction_picker_->GetRange(start_level_inputs_, &smallest, &largest);
|
||||
if (compaction_picker_->IsRangeInCompaction(
|
||||
vstorage_, &smallest, &largest, output_level_, &parent_index_)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
assert(!start_level_inputs_.files.empty());
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -1371,17 +1258,13 @@ Compaction* LevelCompactionBuilder::PickCompaction() {
|
||||
Compaction* LevelCompactionBuilder::GetCompaction() {
|
||||
auto c = new Compaction(
|
||||
vstorage_, ioptions_, mutable_cf_options_, std::move(compaction_inputs_),
|
||||
output_level_,
|
||||
MaxFileSizeForLevel(mutable_cf_options_, output_level_,
|
||||
ioptions_.compaction_style, vstorage_->base_level(),
|
||||
ioptions_.level_compaction_dynamic_level_bytes),
|
||||
output_level_, mutable_cf_options_.MaxFileSizeForLevel(output_level_),
|
||||
mutable_cf_options_.max_compaction_bytes,
|
||||
GetPathId(ioptions_, mutable_cf_options_, output_level_),
|
||||
GetCompressionType(ioptions_, vstorage_, mutable_cf_options_,
|
||||
output_level_, vstorage_->base_level()),
|
||||
GetCompressionOptions(ioptions_, vstorage_, output_level_),
|
||||
/* max_subcompactions */ 0, std::move(grandparents_), is_manual_,
|
||||
start_level_score_, false /* deletion_compaction */, compaction_reason_);
|
||||
std::move(grandparents_), is_manual_, start_level_score_,
|
||||
false /* deletion_compaction */, compaction_reason_);
|
||||
|
||||
// If it's level 0 compaction, make sure we don't execute any other level 0
|
||||
// compactions in parallel
|
||||
@@ -1404,10 +1287,10 @@ uint32_t LevelCompactionBuilder::GetPathId(
|
||||
const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options, int level) {
|
||||
uint32_t p = 0;
|
||||
assert(!ioptions.cf_paths.empty());
|
||||
assert(!ioptions.db_paths.empty());
|
||||
|
||||
// size remaining in the most recent path
|
||||
uint64_t current_path_size = ioptions.cf_paths[0].target_size;
|
||||
uint64_t current_path_size = ioptions.db_paths[0].target_size;
|
||||
|
||||
uint64_t level_size;
|
||||
int cur_level = 0;
|
||||
@@ -1417,7 +1300,7 @@ uint32_t LevelCompactionBuilder::GetPathId(
|
||||
level_size = mutable_cf_options.max_bytes_for_level_base;
|
||||
|
||||
// Last path is the fallback
|
||||
while (p < ioptions.cf_paths.size() - 1) {
|
||||
while (p < ioptions.db_paths.size() - 1) {
|
||||
if (level_size <= current_path_size) {
|
||||
if (cur_level == level) {
|
||||
// Does desired level fit in this path?
|
||||
@@ -1444,7 +1327,7 @@ uint32_t LevelCompactionBuilder::GetPathId(
|
||||
}
|
||||
}
|
||||
p++;
|
||||
current_path_size = ioptions.cf_paths[p].target_size;
|
||||
current_path_size = ioptions.db_paths[p].target_size;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
@@ -1548,4 +1431,199 @@ Compaction* LevelCompactionPicker::PickCompaction(
|
||||
return builder.PickCompaction();
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
bool FIFOCompactionPicker::NeedsCompaction(
|
||||
const VersionStorageInfo* vstorage) const {
|
||||
const int kLevel0 = 0;
|
||||
return vstorage->CompactionScore(kLevel0) >= 1;
|
||||
}
|
||||
|
||||
namespace {
|
||||
uint64_t GetTotalFilesSize(
|
||||
const std::vector<FileMetaData*>& files) {
|
||||
uint64_t total_size = 0;
|
||||
for (const auto& f : files) {
|
||||
total_size += f->fd.file_size;
|
||||
}
|
||||
return total_size;
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
Compaction* FIFOCompactionPicker::PickTTLCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
assert(mutable_cf_options.compaction_options_fifo.ttl > 0);
|
||||
|
||||
const int kLevel0 = 0;
|
||||
const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
|
||||
uint64_t total_size = GetTotalFilesSize(level_files);
|
||||
|
||||
int64_t _current_time;
|
||||
auto status = ioptions_.env->GetCurrentTime(&_current_time);
|
||||
if (!status.ok()) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] FIFO compaction: Couldn't get current time: %s. "
|
||||
"Not doing compactions based on TTL. ",
|
||||
cf_name.c_str(), status.ToString().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
||||
|
||||
std::vector<CompactionInputFiles> inputs;
|
||||
inputs.emplace_back();
|
||||
inputs[0].level = 0;
|
||||
|
||||
// avoid underflow
|
||||
if (current_time > mutable_cf_options.compaction_options_fifo.ttl) {
|
||||
for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
|
||||
auto f = *ritr;
|
||||
if (f->fd.table_reader != nullptr &&
|
||||
f->fd.table_reader->GetTableProperties() != nullptr) {
|
||||
auto creation_time =
|
||||
f->fd.table_reader->GetTableProperties()->creation_time;
|
||||
if (creation_time == 0 ||
|
||||
creation_time >= (current_time -
|
||||
mutable_cf_options.compaction_options_fifo.ttl)) {
|
||||
break;
|
||||
}
|
||||
total_size -= f->compensated_file_size;
|
||||
inputs[0].files.push_back(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return a nullptr and proceed to size-based FIFO compaction if:
|
||||
// 1. there are no files older than ttl OR
|
||||
// 2. there are a few files older than ttl, but deleting them will not bring
|
||||
// the total size to be less than max_table_files_size threshold.
|
||||
if (inputs[0].files.empty() ||
|
||||
total_size >
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (const auto& f : inputs[0].files) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] FIFO compaction: picking file %" PRIu64
|
||||
" with creation time %" PRIu64 " for deletion",
|
||||
cf_name.c_str(), f->fd.GetNumber(),
|
||||
f->fd.table_reader->GetTableProperties()->creation_time);
|
||||
}
|
||||
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
|
||||
kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ true, CompactionReason::kFIFOTtl);
|
||||
return c;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::PickSizeCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
const int kLevel0 = 0;
|
||||
const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
|
||||
uint64_t total_size = GetTotalFilesSize(level_files);
|
||||
|
||||
if (total_size <=
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size ||
|
||||
level_files.size() == 0) {
|
||||
// total size not exceeded
|
||||
if (mutable_cf_options.compaction_options_fifo.allow_compaction &&
|
||||
level_files.size() > 0) {
|
||||
CompactionInputFiles comp_inputs;
|
||||
if (FindIntraL0Compaction(
|
||||
level_files,
|
||||
mutable_cf_options
|
||||
.level0_file_num_compaction_trigger /* min_files_to_compact */,
|
||||
mutable_cf_options.write_buffer_size, &comp_inputs)) {
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
|
||||
16 * 1024 * 1024 /* output file size limit */,
|
||||
0 /* max compaction bytes, not applicable */,
|
||||
0 /* output path ID */, mutable_cf_options.compression, {},
|
||||
/* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ false,
|
||||
CompactionReason::kFIFOReduceNumFiles);
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
ROCKS_LOG_BUFFER(
|
||||
log_buffer,
|
||||
"[%s] FIFO compaction: nothing to do. Total size %" PRIu64
|
||||
", max size %" PRIu64 "\n",
|
||||
cf_name.c_str(), total_size,
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!level0_compactions_in_progress_.empty()) {
|
||||
ROCKS_LOG_BUFFER(
|
||||
log_buffer,
|
||||
"[%s] FIFO compaction: Already executing compaction. No need "
|
||||
"to run parallel compactions since compactions are very fast",
|
||||
cf_name.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<CompactionInputFiles> inputs;
|
||||
inputs.emplace_back();
|
||||
inputs[0].level = 0;
|
||||
|
||||
for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
|
||||
auto f = *ritr;
|
||||
total_size -= f->compensated_file_size;
|
||||
inputs[0].files.push_back(f);
|
||||
char tmp_fsize[16];
|
||||
AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] FIFO compaction: picking file %" PRIu64
|
||||
" with size %s for deletion",
|
||||
cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
|
||||
if (total_size <=
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
|
||||
kNoCompression, {}, /* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
|
||||
return c;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::PickCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
assert(vstorage->num_levels() == 1);
|
||||
|
||||
Compaction* c = nullptr;
|
||||
if (mutable_cf_options.compaction_options_fifo.ttl > 0) {
|
||||
c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
|
||||
}
|
||||
if (c == nullptr) {
|
||||
c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
|
||||
}
|
||||
RegisterCompaction(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::CompactRange(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level, int output_level,
|
||||
uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* manual_conflict) {
|
||||
assert(input_level == 0);
|
||||
assert(output_level == 0);
|
||||
*compaction_end = nullptr;
|
||||
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log);
|
||||
Compaction* c =
|
||||
PickCompaction(cf_name, mutable_cf_options, vstorage, &log_buffer);
|
||||
log_buffer.FlushBufferToLog();
|
||||
return c;
|
||||
}
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
+49
-38
@@ -58,8 +58,7 @@ class CompactionPicker {
|
||||
virtual Compaction* CompactRange(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level, int output_level,
|
||||
uint32_t output_path_id, uint32_t max_subcompactions,
|
||||
const InternalKey* begin, const InternalKey* end,
|
||||
uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* manual_conflict);
|
||||
|
||||
// The maximum allowed output level. Default value is NumberLevels() - 1.
|
||||
@@ -151,8 +150,7 @@ class CompactionPicker {
|
||||
// Will return false if it is impossible to apply this compaction.
|
||||
bool ExpandInputsToCleanCut(const std::string& cf_name,
|
||||
VersionStorageInfo* vstorage,
|
||||
CompactionInputFiles* inputs,
|
||||
InternalKey** next_smallest = nullptr);
|
||||
CompactionInputFiles* inputs);
|
||||
|
||||
// Returns true if any one of the parent files are being compacted
|
||||
bool IsRangeInCompaction(VersionStorageInfo* vstorage,
|
||||
@@ -176,15 +174,6 @@ class CompactionPicker {
|
||||
const CompactionInputFiles& output_level_inputs,
|
||||
std::vector<FileMetaData*>* grandparents);
|
||||
|
||||
void PickFilesMarkedForCompaction(const std::string& cf_name,
|
||||
VersionStorageInfo* vstorage,
|
||||
int* start_level, int* output_level,
|
||||
CompactionInputFiles* start_level_inputs);
|
||||
|
||||
bool GetOverlappingL0Files(VersionStorageInfo* vstorage,
|
||||
CompactionInputFiles* start_level_inputs,
|
||||
int output_level, int* parent_index);
|
||||
|
||||
// Register this compaction in the set of running compactions
|
||||
void RegisterCompaction(Compaction* c);
|
||||
|
||||
@@ -235,6 +224,41 @@ class LevelCompactionPicker : public CompactionPicker {
|
||||
};
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
class FIFOCompactionPicker : public CompactionPicker {
|
||||
public:
|
||||
FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
|
||||
const InternalKeyComparator* icmp)
|
||||
: CompactionPicker(ioptions, icmp) {}
|
||||
|
||||
virtual Compaction* PickCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* version,
|
||||
LogBuffer* log_buffer) override;
|
||||
|
||||
virtual Compaction* CompactRange(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level, int output_level,
|
||||
uint32_t output_path_id, const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* manual_conflict) override;
|
||||
|
||||
// The maximum allowed output level. Always returns 0.
|
||||
virtual int MaxOutputLevel() const override { return 0; }
|
||||
|
||||
virtual bool NeedsCompaction(
|
||||
const VersionStorageInfo* vstorage) const override;
|
||||
|
||||
private:
|
||||
Compaction* PickTTLCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* version,
|
||||
LogBuffer* log_buffer);
|
||||
|
||||
Compaction* PickSizeCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* version,
|
||||
LogBuffer* log_buffer);
|
||||
};
|
||||
|
||||
class NullCompactionPicker : public CompactionPicker {
|
||||
public:
|
||||
NullCompactionPicker(const ImmutableCFOptions& ioptions,
|
||||
@@ -243,49 +267,36 @@ class NullCompactionPicker : public CompactionPicker {
|
||||
virtual ~NullCompactionPicker() {}
|
||||
|
||||
// Always return "nullptr"
|
||||
Compaction* PickCompaction(const std::string& /*cf_name*/,
|
||||
const MutableCFOptions& /*mutable_cf_options*/,
|
||||
VersionStorageInfo* /*vstorage*/,
|
||||
LogBuffer* /*log_buffer*/) override {
|
||||
Compaction* PickCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage,
|
||||
LogBuffer* log_buffer) override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Always return "nullptr"
|
||||
Compaction* CompactRange(const std::string& /*cf_name*/,
|
||||
const MutableCFOptions& /*mutable_cf_options*/,
|
||||
VersionStorageInfo* /*vstorage*/,
|
||||
int /*input_level*/, int /*output_level*/,
|
||||
uint32_t /*output_path_id*/,
|
||||
uint32_t /*max_subcompactions*/,
|
||||
const InternalKey* /*begin*/,
|
||||
const InternalKey* /*end*/,
|
||||
InternalKey** /*compaction_end*/,
|
||||
bool* /*manual_conflict*/) override {
|
||||
Compaction* CompactRange(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level,
|
||||
int output_level, uint32_t output_path_id,
|
||||
const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end,
|
||||
bool* manual_conflict) override {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Always returns false.
|
||||
virtual bool NeedsCompaction(
|
||||
const VersionStorageInfo* /*vstorage*/) const override {
|
||||
const VersionStorageInfo* vstorage) const override {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
bool FindIntraL0Compaction(const std::vector<FileMetaData*>& level_files,
|
||||
size_t min_files_to_compact,
|
||||
uint64_t max_compact_bytes_per_del_file,
|
||||
CompactionInputFiles* comp_inputs);
|
||||
|
||||
CompressionType GetCompressionType(const ImmutableCFOptions& ioptions,
|
||||
const VersionStorageInfo* vstorage,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
int level, int base_level,
|
||||
const bool enable_compression = true);
|
||||
|
||||
CompressionOptions GetCompressionOptions(const ImmutableCFOptions& ioptions,
|
||||
const VersionStorageInfo* vstorage,
|
||||
int level,
|
||||
const bool enable_compression = true);
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
@@ -1,234 +0,0 @@
|
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "db/compaction_picker_fifo.h"
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
#ifndef __STDC_FORMAT_MACROS
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#endif
|
||||
|
||||
#include <inttypes.h>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "db/column_family.h"
|
||||
#include "util/log_buffer.h"
|
||||
#include "util/string_util.h"
|
||||
|
||||
namespace rocksdb {
|
||||
namespace {
|
||||
uint64_t GetTotalFilesSize(const std::vector<FileMetaData*>& files) {
|
||||
uint64_t total_size = 0;
|
||||
for (const auto& f : files) {
|
||||
total_size += f->fd.file_size;
|
||||
}
|
||||
return total_size;
|
||||
}
|
||||
} // anonymous namespace
|
||||
|
||||
bool FIFOCompactionPicker::NeedsCompaction(
|
||||
const VersionStorageInfo* vstorage) const {
|
||||
const int kLevel0 = 0;
|
||||
return vstorage->CompactionScore(kLevel0) >= 1;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::PickTTLCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
assert(mutable_cf_options.ttl > 0);
|
||||
|
||||
const int kLevel0 = 0;
|
||||
const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
|
||||
uint64_t total_size = GetTotalFilesSize(level_files);
|
||||
|
||||
int64_t _current_time;
|
||||
auto status = ioptions_.env->GetCurrentTime(&_current_time);
|
||||
if (!status.ok()) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] FIFO compaction: Couldn't get current time: %s. "
|
||||
"Not doing compactions based on TTL. ",
|
||||
cf_name.c_str(), status.ToString().c_str());
|
||||
return nullptr;
|
||||
}
|
||||
const uint64_t current_time = static_cast<uint64_t>(_current_time);
|
||||
|
||||
std::vector<CompactionInputFiles> inputs;
|
||||
inputs.emplace_back();
|
||||
inputs[0].level = 0;
|
||||
|
||||
// avoid underflow
|
||||
if (current_time > mutable_cf_options.ttl) {
|
||||
for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
|
||||
auto f = *ritr;
|
||||
if (f->fd.table_reader != nullptr &&
|
||||
f->fd.table_reader->GetTableProperties() != nullptr) {
|
||||
auto creation_time =
|
||||
f->fd.table_reader->GetTableProperties()->creation_time;
|
||||
if (creation_time == 0 ||
|
||||
creation_time >= (current_time - mutable_cf_options.ttl)) {
|
||||
break;
|
||||
}
|
||||
total_size -= f->compensated_file_size;
|
||||
inputs[0].files.push_back(f);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return a nullptr and proceed to size-based FIFO compaction if:
|
||||
// 1. there are no files older than ttl OR
|
||||
// 2. there are a few files older than ttl, but deleting them will not bring
|
||||
// the total size to be less than max_table_files_size threshold.
|
||||
if (inputs[0].files.empty() ||
|
||||
total_size >
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
for (const auto& f : inputs[0].files) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] FIFO compaction: picking file %" PRIu64
|
||||
" with creation time %" PRIu64 " for deletion",
|
||||
cf_name.c_str(), f->fd.GetNumber(),
|
||||
f->fd.table_reader->GetTableProperties()->creation_time);
|
||||
}
|
||||
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
|
||||
kNoCompression, ioptions_.compression_opts, /* max_subcompactions */ 0,
|
||||
{}, /* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ true, CompactionReason::kFIFOTtl);
|
||||
return c;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::PickSizeCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
const int kLevel0 = 0;
|
||||
const std::vector<FileMetaData*>& level_files = vstorage->LevelFiles(kLevel0);
|
||||
uint64_t total_size = GetTotalFilesSize(level_files);
|
||||
|
||||
if (total_size <=
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size ||
|
||||
level_files.size() == 0) {
|
||||
// total size not exceeded
|
||||
if (mutable_cf_options.compaction_options_fifo.allow_compaction &&
|
||||
level_files.size() > 0) {
|
||||
CompactionInputFiles comp_inputs;
|
||||
// try to prevent same files from being compacted multiple times, which
|
||||
// could produce large files that may never TTL-expire. Achieve this by
|
||||
// disallowing compactions with files larger than memtable (inflate its
|
||||
// size by 10% to account for uncompressed L0 files that may have size
|
||||
// slightly greater than memtable size limit).
|
||||
size_t max_compact_bytes_per_del_file =
|
||||
static_cast<size_t>(MultiplyCheckOverflow(
|
||||
static_cast<uint64_t>(mutable_cf_options.write_buffer_size),
|
||||
1.1));
|
||||
if (FindIntraL0Compaction(
|
||||
level_files,
|
||||
mutable_cf_options
|
||||
.level0_file_num_compaction_trigger /* min_files_to_compact */
|
||||
,
|
||||
max_compact_bytes_per_del_file, &comp_inputs)) {
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, {comp_inputs}, 0,
|
||||
16 * 1024 * 1024 /* output file size limit */,
|
||||
0 /* max compaction bytes, not applicable */,
|
||||
0 /* output path ID */, mutable_cf_options.compression,
|
||||
ioptions_.compression_opts, 0 /* max_subcompactions */, {},
|
||||
/* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ false,
|
||||
CompactionReason::kFIFOReduceNumFiles);
|
||||
return c;
|
||||
}
|
||||
}
|
||||
|
||||
ROCKS_LOG_BUFFER(
|
||||
log_buffer,
|
||||
"[%s] FIFO compaction: nothing to do. Total size %" PRIu64
|
||||
", max size %" PRIu64 "\n",
|
||||
cf_name.c_str(), total_size,
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size);
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
if (!level0_compactions_in_progress_.empty()) {
|
||||
ROCKS_LOG_BUFFER(
|
||||
log_buffer,
|
||||
"[%s] FIFO compaction: Already executing compaction. No need "
|
||||
"to run parallel compactions since compactions are very fast",
|
||||
cf_name.c_str());
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::vector<CompactionInputFiles> inputs;
|
||||
inputs.emplace_back();
|
||||
inputs[0].level = 0;
|
||||
|
||||
for (auto ritr = level_files.rbegin(); ritr != level_files.rend(); ++ritr) {
|
||||
auto f = *ritr;
|
||||
total_size -= f->compensated_file_size;
|
||||
inputs[0].files.push_back(f);
|
||||
char tmp_fsize[16];
|
||||
AppendHumanBytes(f->fd.GetFileSize(), tmp_fsize, sizeof(tmp_fsize));
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] FIFO compaction: picking file %" PRIu64
|
||||
" with size %s for deletion",
|
||||
cf_name.c_str(), f->fd.GetNumber(), tmp_fsize);
|
||||
if (total_size <=
|
||||
mutable_cf_options.compaction_options_fifo.max_table_files_size) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Compaction* c = new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), 0, 0, 0, 0,
|
||||
kNoCompression, ioptions_.compression_opts, /* max_subcompactions */ 0,
|
||||
{}, /* is manual */ false, vstorage->CompactionScore(0),
|
||||
/* is deletion compaction */ true, CompactionReason::kFIFOMaxSize);
|
||||
return c;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::PickCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, LogBuffer* log_buffer) {
|
||||
assert(vstorage->num_levels() == 1);
|
||||
|
||||
Compaction* c = nullptr;
|
||||
if (mutable_cf_options.ttl > 0) {
|
||||
c = PickTTLCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
|
||||
}
|
||||
if (c == nullptr) {
|
||||
c = PickSizeCompaction(cf_name, mutable_cf_options, vstorage, log_buffer);
|
||||
}
|
||||
RegisterCompaction(c);
|
||||
return c;
|
||||
}
|
||||
|
||||
Compaction* FIFOCompactionPicker::CompactRange(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level, int output_level,
|
||||
uint32_t /*output_path_id*/, uint32_t /*max_subcompactions*/,
|
||||
const InternalKey* /*begin*/, const InternalKey* /*end*/,
|
||||
InternalKey** compaction_end, bool* /*manual_conflict*/) {
|
||||
#ifdef NDEBUG
|
||||
(void)input_level;
|
||||
(void)output_level;
|
||||
#endif
|
||||
assert(input_level == 0);
|
||||
assert(output_level == 0);
|
||||
*compaction_end = nullptr;
|
||||
LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, ioptions_.info_log);
|
||||
Compaction* c =
|
||||
PickCompaction(cf_name, mutable_cf_options, vstorage, &log_buffer);
|
||||
log_buffer.FlushBufferToLog();
|
||||
return c;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
#endif // !ROCKSDB_LITE
|
||||
@@ -1,52 +0,0 @@
|
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
//
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#pragma once
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
#include "db/compaction_picker.h"
|
||||
|
||||
namespace rocksdb {
|
||||
class FIFOCompactionPicker : public CompactionPicker {
|
||||
public:
|
||||
FIFOCompactionPicker(const ImmutableCFOptions& ioptions,
|
||||
const InternalKeyComparator* icmp)
|
||||
: CompactionPicker(ioptions, icmp) {}
|
||||
|
||||
virtual Compaction* PickCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* version,
|
||||
LogBuffer* log_buffer) override;
|
||||
|
||||
virtual Compaction* CompactRange(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, int input_level, int output_level,
|
||||
uint32_t output_path_id, uint32_t max_subcompactions,
|
||||
const InternalKey* begin, const InternalKey* end,
|
||||
InternalKey** compaction_end, bool* manual_conflict) override;
|
||||
|
||||
// The maximum allowed output level. Always returns 0.
|
||||
virtual int MaxOutputLevel() const override { return 0; }
|
||||
|
||||
virtual bool NeedsCompaction(
|
||||
const VersionStorageInfo* vstorage) const override;
|
||||
|
||||
private:
|
||||
Compaction* PickTTLCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* version,
|
||||
LogBuffer* log_buffer);
|
||||
|
||||
Compaction* PickSizeCompaction(const std::string& cf_name,
|
||||
const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* version,
|
||||
LogBuffer* log_buffer);
|
||||
};
|
||||
} // namespace rocksdb
|
||||
#endif // !ROCKSDB_LITE
|
||||
@@ -4,12 +4,10 @@
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "db/compaction_picker.h"
|
||||
|
||||
#include <limits>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include "db/compaction.h"
|
||||
#include "db/compaction_picker_fifo.h"
|
||||
#include "db/compaction_picker_universal.h"
|
||||
|
||||
#include "util/logging.h"
|
||||
@@ -22,7 +20,7 @@ namespace rocksdb {
|
||||
class CountingLogger : public Logger {
|
||||
public:
|
||||
using Logger::Logv;
|
||||
void Logv(const char* /*format*/, va_list /*ap*/) override { log_count++; }
|
||||
virtual void Logv(const char* format, va_list ap) override { log_count++; }
|
||||
size_t log_count;
|
||||
};
|
||||
|
||||
@@ -57,16 +55,14 @@ class CompactionPickerTest : public testing::Test {
|
||||
log_buffer_(InfoLogLevel::INFO_LEVEL, &logger_),
|
||||
file_num_(1),
|
||||
vstorage_(nullptr) {
|
||||
// ioptions_.compaction_pri = kMinOverlappingRatio has its own set of
|
||||
// tests to cover.
|
||||
ioptions_.compaction_pri = kByCompensatedSize;
|
||||
fifo_options_.max_table_files_size = 1;
|
||||
mutable_cf_options_.RefreshDerivedOptions(ioptions_);
|
||||
ioptions_.cf_paths.emplace_back("dummy",
|
||||
ioptions_.db_paths.emplace_back("dummy",
|
||||
std::numeric_limits<uint64_t>::max());
|
||||
}
|
||||
|
||||
~CompactionPickerTest() override {}
|
||||
~CompactionPickerTest() {
|
||||
}
|
||||
|
||||
void NewVersionStorage(int num_levels, CompactionStyle style) {
|
||||
DeleteVersionStorage();
|
||||
@@ -85,17 +81,16 @@ class CompactionPickerTest : public testing::Test {
|
||||
|
||||
void Add(int level, uint32_t file_number, const char* smallest,
|
||||
const char* largest, uint64_t file_size = 1, uint32_t path_id = 0,
|
||||
SequenceNumber smallest_seq = 100, SequenceNumber largest_seq = 100,
|
||||
size_t compensated_file_size = 0) {
|
||||
SequenceNumber smallest_seq = 100,
|
||||
SequenceNumber largest_seq = 100) {
|
||||
assert(level < vstorage_->num_levels());
|
||||
FileMetaData* f = new FileMetaData;
|
||||
f->fd = FileDescriptor(file_number, path_id, file_size);
|
||||
f->smallest = InternalKey(smallest, smallest_seq, kTypeValue);
|
||||
f->largest = InternalKey(largest, largest_seq, kTypeValue);
|
||||
f->fd.smallest_seqno = smallest_seq;
|
||||
f->fd.largest_seqno = largest_seq;
|
||||
f->compensated_file_size =
|
||||
(compensated_file_size != 0) ? compensated_file_size : file_size;
|
||||
f->smallest_seqno = smallest_seq;
|
||||
f->largest_seqno = largest_seq;
|
||||
f->compensated_file_size = file_size;
|
||||
f->refs = 0;
|
||||
vstorage_->AddFile(level, f);
|
||||
files_.emplace_back(f);
|
||||
@@ -394,10 +389,10 @@ TEST_F(CompactionPickerTest, NeedsCompactionUniversal) {
|
||||
NewVersionStorage(1, kCompactionStyleUniversal);
|
||||
UniversalCompactionPicker universal_compaction_picker(
|
||||
ioptions_, &icmp_);
|
||||
UpdateVersionStorageInfo();
|
||||
// must return false when there's no files.
|
||||
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
|
||||
false);
|
||||
UpdateVersionStorageInfo();
|
||||
|
||||
// verify the trigger given different number of L0 files.
|
||||
for (int i = 1;
|
||||
@@ -418,7 +413,6 @@ TEST_F(CompactionPickerTest, CompactionUniversalIngestBehindReservedLevel) {
|
||||
ioptions_.allow_ingest_behind = true;
|
||||
ioptions_.num_levels = 3;
|
||||
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
|
||||
UpdateVersionStorageInfo();
|
||||
// must return false when there's no files.
|
||||
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
|
||||
false);
|
||||
@@ -452,7 +446,6 @@ TEST_F(CompactionPickerTest, CannotTrivialMoveUniversal) {
|
||||
mutable_cf_options_.compaction_options_universal.allow_trivial_move = true;
|
||||
NewVersionStorage(1, kCompactionStyleUniversal);
|
||||
UniversalCompactionPicker universal_compaction_picker(ioptions_, &icmp_);
|
||||
UpdateVersionStorageInfo();
|
||||
// must return false when there's no files.
|
||||
ASSERT_EQ(universal_compaction_picker.NeedsCompaction(vstorage_.get()),
|
||||
false);
|
||||
@@ -618,35 +611,6 @@ TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
|
||||
ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
|
||||
}
|
||||
|
||||
TEST_F(CompactionPickerTest, CompactionPriMinOverlapping4) {
|
||||
NewVersionStorage(6, kCompactionStyleLevel);
|
||||
ioptions_.compaction_pri = kMinOverlappingRatio;
|
||||
mutable_cf_options_.max_bytes_for_level_base = 10000000;
|
||||
mutable_cf_options_.max_bytes_for_level_multiplier = 10;
|
||||
|
||||
// file 7 and 8 over lap with the same file, but file 8 is smaller so
|
||||
// it will be picked.
|
||||
// Overlaps with file 26, 27. And the file is compensated so will be
|
||||
// picked up.
|
||||
Add(2, 6U, "150", "167", 60000000U, 0, 100, 100, 180000000U);
|
||||
Add(2, 7U, "168", "169", 60000000U); // Overlaps with file 27
|
||||
Add(2, 8U, "201", "300", 61000000U); // Overlaps with file 28
|
||||
|
||||
Add(3, 26U, "160", "165", 60000000U);
|
||||
// Boosted file size in output level is not considered.
|
||||
Add(3, 27U, "166", "170", 60000000U, 0, 100, 100, 260000000U);
|
||||
Add(3, 28U, "180", "400", 60000000U);
|
||||
Add(3, 29U, "401", "500", 60000000U);
|
||||
UpdateVersionStorageInfo();
|
||||
|
||||
std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
|
||||
cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
|
||||
ASSERT_TRUE(compaction.get() != nullptr);
|
||||
ASSERT_EQ(1U, compaction->num_input_files(0));
|
||||
// Picking file 8 because overlapping ratio is the biggest.
|
||||
ASSERT_EQ(6U, compaction->input(0, 0)->fd.GetNumber());
|
||||
}
|
||||
|
||||
// This test exhibits the bug where we don't properly reset parent_index in
|
||||
// PickCompaction()
|
||||
TEST_F(CompactionPickerTest, ParentIndexResetBug) {
|
||||
|
||||
@@ -97,17 +97,17 @@ void GetSmallestLargestSeqno(const std::vector<FileMetaData*>& files,
|
||||
SequenceNumber* largest_seqno) {
|
||||
bool is_first = true;
|
||||
for (FileMetaData* f : files) {
|
||||
assert(f->fd.smallest_seqno <= f->fd.largest_seqno);
|
||||
assert(f->smallest_seqno <= f->largest_seqno);
|
||||
if (is_first) {
|
||||
is_first = false;
|
||||
*smallest_seqno = f->fd.smallest_seqno;
|
||||
*largest_seqno = f->fd.largest_seqno;
|
||||
*smallest_seqno = f->smallest_seqno;
|
||||
*largest_seqno = f->largest_seqno;
|
||||
} else {
|
||||
if (f->fd.smallest_seqno < *smallest_seqno) {
|
||||
*smallest_seqno = f->fd.smallest_seqno;
|
||||
if (f->smallest_seqno < *smallest_seqno) {
|
||||
*smallest_seqno = f->smallest_seqno;
|
||||
}
|
||||
if (f->fd.largest_seqno > *largest_seqno) {
|
||||
*largest_seqno = f->fd.largest_seqno;
|
||||
if (f->largest_seqno > *largest_seqno) {
|
||||
*largest_seqno = f->largest_seqno;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -162,13 +162,7 @@ bool UniversalCompactionPicker::IsInputFilesNonOverlapping(Compaction* c) {
|
||||
bool UniversalCompactionPicker::NeedsCompaction(
|
||||
const VersionStorageInfo* vstorage) const {
|
||||
const int kLevel0 = 0;
|
||||
if (vstorage->CompactionScore(kLevel0) >= 1) {
|
||||
return true;
|
||||
}
|
||||
if (!vstorage->FilesMarkedForCompaction().empty()) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
return vstorage->CompactionScore(kLevel0) >= 1;
|
||||
}
|
||||
|
||||
void UniversalCompactionPicker::SortedRun::Dump(char* out_buf,
|
||||
@@ -210,7 +204,7 @@ void UniversalCompactionPicker::SortedRun::DumpSizeInfo(
|
||||
|
||||
std::vector<UniversalCompactionPicker::SortedRun>
|
||||
UniversalCompactionPicker::CalculateSortedRuns(
|
||||
const VersionStorageInfo& vstorage, const ImmutableCFOptions& /*ioptions*/,
|
||||
const VersionStorageInfo& vstorage, const ImmutableCFOptions& ioptions,
|
||||
const MutableCFOptions& mutable_cf_options) {
|
||||
std::vector<UniversalCompactionPicker::SortedRun> ret;
|
||||
for (FileMetaData* f : vstorage.LevelFiles(0)) {
|
||||
@@ -263,9 +257,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
CalculateSortedRuns(*vstorage, ioptions_, mutable_cf_options);
|
||||
|
||||
if (sorted_runs.size() == 0 ||
|
||||
(vstorage->FilesMarkedForCompaction().empty() &&
|
||||
sorted_runs.size() < (unsigned int)mutable_cf_options
|
||||
.level0_file_num_compaction_trigger)) {
|
||||
sorted_runs.size() <
|
||||
(unsigned int)mutable_cf_options.level0_file_num_compaction_trigger) {
|
||||
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: nothing to do\n",
|
||||
cf_name.c_str());
|
||||
TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
|
||||
@@ -279,73 +272,58 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
cf_name.c_str(), sorted_runs.size(), vstorage->LevelSummary(&tmp));
|
||||
|
||||
// Check for size amplification first.
|
||||
Compaction* c = nullptr;
|
||||
if (sorted_runs.size() >=
|
||||
static_cast<size_t>(
|
||||
mutable_cf_options.level0_file_num_compaction_trigger)) {
|
||||
if ((c = PickCompactionToReduceSizeAmp(cf_name, mutable_cf_options,
|
||||
vstorage, score, sorted_runs,
|
||||
log_buffer)) != nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: compacting for size amp\n",
|
||||
Compaction* c;
|
||||
if ((c = PickCompactionToReduceSizeAmp(cf_name, mutable_cf_options, vstorage,
|
||||
score, sorted_runs, log_buffer)) !=
|
||||
nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer, "[%s] Universal: compacting for size amp\n",
|
||||
cf_name.c_str());
|
||||
} else {
|
||||
// Size amplification is within limits. Try reducing read
|
||||
// amplification while maintaining file size ratios.
|
||||
unsigned int ratio =
|
||||
mutable_cf_options.compaction_options_universal.size_ratio;
|
||||
|
||||
if ((c = PickCompactionToReduceSortedRuns(
|
||||
cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
|
||||
sorted_runs, log_buffer)) != nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] Universal: compacting for size ratio\n",
|
||||
cf_name.c_str());
|
||||
} else {
|
||||
// Size amplification is within limits. Try reducing read
|
||||
// amplification while maintaining file size ratios.
|
||||
unsigned int ratio =
|
||||
mutable_cf_options.compaction_options_universal.size_ratio;
|
||||
|
||||
if ((c = PickCompactionToReduceSortedRuns(
|
||||
cf_name, mutable_cf_options, vstorage, score, ratio, UINT_MAX,
|
||||
sorted_runs, log_buffer)) != nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] Universal: compacting for size ratio\n",
|
||||
cf_name.c_str());
|
||||
} else {
|
||||
// Size amplification and file size ratios are within configured limits.
|
||||
// If max read amplification is exceeding configured limits, then force
|
||||
// compaction without looking at filesize ratios and try to reduce
|
||||
// the number of files to fewer than level0_file_num_compaction_trigger.
|
||||
// This is guaranteed by NeedsCompaction()
|
||||
assert(sorted_runs.size() >=
|
||||
static_cast<size_t>(
|
||||
mutable_cf_options.level0_file_num_compaction_trigger));
|
||||
// Get the total number of sorted runs that are not being compacted
|
||||
int num_sr_not_compacted = 0;
|
||||
for (size_t i = 0; i < sorted_runs.size(); i++) {
|
||||
if (sorted_runs[i].being_compacted == false) {
|
||||
num_sr_not_compacted++;
|
||||
}
|
||||
// Size amplification and file size ratios are within configured limits.
|
||||
// If max read amplification is exceeding configured limits, then force
|
||||
// compaction without looking at filesize ratios and try to reduce
|
||||
// the number of files to fewer than level0_file_num_compaction_trigger.
|
||||
// This is guaranteed by NeedsCompaction()
|
||||
assert(sorted_runs.size() >=
|
||||
static_cast<size_t>(
|
||||
mutable_cf_options.level0_file_num_compaction_trigger));
|
||||
// Get the total number of sorted runs that are not being compacted
|
||||
int num_sr_not_compacted = 0;
|
||||
for (size_t i = 0; i < sorted_runs.size(); i++) {
|
||||
if (sorted_runs[i].being_compacted == false) {
|
||||
num_sr_not_compacted++;
|
||||
}
|
||||
}
|
||||
|
||||
// The number of sorted runs that are not being compacted is greater
|
||||
// than the maximum allowed number of sorted runs
|
||||
if (num_sr_not_compacted >
|
||||
mutable_cf_options.level0_file_num_compaction_trigger) {
|
||||
unsigned int num_files =
|
||||
num_sr_not_compacted -
|
||||
mutable_cf_options.level0_file_num_compaction_trigger + 1;
|
||||
if ((c = PickCompactionToReduceSortedRuns(
|
||||
cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
|
||||
num_files, sorted_runs, log_buffer)) != nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] Universal: compacting for file num -- %u\n",
|
||||
cf_name.c_str(), num_files);
|
||||
}
|
||||
// The number of sorted runs that are not being compacted is greater than
|
||||
// the maximum allowed number of sorted runs
|
||||
if (num_sr_not_compacted >
|
||||
mutable_cf_options.level0_file_num_compaction_trigger) {
|
||||
unsigned int num_files =
|
||||
num_sr_not_compacted -
|
||||
mutable_cf_options.level0_file_num_compaction_trigger + 1;
|
||||
if ((c = PickCompactionToReduceSortedRuns(
|
||||
cf_name, mutable_cf_options, vstorage, score, UINT_MAX,
|
||||
num_files, sorted_runs, log_buffer)) != nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] Universal: compacting for file num -- %u\n",
|
||||
cf_name.c_str(), num_files);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (c == nullptr) {
|
||||
if ((c = PickDeleteTriggeredCompaction(cf_name, mutable_cf_options,
|
||||
vstorage, score, sorted_runs,
|
||||
log_buffer)) != nullptr) {
|
||||
ROCKS_LOG_BUFFER(log_buffer,
|
||||
"[%s] Universal: delete triggered compaction\n",
|
||||
cf_name.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (c == nullptr) {
|
||||
TEST_SYNC_POINT_CALLBACK("UniversalCompactionPicker::PickCompaction:Return",
|
||||
nullptr);
|
||||
@@ -365,11 +343,11 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
size_t level_index = 0U;
|
||||
if (c->start_level() == 0) {
|
||||
for (auto f : *c->inputs(0)) {
|
||||
assert(f->fd.smallest_seqno <= f->fd.largest_seqno);
|
||||
assert(f->smallest_seqno <= f->largest_seqno);
|
||||
if (is_first) {
|
||||
is_first = false;
|
||||
}
|
||||
prev_smallest_seqno = f->fd.smallest_seqno;
|
||||
prev_smallest_seqno = f->smallest_seqno;
|
||||
}
|
||||
level_index = 1U;
|
||||
}
|
||||
@@ -395,8 +373,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(
|
||||
}
|
||||
#endif
|
||||
// update statistics
|
||||
RecordInHistogram(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
|
||||
c->inputs(0)->size());
|
||||
MeasureTime(ioptions_.statistics, NUM_FILES_IN_SINGLE_COMPACTION,
|
||||
c->inputs(0)->size());
|
||||
|
||||
RegisterCompaction(c);
|
||||
vstorage->ComputeCompactionScore(ioptions_, mutable_cf_options);
|
||||
@@ -428,9 +406,9 @@ uint32_t UniversalCompactionPicker::GetPathId(
|
||||
file_size *
|
||||
(100 - mutable_cf_options.compaction_options_universal.size_ratio) / 100;
|
||||
uint32_t p = 0;
|
||||
assert(!ioptions.cf_paths.empty());
|
||||
for (; p < ioptions.cf_paths.size() - 1; p++) {
|
||||
uint64_t target_size = ioptions.cf_paths[p].target_size;
|
||||
assert(!ioptions.db_paths.empty());
|
||||
for (; p < ioptions.db_paths.size() - 1; p++) {
|
||||
uint64_t target_size = ioptions.db_paths[p].target_size;
|
||||
if (target_size > file_size &&
|
||||
accumulated_size + (target_size - file_size) > future_size) {
|
||||
return p;
|
||||
@@ -625,21 +603,17 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSortedRuns(
|
||||
|
||||
CompactionReason compaction_reason;
|
||||
if (max_number_of_files_to_compact == UINT_MAX) {
|
||||
compaction_reason = CompactionReason::kUniversalSizeRatio;
|
||||
} else {
|
||||
compaction_reason = CompactionReason::kUniversalSortedRunNum;
|
||||
} else {
|
||||
compaction_reason = CompactionReason::kUniversalSizeRatio;
|
||||
}
|
||||
return new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
|
||||
MaxFileSizeForLevel(mutable_cf_options, output_level,
|
||||
kCompactionStyleUniversal),
|
||||
LLONG_MAX, path_id,
|
||||
mutable_cf_options.MaxFileSizeForLevel(output_level), LLONG_MAX, path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, start_level,
|
||||
1, enable_compression),
|
||||
GetCompressionOptions(ioptions_, vstorage, start_level,
|
||||
enable_compression),
|
||||
/* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
|
||||
score, false /* deletion_compaction */, compaction_reason);
|
||||
/* grandparents */ {}, /* is manual */ false, score,
|
||||
false /* deletion_compaction */, compaction_reason);
|
||||
}
|
||||
|
||||
// Look at overall size amplification. If size amplification
|
||||
@@ -661,10 +635,6 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
|
||||
size_t start_index = 0;
|
||||
const SortedRun* sr = nullptr;
|
||||
|
||||
if (sorted_runs.back().being_compacted) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Skip files that are already being compacted
|
||||
for (size_t loop = 0; loop < sorted_runs.size() - 1; loop++) {
|
||||
sr = &sorted_runs[loop];
|
||||
@@ -771,137 +741,15 @@ Compaction* UniversalCompactionPicker::PickCompactionToReduceSizeAmp(
|
||||
}
|
||||
|
||||
return new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
|
||||
MaxFileSizeForLevel(mutable_cf_options, output_level,
|
||||
kCompactionStyleUniversal),
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs),
|
||||
output_level, mutable_cf_options.MaxFileSizeForLevel(output_level),
|
||||
/* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
|
||||
1),
|
||||
GetCompressionOptions(ioptions_, vstorage, output_level),
|
||||
/* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ false,
|
||||
score, false /* deletion_compaction */,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options,
|
||||
output_level, 1),
|
||||
/* grandparents */ {}, /* is manual */ false, score,
|
||||
false /* deletion_compaction */,
|
||||
CompactionReason::kUniversalSizeAmplification);
|
||||
}
|
||||
|
||||
// Pick files marked for compaction. Typically, files are marked by
|
||||
// CompactOnDeleteCollector due to the presence of tombstones.
|
||||
Compaction* UniversalCompactionPicker::PickDeleteTriggeredCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, double score,
|
||||
const std::vector<SortedRun>& /*sorted_runs*/, LogBuffer* /*log_buffer*/) {
|
||||
CompactionInputFiles start_level_inputs;
|
||||
int output_level;
|
||||
std::vector<CompactionInputFiles> inputs;
|
||||
|
||||
if (vstorage->num_levels() == 1) {
|
||||
// This is single level universal. Since we're basically trying to reclaim
|
||||
// space by processing files marked for compaction due to high tombstone
|
||||
// density, let's do the same thing as compaction to reduce size amp which
|
||||
// has the same goals.
|
||||
bool compact = false;
|
||||
|
||||
start_level_inputs.level = 0;
|
||||
start_level_inputs.files.clear();
|
||||
output_level = 0;
|
||||
for (FileMetaData* f : vstorage->LevelFiles(0)) {
|
||||
if (f->marked_for_compaction) {
|
||||
compact = true;
|
||||
}
|
||||
if (compact) {
|
||||
start_level_inputs.files.push_back(f);
|
||||
}
|
||||
}
|
||||
if (start_level_inputs.size() <= 1) {
|
||||
// If only the last file in L0 is marked for compaction, ignore it
|
||||
return nullptr;
|
||||
}
|
||||
inputs.push_back(start_level_inputs);
|
||||
} else {
|
||||
int start_level;
|
||||
|
||||
// For multi-level universal, the strategy is to make this look more like
|
||||
// leveled. We pick one of the files marked for compaction and compact with
|
||||
// overlapping files in the adjacent level.
|
||||
PickFilesMarkedForCompaction(cf_name, vstorage, &start_level, &output_level,
|
||||
&start_level_inputs);
|
||||
if (start_level_inputs.empty()) {
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
// Pick the first non-empty level after the start_level
|
||||
for (output_level = start_level + 1; output_level < vstorage->num_levels();
|
||||
output_level++) {
|
||||
if (vstorage->NumLevelFiles(output_level) != 0) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// If all higher levels are empty, pick the highest level as output level
|
||||
if (output_level == vstorage->num_levels()) {
|
||||
if (start_level == 0) {
|
||||
output_level = vstorage->num_levels() - 1;
|
||||
} else {
|
||||
// If start level is non-zero and all higher levels are empty, this
|
||||
// compaction will translate into a trivial move. Since the idea is
|
||||
// to reclaim space and trivial move doesn't help with that, we
|
||||
// skip compaction in this case and return nullptr
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
if (ioptions_.allow_ingest_behind &&
|
||||
output_level == vstorage->num_levels() - 1) {
|
||||
assert(output_level > 1);
|
||||
output_level--;
|
||||
}
|
||||
|
||||
if (output_level != 0) {
|
||||
if (start_level == 0) {
|
||||
if (!GetOverlappingL0Files(vstorage, &start_level_inputs, output_level,
|
||||
nullptr)) {
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
CompactionInputFiles output_level_inputs;
|
||||
int parent_index = -1;
|
||||
|
||||
output_level_inputs.level = output_level;
|
||||
if (!SetupOtherInputs(cf_name, mutable_cf_options, vstorage,
|
||||
&start_level_inputs, &output_level_inputs,
|
||||
&parent_index, -1)) {
|
||||
return nullptr;
|
||||
}
|
||||
inputs.push_back(start_level_inputs);
|
||||
if (!output_level_inputs.empty()) {
|
||||
inputs.push_back(output_level_inputs);
|
||||
}
|
||||
if (FilesRangeOverlapWithCompaction(inputs, output_level)) {
|
||||
return nullptr;
|
||||
}
|
||||
} else {
|
||||
inputs.push_back(start_level_inputs);
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t estimated_total_size = 0;
|
||||
// Use size of the output level as estimated file size
|
||||
for (FileMetaData* f : vstorage->LevelFiles(output_level)) {
|
||||
estimated_total_size += f->fd.GetFileSize();
|
||||
}
|
||||
uint32_t path_id =
|
||||
GetPathId(ioptions_, mutable_cf_options, estimated_total_size);
|
||||
return new Compaction(
|
||||
vstorage, ioptions_, mutable_cf_options, std::move(inputs), output_level,
|
||||
MaxFileSizeForLevel(mutable_cf_options, output_level,
|
||||
kCompactionStyleUniversal),
|
||||
/* max_grandparent_overlap_bytes */ LLONG_MAX, path_id,
|
||||
GetCompressionType(ioptions_, vstorage, mutable_cf_options, output_level,
|
||||
1),
|
||||
GetCompressionOptions(ioptions_, vstorage, output_level),
|
||||
/* max_subcompactions */ 0, /* grandparents */ {}, /* is manual */ true,
|
||||
score, false /* deletion_compaction */,
|
||||
CompactionReason::kFilesMarkedForCompaction);
|
||||
}
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
@@ -73,11 +73,6 @@ class UniversalCompactionPicker : public CompactionPicker {
|
||||
VersionStorageInfo* vstorage, double score,
|
||||
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
|
||||
|
||||
Compaction* PickDeleteTriggeredCompaction(
|
||||
const std::string& cf_name, const MutableCFOptions& mutable_cf_options,
|
||||
VersionStorageInfo* vstorage, double score,
|
||||
const std::vector<SortedRun>& sorted_runs, LogBuffer* log_buffer);
|
||||
|
||||
// Used in universal compaction when the enabled_trivial_move
|
||||
// option is set. Checks whether there are any overlapping files
|
||||
// in the input. Returns true if the input files are non
|
||||
|
||||
+37
-256
@@ -2,7 +2,6 @@
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
#include <array>
|
||||
#include <map>
|
||||
#include <string>
|
||||
|
||||
@@ -27,24 +26,24 @@ class KVIter : public Iterator {
|
||||
public:
|
||||
explicit KVIter(const stl_wrappers::KVMap* map)
|
||||
: map_(map), iter_(map_->end()) {}
|
||||
bool Valid() const override { return iter_ != map_->end(); }
|
||||
void SeekToFirst() override { iter_ = map_->begin(); }
|
||||
void SeekToLast() override {
|
||||
virtual bool Valid() const override { return iter_ != map_->end(); }
|
||||
virtual void SeekToFirst() override { iter_ = map_->begin(); }
|
||||
virtual void SeekToLast() override {
|
||||
if (map_->empty()) {
|
||||
iter_ = map_->end();
|
||||
} else {
|
||||
iter_ = map_->find(map_->rbegin()->first);
|
||||
}
|
||||
}
|
||||
void Seek(const Slice& k) override {
|
||||
virtual void Seek(const Slice& k) override {
|
||||
iter_ = map_->lower_bound(k.ToString());
|
||||
}
|
||||
void SeekForPrev(const Slice& k) override {
|
||||
virtual void SeekForPrev(const Slice& k) override {
|
||||
iter_ = map_->upper_bound(k.ToString());
|
||||
Prev();
|
||||
}
|
||||
void Next() override { ++iter_; }
|
||||
void Prev() override {
|
||||
virtual void Next() override { ++iter_; }
|
||||
virtual void Prev() override {
|
||||
if (iter_ == map_->begin()) {
|
||||
iter_ = map_->end();
|
||||
return;
|
||||
@@ -52,9 +51,9 @@ class KVIter : public Iterator {
|
||||
--iter_;
|
||||
}
|
||||
|
||||
Slice key() const override { return iter_->first; }
|
||||
Slice value() const override { return iter_->second; }
|
||||
Status status() const override { return Status::OK(); }
|
||||
virtual Slice key() const override { return iter_->first; }
|
||||
virtual Slice value() const override { return iter_->second; }
|
||||
virtual Status status() const override { return Status::OK(); }
|
||||
|
||||
private:
|
||||
const stl_wrappers::KVMap* const map_;
|
||||
@@ -171,9 +170,9 @@ class DoubleComparator : public Comparator {
|
||||
public:
|
||||
DoubleComparator() {}
|
||||
|
||||
const char* Name() const override { return "DoubleComparator"; }
|
||||
virtual const char* Name() const override { return "DoubleComparator"; }
|
||||
|
||||
int Compare(const Slice& a, const Slice& b) const override {
|
||||
virtual int Compare(const Slice& a, const Slice& b) const override {
|
||||
#ifndef CYGWIN
|
||||
double da = std::stod(a.ToString());
|
||||
double db = std::stod(b.ToString());
|
||||
@@ -189,19 +188,19 @@ class DoubleComparator : public Comparator {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
void FindShortestSeparator(std::string* /*start*/,
|
||||
const Slice& /*limit*/) const override {}
|
||||
virtual void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const override {}
|
||||
|
||||
void FindShortSuccessor(std::string* /*key*/) const override {}
|
||||
virtual void FindShortSuccessor(std::string* key) const override {}
|
||||
};
|
||||
|
||||
class HashComparator : public Comparator {
|
||||
public:
|
||||
HashComparator() {}
|
||||
|
||||
const char* Name() const override { return "HashComparator"; }
|
||||
virtual const char* Name() const override { return "HashComparator"; }
|
||||
|
||||
int Compare(const Slice& a, const Slice& b) const override {
|
||||
virtual int Compare(const Slice& a, const Slice& b) const override {
|
||||
uint32_t ha = Hash(a.data(), a.size(), 66);
|
||||
uint32_t hb = Hash(b.data(), b.size(), 66);
|
||||
if (ha == hb) {
|
||||
@@ -212,19 +211,19 @@ class HashComparator : public Comparator {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
void FindShortestSeparator(std::string* /*start*/,
|
||||
const Slice& /*limit*/) const override {}
|
||||
virtual void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const override {}
|
||||
|
||||
void FindShortSuccessor(std::string* /*key*/) const override {}
|
||||
virtual void FindShortSuccessor(std::string* key) const override {}
|
||||
};
|
||||
|
||||
class TwoStrComparator : public Comparator {
|
||||
public:
|
||||
TwoStrComparator() {}
|
||||
|
||||
const char* Name() const override { return "TwoStrComparator"; }
|
||||
virtual const char* Name() const override { return "TwoStrComparator"; }
|
||||
|
||||
int Compare(const Slice& a, const Slice& b) const override {
|
||||
virtual int Compare(const Slice& a, const Slice& b) const override {
|
||||
assert(a.size() >= 2);
|
||||
assert(b.size() >= 2);
|
||||
size_t size_a1 = static_cast<size_t>(a[0]);
|
||||
@@ -244,16 +243,14 @@ class TwoStrComparator : public Comparator {
|
||||
}
|
||||
return a2.compare(b2);
|
||||
}
|
||||
void FindShortestSeparator(std::string* /*start*/,
|
||||
const Slice& /*limit*/) const override {}
|
||||
virtual void FindShortestSeparator(std::string* start,
|
||||
const Slice& limit) const override {}
|
||||
|
||||
void FindShortSuccessor(std::string* /*key*/) const override {}
|
||||
virtual void FindShortSuccessor(std::string* key) const override {}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
class ComparatorDBTest
|
||||
: public testing::Test,
|
||||
virtual public ::testing::WithParamInterface<uint32_t> {
|
||||
class ComparatorDBTest : public testing::Test {
|
||||
private:
|
||||
std::string dbname_;
|
||||
Env* env_;
|
||||
@@ -264,15 +261,11 @@ class ComparatorDBTest
|
||||
public:
|
||||
ComparatorDBTest() : env_(Env::Default()), db_(nullptr) {
|
||||
comparator = BytewiseComparator();
|
||||
dbname_ = test::PerThreadDBPath("comparator_db_test");
|
||||
BlockBasedTableOptions toptions;
|
||||
toptions.format_version = GetParam();
|
||||
last_options_.table_factory.reset(
|
||||
rocksdb::NewBlockBasedTableFactory(toptions));
|
||||
dbname_ = test::TmpDir() + "/comparator_db_test";
|
||||
EXPECT_OK(DestroyDB(dbname_, last_options_));
|
||||
}
|
||||
|
||||
~ComparatorDBTest() override {
|
||||
~ComparatorDBTest() {
|
||||
delete db_;
|
||||
EXPECT_OK(DestroyDB(dbname_, last_options_));
|
||||
comparator = BytewiseComparator();
|
||||
@@ -280,12 +273,8 @@ class ComparatorDBTest
|
||||
|
||||
DB* GetDB() { return db_; }
|
||||
|
||||
void SetOwnedComparator(const Comparator* cmp, bool owner = true) {
|
||||
if (owner) {
|
||||
comparator_guard.reset(cmp);
|
||||
} else {
|
||||
comparator_guard.reset();
|
||||
}
|
||||
void SetOwnedComparator(const Comparator* cmp) {
|
||||
comparator_guard.reset(cmp);
|
||||
comparator = cmp;
|
||||
last_options_.comparator = cmp;
|
||||
}
|
||||
@@ -314,12 +303,7 @@ class ComparatorDBTest
|
||||
}
|
||||
};
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(FormatDef, ComparatorDBTest,
|
||||
testing::Values(test::kDefaultFormatVersion));
|
||||
INSTANTIATE_TEST_CASE_P(FormatLatest, ComparatorDBTest,
|
||||
testing::Values(test::kLatestFormatVersion));
|
||||
|
||||
TEST_P(ComparatorDBTest, Bytewise) {
|
||||
TEST_F(ComparatorDBTest, Bytewise) {
|
||||
for (int rand_seed = 301; rand_seed < 306; rand_seed++) {
|
||||
DestroyAndReopen();
|
||||
Random rnd(rand_seed);
|
||||
@@ -329,7 +313,7 @@ TEST_P(ComparatorDBTest, Bytewise) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, SimpleSuffixReverseComparator) {
|
||||
TEST_F(ComparatorDBTest, SimpleSuffixReverseComparator) {
|
||||
SetOwnedComparator(new test::SimpleSuffixReverseComparator());
|
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
|
||||
@@ -355,8 +339,8 @@ TEST_P(ComparatorDBTest, SimpleSuffixReverseComparator) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, Uint64Comparator) {
|
||||
SetOwnedComparator(test::Uint64Comparator(), false /* owner */);
|
||||
TEST_F(ComparatorDBTest, Uint64Comparator) {
|
||||
SetOwnedComparator(test::Uint64Comparator());
|
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
|
||||
Options* opt = GetOptions();
|
||||
@@ -379,7 +363,7 @@ TEST_P(ComparatorDBTest, Uint64Comparator) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, DoubleComparator) {
|
||||
TEST_F(ComparatorDBTest, DoubleComparator) {
|
||||
SetOwnedComparator(new DoubleComparator());
|
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
|
||||
@@ -404,7 +388,7 @@ TEST_P(ComparatorDBTest, DoubleComparator) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, HashComparator) {
|
||||
TEST_F(ComparatorDBTest, HashComparator) {
|
||||
SetOwnedComparator(new HashComparator());
|
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
|
||||
@@ -423,7 +407,7 @@ TEST_P(ComparatorDBTest, HashComparator) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, TwoStrComparator) {
|
||||
TEST_F(ComparatorDBTest, TwoStrComparator) {
|
||||
SetOwnedComparator(new TwoStrComparator());
|
||||
|
||||
for (int rnd_seed = 301; rnd_seed < 316; rnd_seed++) {
|
||||
@@ -449,209 +433,6 @@ TEST_P(ComparatorDBTest, TwoStrComparator) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, IsSameLengthImmediateSuccessor) {
|
||||
{
|
||||
// different length
|
||||
Slice s("abcxy");
|
||||
Slice t("abcxyz");
|
||||
ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
Slice s("abcxyz");
|
||||
Slice t("abcxy");
|
||||
ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
// not last byte different
|
||||
Slice s("abc1xyz");
|
||||
Slice t("abc2xyz");
|
||||
ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
// same string
|
||||
Slice s("abcxyz");
|
||||
Slice t("abcxyz");
|
||||
ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
Slice s("abcxy");
|
||||
Slice t("abcxz");
|
||||
ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
Slice s("abcxz");
|
||||
Slice t("abcxy");
|
||||
ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
const char s_array[] = "\x50\x8a\xac";
|
||||
const char t_array[] = "\x50\x8a\xad";
|
||||
Slice s(s_array);
|
||||
Slice t(t_array);
|
||||
ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
const char s_array[] = "\x50\x8a\xff";
|
||||
const char t_array[] = "\x50\x8b\x00";
|
||||
Slice s(s_array, 3);
|
||||
Slice t(t_array, 3);
|
||||
ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
const char s_array[] = "\x50\x8a\xff\xff";
|
||||
const char t_array[] = "\x50\x8b\x00\x00";
|
||||
Slice s(s_array, 4);
|
||||
Slice t(t_array, 4);
|
||||
ASSERT_TRUE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
{
|
||||
const char s_array[] = "\x50\x8a\xff\xff";
|
||||
const char t_array[] = "\x50\x8b\x00\x01";
|
||||
Slice s(s_array, 4);
|
||||
Slice t(t_array, 4);
|
||||
ASSERT_FALSE(BytewiseComparator()->IsSameLengthImmediateSuccessor(s, t));
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, FindShortestSeparator) {
|
||||
std::string s1 = "abc1xyz";
|
||||
std::string s2 = "abc3xy";
|
||||
|
||||
BytewiseComparator()->FindShortestSeparator(&s1, s2);
|
||||
ASSERT_EQ("abc2", s1);
|
||||
|
||||
s1 = "abc5xyztt";
|
||||
|
||||
ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
|
||||
ASSERT_EQ("abc5", s1);
|
||||
|
||||
s1 = "abc3";
|
||||
s2 = "abc2xy";
|
||||
ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
|
||||
ASSERT_EQ("abc3", s1);
|
||||
|
||||
s1 = "abc3xyz";
|
||||
s2 = "abc2xy";
|
||||
ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
|
||||
ASSERT_EQ("abc3", s1);
|
||||
|
||||
s1 = "abc3xyz";
|
||||
s2 = "abc2";
|
||||
ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
|
||||
ASSERT_EQ("abc3", s1);
|
||||
|
||||
std::string old_s1 = s1 = "abc2xy";
|
||||
s2 = "abc2";
|
||||
ReverseBytewiseComparator()->FindShortestSeparator(&s1, s2);
|
||||
ASSERT_TRUE(old_s1 >= s1);
|
||||
ASSERT_TRUE(s1 > s2);
|
||||
}
|
||||
|
||||
TEST_P(ComparatorDBTest, SeparatorSuccessorRandomizeTest) {
|
||||
// Char list for boundary cases.
|
||||
std::array<unsigned char, 6> char_list{{0, 1, 2, 253, 254, 255}};
|
||||
Random rnd(301);
|
||||
|
||||
for (int attempts = 0; attempts < 1000; attempts++) {
|
||||
uint32_t size1 = rnd.Skewed(4);
|
||||
uint32_t size2;
|
||||
|
||||
if (rnd.OneIn(2)) {
|
||||
// size2 to be random size
|
||||
size2 = rnd.Skewed(4);
|
||||
} else {
|
||||
// size1 is within [-2, +2] of size1
|
||||
int diff = static_cast<int>(rnd.Uniform(5)) - 2;
|
||||
int tmp_size2 = static_cast<int>(size1) + diff;
|
||||
if (tmp_size2 < 0) {
|
||||
tmp_size2 = 0;
|
||||
}
|
||||
size2 = static_cast<uint32_t>(tmp_size2);
|
||||
}
|
||||
|
||||
std::string s1;
|
||||
std::string s2;
|
||||
for (uint32_t i = 0; i < size1; i++) {
|
||||
if (rnd.OneIn(2)) {
|
||||
// Use random byte
|
||||
s1 += static_cast<char>(rnd.Uniform(256));
|
||||
} else {
|
||||
// Use one byte in char_list
|
||||
char c = static_cast<char>(char_list[rnd.Uniform(sizeof(char_list))]);
|
||||
s1 += c;
|
||||
}
|
||||
}
|
||||
|
||||
// First set s2 to be the same as s1, and then modify s2.
|
||||
s2 = s1;
|
||||
s2.resize(size2);
|
||||
// We start from the back of the string
|
||||
if (size2 > 0) {
|
||||
uint32_t pos = size2 - 1;
|
||||
do {
|
||||
if (pos >= size1 || rnd.OneIn(4)) {
|
||||
// For 1/4 chance, use random byte
|
||||
s2[pos] = static_cast<char>(rnd.Uniform(256));
|
||||
} else if (rnd.OneIn(4)) {
|
||||
// In 1/4 chance, stop here.
|
||||
break;
|
||||
} else {
|
||||
// Create a char within [-2, +2] of the matching char of s1.
|
||||
int diff = static_cast<int>(rnd.Uniform(5)) - 2;
|
||||
// char may be signed or unsigned based on platform.
|
||||
int s1_char = static_cast<int>(static_cast<unsigned char>(s1[pos]));
|
||||
int s2_char = s1_char + diff;
|
||||
if (s2_char < 0) {
|
||||
s2_char = 0;
|
||||
}
|
||||
if (s2_char > 255) {
|
||||
s2_char = 255;
|
||||
}
|
||||
s2[pos] = static_cast<char>(s2_char);
|
||||
}
|
||||
} while (pos-- != 0);
|
||||
}
|
||||
|
||||
// Test separators
|
||||
for (int rev = 0; rev < 2; rev++) {
|
||||
if (rev == 1) {
|
||||
// switch s1 and s2
|
||||
std::string t = s1;
|
||||
s1 = s2;
|
||||
s2 = t;
|
||||
}
|
||||
std::string separator = s1;
|
||||
BytewiseComparator()->FindShortestSeparator(&separator, s2);
|
||||
std::string rev_separator = s1;
|
||||
ReverseBytewiseComparator()->FindShortestSeparator(&rev_separator, s2);
|
||||
|
||||
if (s1 == s2) {
|
||||
ASSERT_EQ(s1, separator);
|
||||
ASSERT_EQ(s2, rev_separator);
|
||||
} else if (s1 < s2) {
|
||||
ASSERT_TRUE(s1 <= separator);
|
||||
ASSERT_TRUE(s2 > separator);
|
||||
ASSERT_LE(separator.size(), std::max(s1.size(), s2.size()));
|
||||
ASSERT_EQ(s1, rev_separator);
|
||||
} else {
|
||||
ASSERT_TRUE(s1 >= rev_separator);
|
||||
ASSERT_TRUE(s2 < rev_separator);
|
||||
ASSERT_LE(rev_separator.size(), std::max(s1.size(), s2.size()));
|
||||
ASSERT_EQ(s1, separator);
|
||||
}
|
||||
}
|
||||
|
||||
// Test successors
|
||||
std::string succ = s1;
|
||||
BytewiseComparator()->FindShortSuccessor(&succ);
|
||||
ASSERT_TRUE(succ >= s1);
|
||||
|
||||
succ = s1;
|
||||
ReverseBytewiseComparator()->FindShortSuccessor(&succ);
|
||||
ASSERT_TRUE(succ <= s1);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
+6
-16
@@ -19,23 +19,15 @@ void CancelAllBackgroundWork(DB* db, bool wait) {
|
||||
}
|
||||
|
||||
Status DeleteFilesInRange(DB* db, ColumnFamilyHandle* column_family,
|
||||
const Slice* begin, const Slice* end,
|
||||
bool include_end) {
|
||||
RangePtr range(begin, end);
|
||||
return DeleteFilesInRanges(db, column_family, &range, 1, include_end);
|
||||
}
|
||||
|
||||
Status DeleteFilesInRanges(DB* db, ColumnFamilyHandle* column_family,
|
||||
const RangePtr* ranges, size_t n,
|
||||
bool include_end) {
|
||||
const Slice* begin, const Slice* end) {
|
||||
return (static_cast_with_check<DBImpl, DB>(db->GetRootDB()))
|
||||
->DeleteFilesInRanges(column_family, ranges, n, include_end);
|
||||
->DeleteFilesInRange(column_family, begin, end);
|
||||
}
|
||||
|
||||
Status VerifySstFileChecksum(const Options& options,
|
||||
const EnvOptions& env_options,
|
||||
const std::string& file_path) {
|
||||
std::unique_ptr<RandomAccessFile> file;
|
||||
unique_ptr<RandomAccessFile> file;
|
||||
uint64_t file_size;
|
||||
InternalKeyComparator internal_comparator(options.comparator);
|
||||
ImmutableCFOptions ioptions(options);
|
||||
@@ -46,14 +38,12 @@ Status VerifySstFileChecksum(const Options& options,
|
||||
} else {
|
||||
return s;
|
||||
}
|
||||
std::unique_ptr<TableReader> table_reader;
|
||||
unique_ptr<TableReader> table_reader;
|
||||
std::unique_ptr<RandomAccessFileReader> file_reader(
|
||||
new RandomAccessFileReader(std::move(file), file_path));
|
||||
const bool kImmortal = true;
|
||||
s = ioptions.table_factory->NewTableReader(
|
||||
TableReaderOptions(ioptions, options.prefix_extractor.get(), env_options,
|
||||
internal_comparator, false /* skip_filters */,
|
||||
!kImmortal, -1 /* level */),
|
||||
TableReaderOptions(ioptions, env_options, internal_comparator,
|
||||
false /* skip_filters */, -1 /* level */),
|
||||
std::move(file_reader), file_size, &table_reader,
|
||||
false /* prefetch_index_and_filter_in_cache */);
|
||||
if (!s.ok()) {
|
||||
|
||||
+9
-44
@@ -24,8 +24,6 @@
|
||||
#include "rocksdb/env.h"
|
||||
#include "rocksdb/table.h"
|
||||
#include "rocksdb/write_batch.h"
|
||||
#include "table/block_based_table_builder.h"
|
||||
#include "table/meta_blocks.h"
|
||||
#include "util/filename.h"
|
||||
#include "util/string_util.h"
|
||||
#include "util/testharness.h"
|
||||
@@ -39,7 +37,7 @@ class CorruptionTest : public testing::Test {
|
||||
public:
|
||||
test::ErrorEnv env_;
|
||||
std::string dbname_;
|
||||
std::shared_ptr<Cache> tiny_cache_;
|
||||
shared_ptr<Cache> tiny_cache_;
|
||||
Options options_;
|
||||
DB* db_;
|
||||
|
||||
@@ -50,7 +48,7 @@ class CorruptionTest : public testing::Test {
|
||||
tiny_cache_ = NewLRUCache(100, 4);
|
||||
options_.wal_recovery_mode = WALRecoveryMode::kTolerateCorruptedTailRecords;
|
||||
options_.env = &env_;
|
||||
dbname_ = test::PerThreadDBPath("corruption_test");
|
||||
dbname_ = test::TmpDir() + "/corruption_test";
|
||||
DestroyDB(dbname_, options_);
|
||||
|
||||
db_ = nullptr;
|
||||
@@ -62,9 +60,9 @@ class CorruptionTest : public testing::Test {
|
||||
options_.create_if_missing = false;
|
||||
}
|
||||
|
||||
~CorruptionTest() override {
|
||||
delete db_;
|
||||
DestroyDB(dbname_, Options());
|
||||
~CorruptionTest() {
|
||||
delete db_;
|
||||
DestroyDB(dbname_, Options());
|
||||
}
|
||||
|
||||
void CloseDb() {
|
||||
@@ -335,9 +333,9 @@ TEST_F(CorruptionTest, TableFileIndexData) {
|
||||
Corrupt(kTableFile, -2000, 500);
|
||||
Reopen();
|
||||
dbi = reinterpret_cast<DBImpl*>(db_);
|
||||
// one full file may be readable, since only one was corrupted
|
||||
// one full file should be readable, since only one was corrupted
|
||||
// the other file should be fully non-readable, since index was corrupted
|
||||
Check(0, 5000);
|
||||
Check(5000, 5000);
|
||||
ASSERT_NOK(dbi->VerifyChecksum());
|
||||
}
|
||||
|
||||
@@ -469,39 +467,6 @@ TEST_F(CorruptionTest, UnrelatedKeys) {
|
||||
ASSERT_EQ(Value(1000, &tmp2).ToString(), v);
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, RangeDeletionCorrupted) {
|
||||
ASSERT_OK(
|
||||
db_->DeleteRange(WriteOptions(), db_->DefaultColumnFamily(), "a", "b"));
|
||||
ASSERT_OK(db_->Flush(FlushOptions()));
|
||||
std::vector<LiveFileMetaData> metadata;
|
||||
db_->GetLiveFilesMetaData(&metadata);
|
||||
ASSERT_EQ(static_cast<size_t>(1), metadata.size());
|
||||
std::string filename = dbname_ + metadata[0].name;
|
||||
|
||||
std::unique_ptr<RandomAccessFile> file;
|
||||
ASSERT_OK(options_.env->NewRandomAccessFile(filename, &file, EnvOptions()));
|
||||
std::unique_ptr<RandomAccessFileReader> file_reader(
|
||||
new RandomAccessFileReader(std::move(file), filename));
|
||||
|
||||
uint64_t file_size;
|
||||
ASSERT_OK(options_.env->GetFileSize(filename, &file_size));
|
||||
|
||||
BlockHandle range_del_handle;
|
||||
ASSERT_OK(FindMetaBlock(
|
||||
file_reader.get(), file_size, kBlockBasedTableMagicNumber,
|
||||
ImmutableCFOptions(options_), kRangeDelBlock, &range_del_handle));
|
||||
|
||||
ASSERT_OK(TryReopen());
|
||||
CorruptFile(filename, static_cast<int>(range_del_handle.offset()), 1);
|
||||
// The test case does not fail on TryReopen because failure to preload table
|
||||
// handlers is not considered critical.
|
||||
ASSERT_OK(TryReopen());
|
||||
std::string val;
|
||||
// However, it does fail on any read involving that file since that file
|
||||
// cannot be opened with a corrupt range deletion meta-block.
|
||||
ASSERT_TRUE(db_->Get(ReadOptions(), "a", &val).IsCorruption());
|
||||
}
|
||||
|
||||
TEST_F(CorruptionTest, FileSystemStateCorrupted) {
|
||||
for (int iter = 0; iter < 2; ++iter) {
|
||||
Options options;
|
||||
@@ -520,7 +485,7 @@ TEST_F(CorruptionTest, FileSystemStateCorrupted) {
|
||||
db_ = nullptr;
|
||||
|
||||
if (iter == 0) { // corrupt file size
|
||||
std::unique_ptr<WritableFile> file;
|
||||
unique_ptr<WritableFile> file;
|
||||
env_.NewWritableFile(filename, &file, EnvOptions());
|
||||
file->Append(Slice("corrupted sst"));
|
||||
file.reset();
|
||||
@@ -545,7 +510,7 @@ int main(int argc, char** argv) {
|
||||
#else
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) {
|
||||
int main(int argc, char** argv) {
|
||||
fprintf(stderr, "SKIPPED as RepairDB() is not supported in ROCKSDB_LITE\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -25,13 +25,13 @@ class CuckooTableDBTest : public testing::Test {
|
||||
|
||||
public:
|
||||
CuckooTableDBTest() : env_(Env::Default()) {
|
||||
dbname_ = test::PerThreadDBPath("cuckoo_table_db_test");
|
||||
dbname_ = test::TmpDir() + "/cuckoo_table_db_test";
|
||||
EXPECT_OK(DestroyDB(dbname_, Options()));
|
||||
db_ = nullptr;
|
||||
Reopen();
|
||||
}
|
||||
|
||||
~CuckooTableDBTest() override {
|
||||
~CuckooTableDBTest() {
|
||||
delete db_;
|
||||
EXPECT_OK(DestroyDB(dbname_, Options()));
|
||||
}
|
||||
@@ -333,7 +333,7 @@ int main(int argc, char** argv) {
|
||||
#else
|
||||
#include <stdio.h>
|
||||
|
||||
int main(int /*argc*/, char** /*argv*/) {
|
||||
int main(int argc, char** argv) {
|
||||
fprintf(stderr, "SKIPPED as Cuckoo table is not supported in ROCKSDB_LITE\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
+32
-262
@@ -6,11 +6,9 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
// #include <iostream>
|
||||
#include "db/db_test_util.h"
|
||||
#include "port/stack_trace.h"
|
||||
#include "rocksdb/perf_context.h"
|
||||
#include "util/fault_injection_test_env.h"
|
||||
#if !defined(ROCKSDB_LITE)
|
||||
#include "util/sync_point.h"
|
||||
#endif
|
||||
@@ -215,11 +213,11 @@ TEST_F(DBBasicTest, PutSingleDeleteGet) {
|
||||
ASSERT_EQ("v2", Get(1, "foo2"));
|
||||
ASSERT_OK(SingleDelete(1, "foo"));
|
||||
ASSERT_EQ("NOT_FOUND", Get(1, "foo"));
|
||||
// Ski FIFO and universal compaction because they do not apply to the test
|
||||
// case. Skip MergePut because single delete does not get removed when it
|
||||
// encounters a merge.
|
||||
} while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction |
|
||||
kSkipMergePut));
|
||||
// Skip HashCuckooRep as it does not support single delete. FIFO and
|
||||
// universal compaction do not apply to the test case. Skip MergePut
|
||||
// because single delete does not get removed when it encounters a merge.
|
||||
} while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
|
||||
kSkipUniversalCompaction | kSkipMergePut));
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, EmptyFlush) {
|
||||
@@ -237,11 +235,11 @@ TEST_F(DBBasicTest, EmptyFlush) {
|
||||
ASSERT_OK(Flush(1));
|
||||
|
||||
ASSERT_EQ("[ ]", AllEntriesFor("a", 1));
|
||||
// Skip FIFO and universal compaction as they do not apply to the test
|
||||
// case. Skip MergePut because merges cannot be combined with single
|
||||
// deletions.
|
||||
} while (ChangeOptions(kSkipFIFOCompaction | kSkipUniversalCompaction |
|
||||
kSkipMergePut));
|
||||
// Skip HashCuckooRep as it does not support single delete. FIFO and
|
||||
// universal compaction do not apply to the test case. Skip MergePut
|
||||
// because merges cannot be combined with single deletions.
|
||||
} while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction |
|
||||
kSkipUniversalCompaction | kSkipMergePut));
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, GetFromVersions) {
|
||||
@@ -265,6 +263,11 @@ TEST_F(DBBasicTest, GetSnapshot) {
|
||||
std::string key = (i == 0) ? std::string("foo") : std::string(200, 'x');
|
||||
ASSERT_OK(Put(1, key, "v1"));
|
||||
const Snapshot* s1 = db_->GetSnapshot();
|
||||
if (option_config_ == kHashCuckoo) {
|
||||
// Unsupported case.
|
||||
ASSERT_TRUE(s1 == nullptr);
|
||||
break;
|
||||
}
|
||||
ASSERT_OK(Put(1, key, "v2"));
|
||||
ASSERT_EQ("v2", Get(1, key));
|
||||
ASSERT_EQ("v1", Get(1, key, s1));
|
||||
@@ -505,7 +508,7 @@ TEST_F(DBBasicTest, Snapshot) {
|
||||
ASSERT_EQ(0U, GetNumSnapshots());
|
||||
ASSERT_EQ("0v4", Get(0, "foo"));
|
||||
ASSERT_EQ("1v4", Get(1, "foo"));
|
||||
} while (ChangeOptions());
|
||||
} while (ChangeOptions(kSkipHashCuckoo));
|
||||
}
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
@@ -561,7 +564,8 @@ TEST_F(DBBasicTest, CompactBetweenSnapshots) {
|
||||
nullptr);
|
||||
ASSERT_EQ("sixth", Get(1, "foo"));
|
||||
ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]");
|
||||
} while (ChangeOptions(kSkipFIFOCompaction));
|
||||
// skip HashCuckooRep as it does not support snapshot
|
||||
} while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction));
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, DBOpen_Options) {
|
||||
@@ -789,7 +793,7 @@ TEST_F(DBBasicTest, ChecksumTest) {
|
||||
BlockBasedTableOptions table_options;
|
||||
Options options = CurrentOptions();
|
||||
// change when new checksum type added
|
||||
int max_checksum = static_cast<int>(kxxHash64);
|
||||
int max_checksum = static_cast<int>(kxxHash);
|
||||
const int kNumPerFile = 2;
|
||||
|
||||
// generate one table with each type of checksum
|
||||
@@ -804,7 +808,7 @@ TEST_F(DBBasicTest, ChecksumTest) {
|
||||
}
|
||||
|
||||
// verify data with each type of checksum
|
||||
for (int i = 0; i <= kxxHash64; ++i) {
|
||||
for (int i = 0; i <= kxxHash; ++i) {
|
||||
table_options.checksum = static_cast<ChecksumType>(i);
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
Reopen(options);
|
||||
@@ -845,74 +849,44 @@ TEST_F(DBBasicTest, MmapAndBufferOptions) {
|
||||
|
||||
class TestEnv : public EnvWrapper {
|
||||
public:
|
||||
explicit TestEnv() : EnvWrapper(Env::Default()),
|
||||
close_count(0) { }
|
||||
explicit TestEnv(Env* base) : EnvWrapper(base) { };
|
||||
|
||||
class TestLogger : public Logger {
|
||||
public:
|
||||
using Logger::Logv;
|
||||
TestLogger(TestEnv *env_ptr) : Logger() { env = env_ptr; }
|
||||
~TestLogger() override {
|
||||
if (!closed_) {
|
||||
CloseHelper();
|
||||
}
|
||||
virtual void Logv(const char *format, va_list ap) override { };
|
||||
private:
|
||||
virtual Status CloseImpl() override {
|
||||
return Status::NotSupported();
|
||||
}
|
||||
void Logv(const char* /*format*/, va_list /*ap*/) override{};
|
||||
|
||||
protected:
|
||||
Status CloseImpl() override { return CloseHelper(); }
|
||||
|
||||
private:
|
||||
Status CloseHelper() {
|
||||
env->CloseCountInc();;
|
||||
return Status::IOError();
|
||||
}
|
||||
TestEnv *env;
|
||||
};
|
||||
|
||||
void CloseCountInc() { close_count++; }
|
||||
|
||||
int GetCloseCount() { return close_count; }
|
||||
|
||||
Status NewLogger(const std::string& /*fname*/,
|
||||
std::shared_ptr<Logger>* result) override {
|
||||
result->reset(new TestLogger(this));
|
||||
virtual Status NewLogger(const std::string& fname,
|
||||
shared_ptr<Logger>* result) {
|
||||
result->reset(new TestLogger());
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
private:
|
||||
int close_count;
|
||||
};
|
||||
|
||||
TEST_F(DBBasicTest, DBClose) {
|
||||
Options options = GetDefaultOptions();
|
||||
std::string dbname = test::PerThreadDBPath("db_close_test");
|
||||
std::string dbname = test::TmpDir(env_) + "/db_close_test";
|
||||
ASSERT_OK(DestroyDB(dbname, options));
|
||||
|
||||
DB* db = nullptr;
|
||||
TestEnv* env = new TestEnv();
|
||||
options.create_if_missing = true;
|
||||
options.env = env;
|
||||
options.env = new TestEnv(Env::Default());
|
||||
Status s = DB::Open(options, dbname, &db);
|
||||
ASSERT_OK(s);
|
||||
ASSERT_TRUE(db != nullptr);
|
||||
|
||||
s = db->Close();
|
||||
ASSERT_EQ(env->GetCloseCount(), 1);
|
||||
ASSERT_EQ(s, Status::IOError());
|
||||
ASSERT_EQ(s, Status::NotSupported());
|
||||
|
||||
delete db;
|
||||
ASSERT_EQ(env->GetCloseCount(), 1);
|
||||
|
||||
// Do not call DB::Close() and ensure our logger Close() still gets called
|
||||
s = DB::Open(options, dbname, &db);
|
||||
ASSERT_OK(s);
|
||||
ASSERT_TRUE(db != nullptr);
|
||||
delete db;
|
||||
ASSERT_EQ(env->GetCloseCount(), 2);
|
||||
|
||||
// Provide our own logger and ensure DB::Close() does not close it
|
||||
options.info_log.reset(new TestEnv::TestLogger(env));
|
||||
options.info_log.reset(new TestEnv::TestLogger());
|
||||
options.create_if_missing = false;
|
||||
s = DB::Open(options, dbname, &db);
|
||||
ASSERT_OK(s);
|
||||
@@ -921,213 +895,9 @@ TEST_F(DBBasicTest, DBClose) {
|
||||
s = db->Close();
|
||||
ASSERT_EQ(s, Status::OK());
|
||||
delete db;
|
||||
ASSERT_EQ(env->GetCloseCount(), 2);
|
||||
options.info_log.reset();
|
||||
ASSERT_EQ(env->GetCloseCount(), 3);
|
||||
|
||||
delete options.env;
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, DBCloseFlushError) {
|
||||
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
|
||||
new FaultInjectionTestEnv(Env::Default()));
|
||||
Options options = GetDefaultOptions();
|
||||
options.create_if_missing = true;
|
||||
options.manual_wal_flush = true;
|
||||
options.write_buffer_size=100;
|
||||
options.env = fault_injection_env.get();
|
||||
|
||||
Reopen(options);
|
||||
ASSERT_OK(Put("key1", "value1"));
|
||||
ASSERT_OK(Put("key2", "value2"));
|
||||
ASSERT_OK(dbfull()->TEST_SwitchMemtable());
|
||||
ASSERT_OK(Put("key3", "value3"));
|
||||
fault_injection_env->SetFilesystemActive(false);
|
||||
Status s = dbfull()->Close();
|
||||
fault_injection_env->SetFilesystemActive(true);
|
||||
ASSERT_NE(s, Status::OK());
|
||||
|
||||
Destroy(options);
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, MultiGetMultiCF) {
|
||||
Options options = CurrentOptions();
|
||||
CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
|
||||
"alyosha", "popovich"},
|
||||
options);
|
||||
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
|
||||
"cf" + std::to_string(i) + "_val"));
|
||||
}
|
||||
|
||||
int get_sv_count = 0;
|
||||
rocksdb::DBImpl* db = reinterpret_cast<DBImpl*>(db_);
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) {
|
||||
if (++get_sv_count == 2) {
|
||||
// After MultiGet refs a couple of CFs, flush all CFs so MultiGet
|
||||
// is forced to repeat the process
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
ASSERT_OK(Flush(i));
|
||||
ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
|
||||
"cf" + std::to_string(i) + "_val2"));
|
||||
}
|
||||
}
|
||||
if (get_sv_count == 11) {
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
|
||||
db->GetColumnFamilyHandle(i))
|
||||
->cfd();
|
||||
ASSERT_EQ(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse);
|
||||
}
|
||||
}
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
std::vector<int> cfs;
|
||||
std::vector<std::string> keys;
|
||||
std::vector<std::string> values;
|
||||
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
cfs.push_back(i);
|
||||
keys.push_back("cf" + std::to_string(i) + "_key");
|
||||
}
|
||||
|
||||
values = MultiGet(cfs, keys);
|
||||
ASSERT_EQ(values.size(), 8);
|
||||
for (unsigned int j = 0; j < values.size(); ++j) {
|
||||
ASSERT_EQ(values[j], "cf" + std::to_string(j) + "_val2");
|
||||
}
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
|
||||
reinterpret_cast<DBImpl*>(db_)->GetColumnFamilyHandle(i))
|
||||
->cfd();
|
||||
ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse);
|
||||
ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVObsolete);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, MultiGetMultiCFMutex) {
|
||||
Options options = CurrentOptions();
|
||||
CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
|
||||
"alyosha", "popovich"},
|
||||
options);
|
||||
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
|
||||
"cf" + std::to_string(i) + "_val"));
|
||||
}
|
||||
|
||||
int get_sv_count = 0;
|
||||
int retries = 0;
|
||||
bool last_try = false;
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::MultiGet::LastTry", [&](void* /*arg*/) {
|
||||
last_try = true;
|
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) {
|
||||
if (last_try) {
|
||||
return;
|
||||
}
|
||||
if (++get_sv_count == 2) {
|
||||
++retries;
|
||||
get_sv_count = 0;
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
ASSERT_OK(Flush(i));
|
||||
ASSERT_OK(Put(
|
||||
i, "cf" + std::to_string(i) + "_key",
|
||||
"cf" + std::to_string(i) + "_val" + std::to_string(retries)));
|
||||
}
|
||||
}
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
std::vector<int> cfs;
|
||||
std::vector<std::string> keys;
|
||||
std::vector<std::string> values;
|
||||
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
cfs.push_back(i);
|
||||
keys.push_back("cf" + std::to_string(i) + "_key");
|
||||
}
|
||||
|
||||
values = MultiGet(cfs, keys);
|
||||
ASSERT_TRUE(last_try);
|
||||
ASSERT_EQ(values.size(), 8);
|
||||
for (unsigned int j = 0; j < values.size(); ++j) {
|
||||
ASSERT_EQ(values[j],
|
||||
"cf" + std::to_string(j) + "_val" + std::to_string(retries));
|
||||
}
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
|
||||
reinterpret_cast<DBImpl*>(db_)->GetColumnFamilyHandle(i))
|
||||
->cfd();
|
||||
ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse);
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBBasicTest, MultiGetMultiCFSnapshot) {
|
||||
Options options = CurrentOptions();
|
||||
CreateAndReopenWithCF({"pikachu", "ilya", "muromec", "dobrynia", "nikitich",
|
||||
"alyosha", "popovich"},
|
||||
options);
|
||||
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
|
||||
"cf" + std::to_string(i) + "_val"));
|
||||
}
|
||||
|
||||
int get_sv_count = 0;
|
||||
rocksdb::DBImpl* db = reinterpret_cast<DBImpl*>(db_);
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::MultiGet::AfterRefSV", [&](void* /*arg*/) {
|
||||
if (++get_sv_count == 2) {
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
ASSERT_OK(Flush(i));
|
||||
ASSERT_OK(Put(i, "cf" + std::to_string(i) + "_key",
|
||||
"cf" + std::to_string(i) + "_val2"));
|
||||
}
|
||||
}
|
||||
if (get_sv_count == 8) {
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
|
||||
db->GetColumnFamilyHandle(i))
|
||||
->cfd();
|
||||
ASSERT_TRUE(
|
||||
(cfd->TEST_GetLocalSV()->Get() == SuperVersion::kSVInUse) ||
|
||||
(cfd->TEST_GetLocalSV()->Get() == SuperVersion::kSVObsolete));
|
||||
}
|
||||
}
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
std::vector<int> cfs;
|
||||
std::vector<std::string> keys;
|
||||
std::vector<std::string> values;
|
||||
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
cfs.push_back(i);
|
||||
keys.push_back("cf" + std::to_string(i) + "_key");
|
||||
}
|
||||
|
||||
const Snapshot* snapshot = db_->GetSnapshot();
|
||||
values = MultiGet(cfs, keys, snapshot);
|
||||
db_->ReleaseSnapshot(snapshot);
|
||||
ASSERT_EQ(values.size(), 8);
|
||||
for (unsigned int j = 0; j < values.size(); ++j) {
|
||||
ASSERT_EQ(values[j], "cf" + std::to_string(j) + "_val");
|
||||
}
|
||||
for (int i = 0; i < 8; ++i) {
|
||||
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(
|
||||
reinterpret_cast<DBImpl*>(db_)->GetColumnFamilyHandle(i))
|
||||
->cfd();
|
||||
ASSERT_NE(cfd->TEST_GetLocalSV()->Get(), SuperVersion::kSVInUse);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
+7
-138
@@ -47,7 +47,7 @@ class DBBlockCacheTest : public DBTestBase {
|
||||
return options;
|
||||
}
|
||||
|
||||
void InitTable(const Options& /*options*/) {
|
||||
void InitTable(const Options& options) {
|
||||
std::string value(kValueSize, 'a');
|
||||
for (size_t i = 0; i < kNumBlocks; i++) {
|
||||
ASSERT_OK(Put(ToString(i), value.c_str()));
|
||||
@@ -111,31 +111,6 @@ class DBBlockCacheTest : public DBTestBase {
|
||||
}
|
||||
};
|
||||
|
||||
TEST_F(DBBlockCacheTest, IteratorBlockCacheUsage) {
|
||||
ReadOptions read_options;
|
||||
read_options.fill_cache = false;
|
||||
auto table_options = GetTableOptions();
|
||||
auto options = GetOptions(table_options);
|
||||
InitTable(options);
|
||||
|
||||
std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
|
||||
table_options.block_cache = cache;
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
Reopen(options);
|
||||
RecordCacheCounters(options);
|
||||
|
||||
std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
|
||||
Iterator* iter = nullptr;
|
||||
|
||||
ASSERT_EQ(0, cache->GetUsage());
|
||||
iter = db_->NewIterator(read_options);
|
||||
iter->Seek(ToString(0));
|
||||
ASSERT_LT(0, cache->GetUsage());
|
||||
delete iter;
|
||||
iter = nullptr;
|
||||
ASSERT_EQ(0, cache->GetUsage());
|
||||
}
|
||||
|
||||
TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
|
||||
ReadOptions read_options;
|
||||
auto table_options = GetTableOptions();
|
||||
@@ -173,7 +148,7 @@ TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
|
||||
delete iter;
|
||||
iter = nullptr;
|
||||
|
||||
// Release iterators and access cache again.
|
||||
// Release interators and access cache again.
|
||||
for (size_t i = 0; i < kNumBlocks - 1; i++) {
|
||||
iterators[i].reset();
|
||||
CheckCacheCounters(options, 0, 0, 0, 0);
|
||||
@@ -305,41 +280,6 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksOfNewTableAddedToCache) {
|
||||
TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
|
||||
}
|
||||
|
||||
// With fill_cache = false, fills up the cache, then iterates over the entire
|
||||
// db, verify dummy entries inserted in `BlockBasedTable::NewDataBlockIterator`
|
||||
// does not cause heap-use-after-free errors in COMPILE_WITH_ASAN=1 runs
|
||||
TEST_F(DBBlockCacheTest, FillCacheAndIterateDB) {
|
||||
ReadOptions read_options;
|
||||
read_options.fill_cache = false;
|
||||
auto table_options = GetTableOptions();
|
||||
auto options = GetOptions(table_options);
|
||||
InitTable(options);
|
||||
|
||||
std::shared_ptr<Cache> cache = NewLRUCache(10, 0, true);
|
||||
table_options.block_cache = cache;
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
Reopen(options);
|
||||
ASSERT_OK(Put("key1", "val1"));
|
||||
ASSERT_OK(Put("key2", "val2"));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_OK(Put("key3", "val3"));
|
||||
ASSERT_OK(Put("key4", "val4"));
|
||||
ASSERT_OK(Flush());
|
||||
ASSERT_OK(Put("key5", "val5"));
|
||||
ASSERT_OK(Put("key6", "val6"));
|
||||
ASSERT_OK(Flush());
|
||||
|
||||
Iterator* iter = nullptr;
|
||||
|
||||
iter = db_->NewIterator(read_options);
|
||||
iter->Seek(ToString(0));
|
||||
while (iter->Valid()) {
|
||||
iter->Next();
|
||||
}
|
||||
delete iter;
|
||||
iter = nullptr;
|
||||
}
|
||||
|
||||
TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) {
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
@@ -349,7 +289,7 @@ TEST_F(DBBlockCacheTest, IndexAndFilterBlocksStats) {
|
||||
// 200 bytes are enough to hold the first two blocks
|
||||
std::shared_ptr<Cache> cache = NewLRUCache(200, 0, false);
|
||||
table_options.block_cache = cache;
|
||||
table_options.filter_policy.reset(NewBloomFilterPolicy(20, true));
|
||||
table_options.filter_policy.reset(NewBloomFilterPolicy(20));
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
@@ -390,14 +330,11 @@ class MockCache : public LRUCache {
|
||||
static uint32_t high_pri_insert_count;
|
||||
static uint32_t low_pri_insert_count;
|
||||
|
||||
MockCache()
|
||||
: LRUCache((size_t)1 << 25 /*capacity*/, 0 /*num_shard_bits*/,
|
||||
false /*strict_capacity_limit*/, 0.0 /*high_pri_pool_ratio*/) {
|
||||
}
|
||||
MockCache() : LRUCache(1 << 25, 0, false, 0.0) {}
|
||||
|
||||
Status Insert(const Slice& key, void* value, size_t charge,
|
||||
void (*deleter)(const Slice& key, void* value), Handle** handle,
|
||||
Priority priority) override {
|
||||
virtual Status Insert(const Slice& key, void* value, size_t charge,
|
||||
void (*deleter)(const Slice& key, void* value),
|
||||
Handle** handle, Priority priority) override {
|
||||
if (priority == Priority::LOW) {
|
||||
low_pri_insert_count++;
|
||||
} else {
|
||||
@@ -631,74 +568,6 @@ TEST_F(DBBlockCacheTest, CompressedCache) {
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBBlockCacheTest, CacheCompressionDict) {
|
||||
const int kNumFiles = 4;
|
||||
const int kNumEntriesPerFile = 128;
|
||||
const int kNumBytesPerEntry = 1024;
|
||||
|
||||
// Try all the available libraries that support dictionary compression
|
||||
std::vector<CompressionType> compression_types;
|
||||
#ifdef ZLIB
|
||||
compression_types.push_back(kZlibCompression);
|
||||
#endif // ZLIB
|
||||
#if LZ4_VERSION_NUMBER >= 10400
|
||||
compression_types.push_back(kLZ4Compression);
|
||||
compression_types.push_back(kLZ4HCCompression);
|
||||
#endif // LZ4_VERSION_NUMBER >= 10400
|
||||
#if ZSTD_VERSION_NUMBER >= 500
|
||||
compression_types.push_back(kZSTD);
|
||||
#endif // ZSTD_VERSION_NUMBER >= 500
|
||||
Random rnd(301);
|
||||
for (auto compression_type : compression_types) {
|
||||
Options options = CurrentOptions();
|
||||
options.compression = compression_type;
|
||||
options.compression_opts.max_dict_bytes = 4096;
|
||||
options.create_if_missing = true;
|
||||
options.num_levels = 2;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
options.target_file_size_base = kNumEntriesPerFile * kNumBytesPerEntry;
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.block_cache.reset(new MockCache());
|
||||
options.table_factory.reset(new BlockBasedTableFactory(table_options));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
for (int i = 0; i < kNumFiles; ++i) {
|
||||
ASSERT_EQ(i, NumTableFilesAtLevel(0, 0));
|
||||
for (int j = 0; j < kNumEntriesPerFile; ++j) {
|
||||
std::string value = RandomString(&rnd, kNumBytesPerEntry);
|
||||
ASSERT_OK(Put(Key(j * kNumFiles + i), value.c_str()));
|
||||
}
|
||||
ASSERT_OK(Flush());
|
||||
}
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
ASSERT_EQ(0, NumTableFilesAtLevel(0));
|
||||
ASSERT_EQ(kNumFiles, NumTableFilesAtLevel(1));
|
||||
|
||||
// Seek to a key in a file. It should cause the SST's dictionary meta-block
|
||||
// to be read.
|
||||
RecordCacheCounters(options);
|
||||
ASSERT_EQ(0,
|
||||
TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS));
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_ADD));
|
||||
ASSERT_EQ(
|
||||
TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
|
||||
0);
|
||||
ReadOptions read_options;
|
||||
ASSERT_NE("NOT_FOUND", Get(Key(kNumFiles * kNumEntriesPerFile - 1)));
|
||||
// Two blocks missed/added: dictionary and data block
|
||||
// One block hit: index since it's prefetched
|
||||
CheckCacheCounters(options, 2 /* expected_misses */, 1 /* expected_hits */,
|
||||
2 /* expected_inserts */, 0 /* expected_failures */);
|
||||
ASSERT_EQ(1,
|
||||
TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_MISS));
|
||||
ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_ADD));
|
||||
ASSERT_GT(
|
||||
TestGetTickerCount(options, BLOCK_CACHE_COMPRESSION_DICT_BYTES_INSERT),
|
||||
0);
|
||||
}
|
||||
}
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
+12
-612
@@ -22,51 +22,27 @@ class DBBloomFilterTest : public DBTestBase {
|
||||
|
||||
class DBBloomFilterTestWithParam
|
||||
: public DBTestBase,
|
||||
public testing::WithParamInterface<std::tuple<bool, bool, uint32_t>> {
|
||||
public testing::WithParamInterface<std::tuple<bool, bool>> {
|
||||
// public testing::WithParamInterface<bool> {
|
||||
protected:
|
||||
bool use_block_based_filter_;
|
||||
bool partition_filters_;
|
||||
uint32_t format_version_;
|
||||
|
||||
public:
|
||||
DBBloomFilterTestWithParam() : DBTestBase("/db_bloom_filter_tests") {}
|
||||
|
||||
~DBBloomFilterTestWithParam() override {}
|
||||
~DBBloomFilterTestWithParam() {}
|
||||
|
||||
void SetUp() override {
|
||||
use_block_based_filter_ = std::get<0>(GetParam());
|
||||
partition_filters_ = std::get<1>(GetParam());
|
||||
format_version_ = std::get<2>(GetParam());
|
||||
}
|
||||
};
|
||||
|
||||
class DBBloomFilterTestDefFormatVersion : public DBBloomFilterTestWithParam {};
|
||||
|
||||
class SliceTransformLimitedDomainGeneric : public SliceTransform {
|
||||
const char* Name() const override {
|
||||
return "SliceTransformLimitedDomainGeneric";
|
||||
}
|
||||
|
||||
Slice Transform(const Slice& src) const override {
|
||||
return Slice(src.data(), 5);
|
||||
}
|
||||
|
||||
bool InDomain(const Slice& src) const override {
|
||||
// prefix will be x????
|
||||
return src.size() >= 5;
|
||||
}
|
||||
|
||||
bool InRange(const Slice& dst) const override {
|
||||
// prefix will be x????
|
||||
return dst.size() == 5;
|
||||
}
|
||||
};
|
||||
|
||||
// KeyMayExist can lead to a few false positives, but not false negatives.
|
||||
// To make test deterministic, use a much larger number of bits per key-20 than
|
||||
// bits in the key, so that false positives are eliminated
|
||||
TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
|
||||
TEST_P(DBBloomFilterTestWithParam, KeyMayExist) {
|
||||
do {
|
||||
ReadOptions ropts;
|
||||
std::string value;
|
||||
@@ -141,79 +117,11 @@ TEST_P(DBBloomFilterTestDefFormatVersion, KeyMayExist) {
|
||||
ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction));
|
||||
}
|
||||
|
||||
TEST_F(DBBloomFilterTest, GetFilterByPrefixBloomCustomPrefixExtractor) {
|
||||
for (bool partition_filters : {true, false}) {
|
||||
Options options = last_options_;
|
||||
options.prefix_extractor =
|
||||
std::make_shared<SliceTransformLimitedDomainGeneric>();
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
get_perf_context()->EnablePerLevelPerfContext();
|
||||
BlockBasedTableOptions bbto;
|
||||
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
||||
if (partition_filters) {
|
||||
bbto.partition_filters = true;
|
||||
bbto.index_type = BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
||||
}
|
||||
bbto.whole_key_filtering = false;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
WriteOptions wo;
|
||||
ReadOptions ro;
|
||||
FlushOptions fo;
|
||||
fo.wait = true;
|
||||
std::string value;
|
||||
|
||||
ASSERT_OK(dbfull()->Put(wo, "barbarbar", "foo"));
|
||||
ASSERT_OK(dbfull()->Put(wo, "barbarbar2", "foo2"));
|
||||
ASSERT_OK(dbfull()->Put(wo, "foofoofoo", "bar"));
|
||||
|
||||
dbfull()->Flush(fo);
|
||||
|
||||
ASSERT_EQ("foo", Get("barbarbar"));
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
|
||||
ASSERT_EQ(
|
||||
0,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
ASSERT_EQ("foo2", Get("barbarbar2"));
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
|
||||
ASSERT_EQ(
|
||||
0,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
ASSERT_EQ("NOT_FOUND", Get("barbarbar3"));
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
|
||||
ASSERT_EQ(
|
||||
0,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
|
||||
ASSERT_EQ("NOT_FOUND", Get("barfoofoo"));
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 1);
|
||||
ASSERT_EQ(
|
||||
1,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
|
||||
ASSERT_EQ("NOT_FOUND", Get("foobarbar"));
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
|
||||
ASSERT_EQ(
|
||||
2,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
|
||||
ro.total_order_seek = true;
|
||||
ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
|
||||
ASSERT_EQ(
|
||||
2,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
get_perf_context()->Reset();
|
||||
}
|
||||
}
|
||||
|
||||
TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
|
||||
for (bool partition_filters : {true, false}) {
|
||||
Options options = last_options_;
|
||||
options.prefix_extractor.reset(NewFixedPrefixTransform(8));
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
get_perf_context()->EnablePerLevelPerfContext();
|
||||
BlockBasedTableOptions bbto;
|
||||
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
||||
if (partition_filters) {
|
||||
@@ -252,10 +160,6 @@ TEST_F(DBBloomFilterTest, GetFilterByPrefixBloom) {
|
||||
ro.total_order_seek = true;
|
||||
ASSERT_TRUE(db_->Get(ro, "foobarbar", &value).IsNotFound());
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 2);
|
||||
ASSERT_EQ(
|
||||
2,
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful);
|
||||
get_perf_context()->Reset();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,7 +168,6 @@ TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
|
||||
Options options = last_options_;
|
||||
options.prefix_extractor.reset(NewFixedPrefixTransform(3));
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
get_perf_context()->EnablePerLevelPerfContext();
|
||||
|
||||
BlockBasedTableOptions bbto;
|
||||
bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
|
||||
@@ -412,14 +315,6 @@ TEST_F(DBBloomFilterTest, WholeKeyFilterProp) {
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
|
||||
ASSERT_EQ("bar", Get("barfoo"));
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 12);
|
||||
uint64_t bloom_filter_useful_all_levels = 0;
|
||||
for (auto& kv : (*(get_perf_context()->level_to_perf_context))) {
|
||||
if (kv.second.bloom_filter_useful > 0) {
|
||||
bloom_filter_useful_all_levels += kv.second.bloom_filter_useful;
|
||||
}
|
||||
}
|
||||
ASSERT_EQ(12, bloom_filter_useful_all_levels);
|
||||
get_perf_context()->Reset();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -439,11 +334,6 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
|
||||
table_options.index_type =
|
||||
BlockBasedTableOptions::IndexType::kTwoLevelIndexSearch;
|
||||
}
|
||||
table_options.format_version = format_version_;
|
||||
if (format_version_ >= 4) {
|
||||
// value delta encoding challenged more with index interval > 1
|
||||
table_options.index_block_restart_interval = 8;
|
||||
}
|
||||
table_options.metadata_block_size = 32;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
|
||||
@@ -499,34 +389,15 @@ TEST_P(DBBloomFilterTestWithParam, BloomFilter) {
|
||||
} while (ChangeCompactOptions());
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_VALGRIND_RUN
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
FormatDef, DBBloomFilterTestDefFormatVersion,
|
||||
::testing::Values(std::make_tuple(true, false, test::kDefaultFormatVersion),
|
||||
std::make_tuple(false, true, test::kDefaultFormatVersion),
|
||||
std::make_tuple(false, false,
|
||||
test::kDefaultFormatVersion)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
FormatDef, DBBloomFilterTestWithParam,
|
||||
::testing::Values(std::make_tuple(true, false, test::kDefaultFormatVersion),
|
||||
std::make_tuple(false, true, test::kDefaultFormatVersion),
|
||||
std::make_tuple(false, false,
|
||||
test::kDefaultFormatVersion)));
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
FormatLatest, DBBloomFilterTestWithParam,
|
||||
::testing::Values(std::make_tuple(true, false, test::kLatestFormatVersion),
|
||||
std::make_tuple(false, true, test::kLatestFormatVersion),
|
||||
std::make_tuple(false, false,
|
||||
test::kLatestFormatVersion)));
|
||||
#endif // ROCKSDB_VALGRIND_RUN
|
||||
INSTANTIATE_TEST_CASE_P(DBBloomFilterTestWithParam, DBBloomFilterTestWithParam,
|
||||
::testing::Values(std::make_tuple(true, false),
|
||||
std::make_tuple(false, true),
|
||||
std::make_tuple(false, false)));
|
||||
|
||||
TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
||||
while (ChangeFilterOptions()) {
|
||||
Options options = CurrentOptions();
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
get_perf_context()->EnablePerLevelPerfContext();
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
|
||||
const int maxKey = 10000;
|
||||
@@ -548,10 +419,6 @@ TEST_F(DBBloomFilterTest, BloomFilterRate) {
|
||||
ASSERT_EQ("NOT_FOUND", Get(1, Key(i + 33333)));
|
||||
}
|
||||
ASSERT_GE(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), maxKey * 0.98);
|
||||
ASSERT_GE(
|
||||
(*(get_perf_context()->level_to_perf_context))[0].bloom_filter_useful,
|
||||
maxKey * 0.98);
|
||||
get_perf_context()->Reset();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -642,7 +509,7 @@ class WrappedBloom : public FilterPolicy {
|
||||
explicit WrappedBloom(int bits_per_key)
|
||||
: filter_(NewBloomFilterPolicy(bits_per_key)), counter_(0) {}
|
||||
|
||||
~WrappedBloom() override { delete filter_; }
|
||||
~WrappedBloom() { delete filter_; }
|
||||
|
||||
const char* Name() const override { return "WrappedRocksDbFilterPolicy"; }
|
||||
|
||||
@@ -786,56 +653,6 @@ TEST_F(DBBloomFilterTest, PrefixExtractorBlockFilter) {
|
||||
delete iter;
|
||||
}
|
||||
|
||||
TEST_F(DBBloomFilterTest, MemtableWholeKeyBloomFilter) {
|
||||
// regression test for #2743. the range delete tombstones in memtable should
|
||||
// be added even when Get() skips searching due to its prefix bloom filter
|
||||
const int kMemtableSize = 1 << 20; // 1MB
|
||||
const int kMemtablePrefixFilterSize = 1 << 13; // 8KB
|
||||
const int kPrefixLen = 4;
|
||||
Options options = CurrentOptions();
|
||||
options.memtable_prefix_bloom_size_ratio =
|
||||
static_cast<double>(kMemtablePrefixFilterSize) / kMemtableSize;
|
||||
options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(kPrefixLen));
|
||||
options.write_buffer_size = kMemtableSize;
|
||||
options.memtable_whole_key_filtering = false;
|
||||
Reopen(options);
|
||||
std::string key1("AAAABBBB");
|
||||
std::string key2("AAAACCCC"); // not in DB
|
||||
std::string key3("AAAADDDD");
|
||||
std::string key4("AAAAEEEE");
|
||||
std::string value1("Value1");
|
||||
std::string value3("Value3");
|
||||
std::string value4("Value4");
|
||||
|
||||
ASSERT_OK(Put(key1, value1, WriteOptions()));
|
||||
|
||||
// check memtable bloom stats
|
||||
ASSERT_EQ("NOT_FOUND", Get(key2));
|
||||
ASSERT_EQ(0, get_perf_context()->bloom_memtable_miss_count);
|
||||
// same prefix, bloom filter false positive
|
||||
ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
|
||||
|
||||
// enable whole key bloom filter
|
||||
options.memtable_whole_key_filtering = true;
|
||||
Reopen(options);
|
||||
// check memtable bloom stats
|
||||
ASSERT_OK(Put(key3, value3, WriteOptions()));
|
||||
ASSERT_EQ("NOT_FOUND", Get(key2));
|
||||
// whole key bloom filter kicks in and determines it's a miss
|
||||
ASSERT_EQ(1, get_perf_context()->bloom_memtable_miss_count);
|
||||
ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
|
||||
|
||||
// verify whole key filtering does not depend on prefix_extractor
|
||||
options.prefix_extractor.reset();
|
||||
Reopen(options);
|
||||
// check memtable bloom stats
|
||||
ASSERT_OK(Put(key4, value4, WriteOptions()));
|
||||
ASSERT_EQ("NOT_FOUND", Get(key2));
|
||||
// whole key bloom filter kicks in and determines it's a miss
|
||||
ASSERT_EQ(2, get_perf_context()->bloom_memtable_miss_count);
|
||||
ASSERT_EQ(1, get_perf_context()->bloom_memtable_hit_count);
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
class BloomStatsTestWithParam
|
||||
: public DBBloomFilterTest,
|
||||
@@ -873,7 +690,7 @@ class BloomStatsTestWithParam
|
||||
DestroyAndReopen(options_);
|
||||
}
|
||||
|
||||
~BloomStatsTestWithParam() override {
|
||||
~BloomStatsTestWithParam() {
|
||||
get_perf_context()->Reset();
|
||||
Destroy(options_);
|
||||
}
|
||||
@@ -947,7 +764,7 @@ TEST_P(BloomStatsTestWithParam, BloomStatsTestWithIter) {
|
||||
ASSERT_OK(Put(key1, value1, WriteOptions()));
|
||||
ASSERT_OK(Put(key3, value3, WriteOptions()));
|
||||
|
||||
std::unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
|
||||
unique_ptr<Iterator> iter(dbfull()->NewIterator(ReadOptions()));
|
||||
|
||||
// check memtable bloom stats
|
||||
iter->Seek(key1);
|
||||
@@ -1123,7 +940,6 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(bbto));
|
||||
options.optimize_filters_for_hits = true;
|
||||
options.statistics = rocksdb::CreateDBStatistics();
|
||||
get_perf_context()->EnablePerLevelPerfContext();
|
||||
CreateAndReopenWithCF({"mypikachu"}, options);
|
||||
|
||||
int numkeys = 200000;
|
||||
@@ -1170,14 +986,6 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
|
||||
// no bloom filter. Most keys be checked bloom filters twice.
|
||||
ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 65000 * 2);
|
||||
ASSERT_LT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 120000 * 2);
|
||||
uint64_t bloom_filter_useful_all_levels = 0;
|
||||
for (auto& kv : (*(get_perf_context()->level_to_perf_context))) {
|
||||
if (kv.second.bloom_filter_useful > 0) {
|
||||
bloom_filter_useful_all_levels += kv.second.bloom_filter_useful;
|
||||
}
|
||||
}
|
||||
ASSERT_GT(bloom_filter_useful_all_levels, 65000 * 2);
|
||||
ASSERT_LT(bloom_filter_useful_all_levels, 120000 * 2);
|
||||
|
||||
for (int i = 0; i < numkeys; i += 2) {
|
||||
ASSERT_EQ(Get(1, Key(i)), "val");
|
||||
@@ -1249,10 +1057,10 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
|
||||
int32_t non_trivial_move = 0;
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::BackgroundCompaction:TrivialMove",
|
||||
[&](void* /*arg*/) { trivial_move++; });
|
||||
[&](void* arg) { trivial_move++; });
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::BackgroundCompaction:NonTrivial",
|
||||
[&](void* /*arg*/) { non_trivial_move++; });
|
||||
[&](void* arg) { non_trivial_move++; });
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
CompactRangeOptions compact_options;
|
||||
@@ -1287,414 +1095,6 @@ TEST_F(DBBloomFilterTest, OptimizeFiltersForHits) {
|
||||
ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
|
||||
ASSERT_EQ(2 /* index and data block */,
|
||||
TestGetTickerCount(options, BLOCK_CACHE_ADD));
|
||||
get_perf_context()->Reset();
|
||||
}
|
||||
|
||||
int CountIter(std::unique_ptr<Iterator>& iter, const Slice& key) {
|
||||
int count = 0;
|
||||
for (iter->Seek(key); iter->Valid() && iter->status() == Status::OK();
|
||||
iter->Next()) {
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// use iterate_upper_bound to hint compatiability of existing bloom filters.
|
||||
// The BF is considered compatible if 1) upper bound and seek key transform
|
||||
// into the same string, or 2) the transformed seek key is of the same length
|
||||
// as the upper bound and two keys are adjacent according to the comparator.
|
||||
TEST_F(DBBloomFilterTest, DynamicBloomFilterUpperBound) {
|
||||
int iteration = 0;
|
||||
for (bool use_block_based_builder : {true, false}) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.prefix_extractor.reset(NewCappedPrefixTransform(4));
|
||||
options.disable_auto_compactions = true;
|
||||
options.statistics = CreateDBStatistics();
|
||||
// Enable prefix bloom for SST files
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.filter_policy.reset(
|
||||
NewBloomFilterPolicy(10, use_block_based_builder));
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(Put("abcdxxx0", "val1"));
|
||||
ASSERT_OK(Put("abcdxxx1", "val2"));
|
||||
ASSERT_OK(Put("abcdxxx2", "val3"));
|
||||
ASSERT_OK(Put("abcdxxx3", "val4"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
{
|
||||
// prefix_extractor has not changed, BF will always be read
|
||||
Slice upper_bound("abce");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abcd0000"), 4);
|
||||
}
|
||||
{
|
||||
Slice upper_bound("abcdzzzz");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abcd0000"), 4);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:5"}}));
|
||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
||||
"rocksdb.FixedPrefix.5"));
|
||||
{
|
||||
// BF changed, [abcdxx00, abce) is a valid bound, will trigger BF read
|
||||
Slice upper_bound("abce");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abcdxx00"), 4);
|
||||
// should check bloom filter since upper bound meets requirement
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
2 + iteration);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
{
|
||||
// [abcdxx01, abcey) is not valid bound since upper bound is too long for
|
||||
// the BF in SST (capped:4)
|
||||
Slice upper_bound("abcey");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abcdxx01"), 4);
|
||||
// should skip bloom filter since upper bound is too long
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
2 + iteration);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
{
|
||||
// [abcdxx02, abcdy) is a valid bound since the prefix is the same
|
||||
Slice upper_bound("abcdy");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abcdxx02"), 4);
|
||||
// should check bloom filter since upper bound matches transformed seek
|
||||
// key
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
2 + iteration * 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
{
|
||||
// [aaaaaaaa, abce) is not a valid bound since 1) they don't share the
|
||||
// same prefix, 2) the prefixes are not consecutive
|
||||
Slice upper_bound("abce");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "aaaaaaaa"), 0);
|
||||
// should skip bloom filter since mismatch is found
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
2 + iteration * 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:3"}}));
|
||||
{
|
||||
// [abc, abd) is not a valid bound since the upper bound is too short
|
||||
// for BF (capped:4)
|
||||
Slice upper_bound("abd");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abc"), 4);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
2 + iteration * 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:4"}}));
|
||||
{
|
||||
// set back to capped:4 and verify BF is always read
|
||||
Slice upper_bound("abd");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "abc"), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
3 + iteration * 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
||||
}
|
||||
iteration++;
|
||||
}
|
||||
}
|
||||
|
||||
// Create multiple SST files each with a different prefix_extractor config,
|
||||
// verify iterators can read all SST files using the latest config.
|
||||
TEST_F(DBBloomFilterTest, DynamicBloomFilterMultipleSST) {
|
||||
int iteration = 0;
|
||||
for (bool use_block_based_builder : {true, false}) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
||||
options.disable_auto_compactions = true;
|
||||
options.statistics = CreateDBStatistics();
|
||||
// Enable prefix bloom for SST files
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.filter_policy.reset(
|
||||
NewBloomFilterPolicy(10, use_block_based_builder));
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Slice upper_bound("foz90000");
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
|
||||
// first SST with fixed:1 BF
|
||||
ASSERT_OK(Put("foo2", "bar2"));
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_OK(Put("foq1", "bar1"));
|
||||
ASSERT_OK(Put("fpa", "0"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
std::unique_ptr<Iterator> iter_old(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter_old, "foo"), 4);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 1);
|
||||
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
||||
"rocksdb.CappedPrefix.3"));
|
||||
read_options.iterate_upper_bound = &upper_bound;
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "foo"), 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
1 + iteration);
|
||||
ASSERT_EQ(CountIter(iter, "gpk"), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
1 + iteration);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
|
||||
// second SST with capped:3 BF
|
||||
ASSERT_OK(Put("foo3", "bar3"));
|
||||
ASSERT_OK(Put("foo4", "bar4"));
|
||||
ASSERT_OK(Put("foq5", "bar5"));
|
||||
ASSERT_OK(Put("fpb", "1"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
{
|
||||
// BF is cappped:3 now
|
||||
std::unique_ptr<Iterator> iter_tmp(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter_tmp, "foo"), 4);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
2 + iteration * 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
ASSERT_EQ(CountIter(iter_tmp, "gpk"), 0);
|
||||
// both counters are incremented because BF is "not changed" for 1 of the
|
||||
// 2 SST files, so filter is checked once and found no match.
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
3 + iteration * 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
||||
}
|
||||
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "fixed:2"}}));
|
||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
||||
"rocksdb.FixedPrefix.2"));
|
||||
// third SST with fixed:2 BF
|
||||
ASSERT_OK(Put("foo6", "bar6"));
|
||||
ASSERT_OK(Put("foo7", "bar7"));
|
||||
ASSERT_OK(Put("foq8", "bar8"));
|
||||
ASSERT_OK(Put("fpc", "2"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
{
|
||||
// BF is fixed:2 now
|
||||
std::unique_ptr<Iterator> iter_tmp(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter_tmp, "foo"), 9);
|
||||
// the first and last BF are checked
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
4 + iteration * 3);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 1);
|
||||
ASSERT_EQ(CountIter(iter_tmp, "gpk"), 0);
|
||||
// only last BF is checked and not found
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
5 + iteration * 3);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 2);
|
||||
}
|
||||
|
||||
// iter_old can only see the first SST, so checked plus 1
|
||||
ASSERT_EQ(CountIter(iter_old, "foo"), 4);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
6 + iteration * 3);
|
||||
// iter was created after the first setoptions call so only full filter
|
||||
// will check the filter
|
||||
ASSERT_EQ(CountIter(iter, "foo"), 2);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
6 + iteration * 4);
|
||||
|
||||
{
|
||||
// keys in all three SSTs are visible to iterator
|
||||
// The range of [foo, foz90000] is compatible with (fixed:1) and (fixed:2)
|
||||
// so +2 for checked counter
|
||||
std::unique_ptr<Iterator> iter_all(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter_all, "foo"), 9);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
7 + iteration * 5);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 2);
|
||||
ASSERT_EQ(CountIter(iter_all, "gpk"), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
8 + iteration * 5);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 3);
|
||||
}
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
||||
"rocksdb.CappedPrefix.3"));
|
||||
{
|
||||
std::unique_ptr<Iterator> iter_all(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter_all, "foo"), 6);
|
||||
// all three SST are checked because the current options has the same as
|
||||
// the remaining SST (capped:3)
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
9 + iteration * 7);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 3);
|
||||
ASSERT_EQ(CountIter(iter_all, "gpk"), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED),
|
||||
10 + iteration * 7);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 4);
|
||||
}
|
||||
// TODO(Zhongyi): Maybe also need to add Get calls to test point look up?
|
||||
iteration++;
|
||||
}
|
||||
}
|
||||
|
||||
// Create a new column family in a running DB, change prefix_extractor
|
||||
// dynamically, verify the iterator created on the new column family behaves
|
||||
// as expected
|
||||
TEST_F(DBBloomFilterTest, DynamicBloomFilterNewColumnFamily) {
|
||||
int iteration = 0;
|
||||
for (bool use_block_based_builder : {true, false}) {
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
||||
options.disable_auto_compactions = true;
|
||||
options.statistics = CreateDBStatistics();
|
||||
// Enable prefix bloom for SST files
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.filter_policy.reset(
|
||||
NewBloomFilterPolicy(10, use_block_based_builder));
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
CreateAndReopenWithCF({"pikachu" + std::to_string(iteration)}, options);
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
// create a new CF and set prefix_extractor dynamically
|
||||
options.prefix_extractor.reset(NewCappedPrefixTransform(3));
|
||||
CreateColumnFamilies({"ramen_dojo_" + std::to_string(iteration)}, options);
|
||||
ASSERT_EQ(0,
|
||||
strcmp(dbfull()->GetOptions(handles_[2]).prefix_extractor->Name(),
|
||||
"rocksdb.CappedPrefix.3"));
|
||||
ASSERT_OK(Put(2, "foo3", "bar3"));
|
||||
ASSERT_OK(Put(2, "foo4", "bar4"));
|
||||
ASSERT_OK(Put(2, "foo5", "bar5"));
|
||||
ASSERT_OK(Put(2, "foq6", "bar6"));
|
||||
ASSERT_OK(Put(2, "fpq7", "bar7"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
{
|
||||
std::unique_ptr<Iterator> iter(
|
||||
db_->NewIterator(read_options, handles_[2]));
|
||||
ASSERT_EQ(CountIter(iter, "foo"), 3);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
ASSERT_OK(
|
||||
dbfull()->SetOptions(handles_[2], {{"prefix_extractor", "fixed:2"}}));
|
||||
ASSERT_EQ(0,
|
||||
strcmp(dbfull()->GetOptions(handles_[2]).prefix_extractor->Name(),
|
||||
"rocksdb.FixedPrefix.2"));
|
||||
{
|
||||
std::unique_ptr<Iterator> iter(
|
||||
db_->NewIterator(read_options, handles_[2]));
|
||||
ASSERT_EQ(CountIter(iter, "foo"), 4);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
ASSERT_OK(dbfull()->DropColumnFamily(handles_[2]));
|
||||
dbfull()->DestroyColumnFamilyHandle(handles_[2]);
|
||||
handles_[2] = nullptr;
|
||||
ASSERT_OK(dbfull()->DropColumnFamily(handles_[1]));
|
||||
dbfull()->DestroyColumnFamilyHandle(handles_[1]);
|
||||
handles_[1] = nullptr;
|
||||
iteration++;
|
||||
}
|
||||
}
|
||||
|
||||
// Verify it's possible to change prefix_extractor at runtime and iterators
|
||||
// behaves as expected
|
||||
TEST_F(DBBloomFilterTest, DynamicBloomFilterOptions) {
|
||||
int iteration = 0;
|
||||
for (bool use_block_based_builder : {true, false}) {
|
||||
Options options;
|
||||
options.create_if_missing = true;
|
||||
options.prefix_extractor.reset(NewFixedPrefixTransform(1));
|
||||
options.disable_auto_compactions = true;
|
||||
options.statistics = CreateDBStatistics();
|
||||
// Enable prefix bloom for SST files
|
||||
BlockBasedTableOptions table_options;
|
||||
table_options.cache_index_and_filter_blocks = true;
|
||||
table_options.filter_policy.reset(
|
||||
NewBloomFilterPolicy(10, use_block_based_builder));
|
||||
options.table_factory.reset(NewBlockBasedTableFactory(table_options));
|
||||
DestroyAndReopen(options);
|
||||
|
||||
ASSERT_OK(Put("foo2", "bar2"));
|
||||
ASSERT_OK(Put("foo", "bar"));
|
||||
ASSERT_OK(Put("foo1", "bar1"));
|
||||
ASSERT_OK(Put("fpa", "0"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
ASSERT_OK(Put("foo3", "bar3"));
|
||||
ASSERT_OK(Put("foo4", "bar4"));
|
||||
ASSERT_OK(Put("foo5", "bar5"));
|
||||
ASSERT_OK(Put("fpb", "1"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
ASSERT_OK(Put("foo6", "bar6"));
|
||||
ASSERT_OK(Put("foo7", "bar7"));
|
||||
ASSERT_OK(Put("foo8", "bar8"));
|
||||
ASSERT_OK(Put("fpc", "2"));
|
||||
dbfull()->Flush(FlushOptions());
|
||||
|
||||
ReadOptions read_options;
|
||||
read_options.prefix_same_as_start = true;
|
||||
{
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter, "foo"), 12);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 3);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
std::unique_ptr<Iterator> iter_old(db_->NewIterator(read_options));
|
||||
ASSERT_EQ(CountIter(iter_old, "foo"), 12);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 6);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
|
||||
ASSERT_OK(dbfull()->SetOptions({{"prefix_extractor", "capped:3"}}));
|
||||
ASSERT_EQ(0, strcmp(dbfull()->GetOptions().prefix_extractor->Name(),
|
||||
"rocksdb.CappedPrefix.3"));
|
||||
{
|
||||
std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
|
||||
// "fp*" should be skipped
|
||||
ASSERT_EQ(CountIter(iter, "foo"), 9);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 6);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
}
|
||||
|
||||
// iterator created before should not be affected and see all keys
|
||||
ASSERT_EQ(CountIter(iter_old, "foo"), 12);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 9);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 0);
|
||||
ASSERT_EQ(CountIter(iter_old, "abc"), 0);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_CHECKED), 12);
|
||||
ASSERT_EQ(TestGetTickerCount(options, BLOOM_FILTER_PREFIX_USEFUL), 3);
|
||||
iteration++;
|
||||
}
|
||||
}
|
||||
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
+101
-103
@@ -45,7 +45,6 @@ class DBTestCompactionFilterWithCompactParam
|
||||
}
|
||||
};
|
||||
|
||||
#ifndef ROCKSDB_VALGRIND_RUN
|
||||
INSTANTIATE_TEST_CASE_P(
|
||||
DBTestCompactionFilterWithCompactOption,
|
||||
DBTestCompactionFilterWithCompactParam,
|
||||
@@ -54,42 +53,36 @@ INSTANTIATE_TEST_CASE_P(
|
||||
DBTestBase::OptionConfig::kUniversalCompactionMultiLevel,
|
||||
DBTestBase::OptionConfig::kLevelSubcompactions,
|
||||
DBTestBase::OptionConfig::kUniversalSubcompactions));
|
||||
#else
|
||||
// Run fewer cases in valgrind
|
||||
INSTANTIATE_TEST_CASE_P(DBTestCompactionFilterWithCompactOption,
|
||||
DBTestCompactionFilterWithCompactParam,
|
||||
::testing::Values(DBTestBase::OptionConfig::kDefault));
|
||||
#endif // ROCKSDB_VALGRIND_RUN
|
||||
|
||||
class KeepFilter : public CompactionFilter {
|
||||
public:
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value, bool* value_changed) const
|
||||
override {
|
||||
cfilter_count++;
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "KeepFilter"; }
|
||||
virtual const char* Name() const override { return "KeepFilter"; }
|
||||
};
|
||||
|
||||
class DeleteFilter : public CompactionFilter {
|
||||
public:
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value, bool* value_changed) const
|
||||
override {
|
||||
cfilter_count++;
|
||||
return true;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "DeleteFilter"; }
|
||||
virtual const char* Name() const override { return "DeleteFilter"; }
|
||||
};
|
||||
|
||||
class DeleteISFilter : public CompactionFilter {
|
||||
public:
|
||||
bool Filter(int /*level*/, const Slice& key, const Slice& /*value*/,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value,
|
||||
bool* value_changed) const override {
|
||||
cfilter_count++;
|
||||
int i = std::stoi(key.ToString());
|
||||
if (i > 5 && i <= 105) {
|
||||
@@ -98,23 +91,23 @@ class DeleteISFilter : public CompactionFilter {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IgnoreSnapshots() const override { return true; }
|
||||
virtual bool IgnoreSnapshots() const override { return true; }
|
||||
|
||||
const char* Name() const override { return "DeleteFilter"; }
|
||||
virtual const char* Name() const override { return "DeleteFilter"; }
|
||||
};
|
||||
|
||||
// Skip x if floor(x/10) is even, use range skips. Requires that keys are
|
||||
// zero-padded to length 10.
|
||||
class SkipEvenFilter : public CompactionFilter {
|
||||
public:
|
||||
Decision FilterV2(int /*level*/, const Slice& key, ValueType /*value_type*/,
|
||||
const Slice& /*existing_value*/, std::string* /*new_value*/,
|
||||
std::string* skip_until) const override {
|
||||
virtual Decision FilterV2(int level, const Slice& key, ValueType value_type,
|
||||
const Slice& existing_value, std::string* new_value,
|
||||
std::string* skip_until) const override {
|
||||
cfilter_count++;
|
||||
int i = std::stoi(key.ToString());
|
||||
if (i / 10 % 2 == 0) {
|
||||
char key_str[100];
|
||||
snprintf(key_str, sizeof(key_str), "%010d", i / 10 * 10 + 10);
|
||||
snprintf(key_str, sizeof(key), "%010d", i / 10 * 10 + 10);
|
||||
*skip_until = key_str;
|
||||
++cfilter_skips;
|
||||
return Decision::kRemoveAndSkipUntil;
|
||||
@@ -122,22 +115,22 @@ class SkipEvenFilter : public CompactionFilter {
|
||||
return Decision::kKeep;
|
||||
}
|
||||
|
||||
bool IgnoreSnapshots() const override { return true; }
|
||||
virtual bool IgnoreSnapshots() const override { return true; }
|
||||
|
||||
const char* Name() const override { return "DeleteFilter"; }
|
||||
virtual const char* Name() const override { return "DeleteFilter"; }
|
||||
};
|
||||
|
||||
class DelayFilter : public CompactionFilter {
|
||||
public:
|
||||
explicit DelayFilter(DBTestBase* d) : db_test(d) {}
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value,
|
||||
bool* value_changed) const override {
|
||||
db_test->env_->addon_time_.fetch_add(1000);
|
||||
return true;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "DelayFilter"; }
|
||||
virtual const char* Name() const override { return "DelayFilter"; }
|
||||
|
||||
private:
|
||||
DBTestBase* db_test;
|
||||
@@ -147,13 +140,13 @@ class ConditionalFilter : public CompactionFilter {
|
||||
public:
|
||||
explicit ConditionalFilter(const std::string* filtered_value)
|
||||
: filtered_value_(filtered_value) {}
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& value,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value,
|
||||
bool* value_changed) const override {
|
||||
return value.ToString() == *filtered_value_;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "ConditionalFilter"; }
|
||||
virtual const char* Name() const override { return "ConditionalFilter"; }
|
||||
|
||||
private:
|
||||
const std::string* filtered_value_;
|
||||
@@ -163,15 +156,16 @@ class ChangeFilter : public CompactionFilter {
|
||||
public:
|
||||
explicit ChangeFilter() {}
|
||||
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
|
||||
std::string* new_value, bool* value_changed) const override {
|
||||
virtual bool Filter(int level, const Slice& key, const Slice& value,
|
||||
std::string* new_value, bool* value_changed) const
|
||||
override {
|
||||
assert(new_value != nullptr);
|
||||
*new_value = NEW_VALUE;
|
||||
*value_changed = true;
|
||||
return false;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "ChangeFilter"; }
|
||||
virtual const char* Name() const override { return "ChangeFilter"; }
|
||||
};
|
||||
|
||||
class KeepFilterFactory : public CompactionFilterFactory {
|
||||
@@ -182,7 +176,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
|
||||
check_context_cf_id_(check_context_cf_id),
|
||||
compaction_filter_created_(false) {}
|
||||
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
if (check_context_) {
|
||||
EXPECT_EQ(expect_full_compaction_.load(), context.is_full_compaction);
|
||||
@@ -197,7 +191,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
|
||||
|
||||
bool compaction_filter_created() const { return compaction_filter_created_; }
|
||||
|
||||
const char* Name() const override { return "KeepFilterFactory"; }
|
||||
virtual const char* Name() const override { return "KeepFilterFactory"; }
|
||||
bool check_context_;
|
||||
bool check_context_cf_id_;
|
||||
std::atomic_bool expect_full_compaction_;
|
||||
@@ -208,7 +202,7 @@ class KeepFilterFactory : public CompactionFilterFactory {
|
||||
|
||||
class DeleteFilterFactory : public CompactionFilterFactory {
|
||||
public:
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
if (context.is_manual_compaction) {
|
||||
return std::unique_ptr<CompactionFilter>(new DeleteFilter());
|
||||
@@ -217,13 +211,13 @@ class DeleteFilterFactory : public CompactionFilterFactory {
|
||||
}
|
||||
}
|
||||
|
||||
const char* Name() const override { return "DeleteFilterFactory"; }
|
||||
virtual const char* Name() const override { return "DeleteFilterFactory"; }
|
||||
};
|
||||
|
||||
// Delete Filter Factory which ignores snapshots
|
||||
class DeleteISFilterFactory : public CompactionFilterFactory {
|
||||
public:
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
if (context.is_manual_compaction) {
|
||||
return std::unique_ptr<CompactionFilter>(new DeleteISFilter());
|
||||
@@ -232,12 +226,12 @@ class DeleteISFilterFactory : public CompactionFilterFactory {
|
||||
}
|
||||
}
|
||||
|
||||
const char* Name() const override { return "DeleteFilterFactory"; }
|
||||
virtual const char* Name() const override { return "DeleteFilterFactory"; }
|
||||
};
|
||||
|
||||
class SkipEvenFilterFactory : public CompactionFilterFactory {
|
||||
public:
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
if (context.is_manual_compaction) {
|
||||
return std::unique_ptr<CompactionFilter>(new SkipEvenFilter());
|
||||
@@ -246,18 +240,18 @@ class SkipEvenFilterFactory : public CompactionFilterFactory {
|
||||
}
|
||||
}
|
||||
|
||||
const char* Name() const override { return "SkipEvenFilterFactory"; }
|
||||
virtual const char* Name() const override { return "SkipEvenFilterFactory"; }
|
||||
};
|
||||
|
||||
class DelayFilterFactory : public CompactionFilterFactory {
|
||||
public:
|
||||
explicit DelayFilterFactory(DBTestBase* d) : db_test(d) {}
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& /*context*/) override {
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
return std::unique_ptr<CompactionFilter>(new DelayFilter(db_test));
|
||||
}
|
||||
|
||||
const char* Name() const override { return "DelayFilterFactory"; }
|
||||
virtual const char* Name() const override { return "DelayFilterFactory"; }
|
||||
|
||||
private:
|
||||
DBTestBase* db_test;
|
||||
@@ -268,13 +262,15 @@ class ConditionalFilterFactory : public CompactionFilterFactory {
|
||||
explicit ConditionalFilterFactory(const Slice& filtered_value)
|
||||
: filtered_value_(filtered_value.ToString()) {}
|
||||
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& /*context*/) override {
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
return std::unique_ptr<CompactionFilter>(
|
||||
new ConditionalFilter(&filtered_value_));
|
||||
}
|
||||
|
||||
const char* Name() const override { return "ConditionalFilterFactory"; }
|
||||
virtual const char* Name() const override {
|
||||
return "ConditionalFilterFactory";
|
||||
}
|
||||
|
||||
private:
|
||||
std::string filtered_value_;
|
||||
@@ -284,12 +280,12 @@ class ChangeFilterFactory : public CompactionFilterFactory {
|
||||
public:
|
||||
explicit ChangeFilterFactory() {}
|
||||
|
||||
std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& /*context*/) override {
|
||||
virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter(
|
||||
const CompactionFilter::Context& context) override {
|
||||
return std::unique_ptr<CompactionFilter>(new ChangeFilter());
|
||||
}
|
||||
|
||||
const char* Name() const override { return "ChangeFilterFactory"; }
|
||||
virtual const char* Name() const override { return "ChangeFilterFactory"; }
|
||||
};
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
@@ -335,10 +331,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
|
||||
Arena arena;
|
||||
{
|
||||
InternalKeyComparator icmp(options.comparator);
|
||||
ReadRangeDelAggregator range_del_agg(&icmp,
|
||||
kMaxSequenceNumber /* upper_bound */);
|
||||
ScopedArenaIterator iter(dbfull()->NewInternalIterator(
|
||||
&arena, &range_del_agg, kMaxSequenceNumber, handles_[1]));
|
||||
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
|
||||
ScopedArenaIterator iter(
|
||||
dbfull()->NewInternalIterator(&arena, &range_del_agg, handles_[1]));
|
||||
iter->SeekToFirst();
|
||||
ASSERT_OK(iter->status());
|
||||
while (iter->Valid()) {
|
||||
@@ -352,7 +347,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
|
||||
}
|
||||
}
|
||||
ASSERT_EQ(total, 100000);
|
||||
ASSERT_EQ(count, 0);
|
||||
ASSERT_EQ(count, 1);
|
||||
|
||||
// overwrite all the 100K keys once again.
|
||||
for (int i = 0; i < 100000; i++) {
|
||||
@@ -425,10 +420,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
|
||||
count = 0;
|
||||
{
|
||||
InternalKeyComparator icmp(options.comparator);
|
||||
ReadRangeDelAggregator range_del_agg(&icmp,
|
||||
kMaxSequenceNumber /* upper_bound */);
|
||||
ScopedArenaIterator iter(dbfull()->NewInternalIterator(
|
||||
&arena, &range_del_agg, kMaxSequenceNumber, handles_[1]));
|
||||
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
|
||||
ScopedArenaIterator iter(
|
||||
dbfull()->NewInternalIterator(&arena, &range_del_agg, handles_[1]));
|
||||
iter->SeekToFirst();
|
||||
ASSERT_OK(iter->status());
|
||||
while (iter->Valid()) {
|
||||
@@ -643,10 +637,9 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
|
||||
int total = 0;
|
||||
Arena arena;
|
||||
InternalKeyComparator icmp(options.comparator);
|
||||
ReadRangeDelAggregator range_del_agg(&icmp,
|
||||
kMaxSequenceNumber /* snapshots */);
|
||||
ScopedArenaIterator iter(dbfull()->NewInternalIterator(
|
||||
&arena, &range_del_agg, kMaxSequenceNumber));
|
||||
RangeDelAggregator range_del_agg(icmp, {} /* snapshots */);
|
||||
ScopedArenaIterator iter(
|
||||
dbfull()->NewInternalIterator(&arena, &range_del_agg));
|
||||
iter->SeekToFirst();
|
||||
ASSERT_OK(iter->status());
|
||||
while (iter->Valid()) {
|
||||
@@ -659,7 +652,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextManual) {
|
||||
iter->Next();
|
||||
}
|
||||
ASSERT_EQ(total, 700);
|
||||
ASSERT_EQ(count, 0);
|
||||
ASSERT_EQ(count, 1);
|
||||
}
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
@@ -694,7 +687,44 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
// Compaction filters aplies to all records, regardless snapshots.
|
||||
// Compaction filters should only be applied to records that are newer than the
|
||||
// latest snapshot. This test inserts records and applies a delete filter.
|
||||
TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
|
||||
Options options = CurrentOptions();
|
||||
options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
|
||||
options.disable_auto_compactions = true;
|
||||
options.create_if_missing = true;
|
||||
DestroyAndReopen(options);
|
||||
|
||||
// Put some data.
|
||||
const Snapshot* snapshot = nullptr;
|
||||
for (int table = 0; table < 4; ++table) {
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
Put(ToString(table * 100 + i), "val");
|
||||
}
|
||||
Flush();
|
||||
|
||||
if (table == 0) {
|
||||
snapshot = db_->GetSnapshot();
|
||||
}
|
||||
}
|
||||
assert(snapshot != nullptr);
|
||||
|
||||
cfilter_count = 0;
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
||||
// The filter should delete 10 records.
|
||||
ASSERT_EQ(30U, cfilter_count);
|
||||
|
||||
// Release the snapshot and compact again -> now all records should be
|
||||
// removed.
|
||||
db_->ReleaseSnapshot(snapshot);
|
||||
ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr));
|
||||
ASSERT_EQ(0U, CountLiveFiles());
|
||||
}
|
||||
|
||||
// Compaction filters should only be applied to records that are newer than the
|
||||
// latest snapshot. However, if the compaction filter asks to ignore snapshots
|
||||
// records newer than the snapshot will also be processed
|
||||
TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
|
||||
std::string five = ToString(5);
|
||||
Options options = CurrentOptions();
|
||||
@@ -735,7 +765,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
|
||||
iter->Next();
|
||||
}
|
||||
ASSERT_EQ(count, 6);
|
||||
read_options.snapshot = nullptr;
|
||||
read_options.snapshot = 0;
|
||||
std::unique_ptr<Iterator> iter1(db_->NewIterator(read_options));
|
||||
iter1->SeekToFirst();
|
||||
count = 0;
|
||||
@@ -809,7 +839,7 @@ TEST_F(DBTestCompactionFilter, SkipUntilWithBloomFilter) {
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Put("0000000010", "v10");
|
||||
Put("0000000020", "v20"); // skipped
|
||||
Put("0000000020", "v20"); // skipped
|
||||
Put("0000000050", "v50");
|
||||
Flush();
|
||||
|
||||
@@ -832,38 +862,6 @@ TEST_F(DBTestCompactionFilter, SkipUntilWithBloomFilter) {
|
||||
EXPECT_EQ("v50", val);
|
||||
}
|
||||
|
||||
class TestNotSupportedFilter : public CompactionFilter {
|
||||
public:
|
||||
bool Filter(int /*level*/, const Slice& /*key*/, const Slice& /*value*/,
|
||||
std::string* /*new_value*/,
|
||||
bool* /*value_changed*/) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
const char* Name() const override { return "NotSupported"; }
|
||||
bool IgnoreSnapshots() const override { return false; }
|
||||
};
|
||||
|
||||
TEST_F(DBTestCompactionFilter, IgnoreSnapshotsFalse) {
|
||||
Options options = CurrentOptions();
|
||||
options.compaction_filter = new TestNotSupportedFilter();
|
||||
DestroyAndReopen(options);
|
||||
|
||||
Put("a", "v10");
|
||||
Put("z", "v20");
|
||||
Flush();
|
||||
|
||||
Put("a", "v10");
|
||||
Put("z", "v20");
|
||||
Flush();
|
||||
|
||||
// Comapction should fail because IgnoreSnapshots() = false
|
||||
EXPECT_TRUE(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)
|
||||
.IsNotSupported());
|
||||
|
||||
delete options.compaction_filter;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
+40
-1301
File diff suppressed because it is too large
Load Diff
+27
-26
@@ -27,7 +27,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase) {
|
||||
return;
|
||||
}
|
||||
// Use InMemoryEnv, or it would be too slow.
|
||||
std::unique_ptr<Env> env(new MockEnv(env_));
|
||||
unique_ptr<Env> env(new MockEnv(env_));
|
||||
|
||||
const int kNKeys = 1000;
|
||||
int keys[kNKeys];
|
||||
@@ -125,7 +125,6 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
|
||||
int kMaxKey = 1000000;
|
||||
|
||||
Options options = CurrentOptions();
|
||||
options.compression = kNoCompression;
|
||||
options.create_if_missing = true;
|
||||
options.write_buffer_size = 20480;
|
||||
options.max_write_buffer_number = 2;
|
||||
@@ -168,8 +167,8 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
|
||||
ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
|
||||
ASSERT_EQ(4U, int_prop);
|
||||
|
||||
// Insert extra about 28K to L0. After they are compacted to L4, the base
|
||||
// level should be changed to L3.
|
||||
// Insert extra about 28K to L0. After they are compacted to L4, base level
|
||||
// should be changed to L3.
|
||||
ASSERT_OK(dbfull()->SetOptions({
|
||||
{"disable_auto_compactions", "true"},
|
||||
}));
|
||||
@@ -190,7 +189,13 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
|
||||
ASSERT_TRUE(db_->GetProperty("rocksdb.num-files-at-level2", &str_prop));
|
||||
ASSERT_EQ("0", str_prop);
|
||||
|
||||
// Write even more data while leaving the base level at L3.
|
||||
// Trigger parallel compaction, and the first one would change the base
|
||||
// level.
|
||||
// Hold compaction jobs to make sure
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"CompactionJob::Run():Start",
|
||||
[&](void* arg) { env_->SleepForMicroseconds(100000); });
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
ASSERT_OK(dbfull()->SetOptions({
|
||||
{"disable_auto_compactions", "true"},
|
||||
}));
|
||||
@@ -203,12 +208,18 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
|
||||
{"disable_auto_compactions", "false"},
|
||||
}));
|
||||
Flush();
|
||||
// Wait for 200 milliseconds before proceeding compactions to make sure two
|
||||
// parallel ones are executed.
|
||||
env_->SleepForMicroseconds(200000);
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
|
||||
ASSERT_EQ(3U, int_prop);
|
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||
|
||||
// Fill up L0, and then run an (auto) L0->Lmax compaction to raise the base
|
||||
// level to 2.
|
||||
// Trigger a condition that the compaction changes base level and L0->Lbase
|
||||
// happens at the same time.
|
||||
// We try to make last levels' targets to be 40K, 160K, 640K, add triggers
|
||||
// another compaction from 40K->160K.
|
||||
ASSERT_OK(dbfull()->SetOptions({
|
||||
{"disable_auto_compactions", "true"},
|
||||
}));
|
||||
@@ -218,31 +229,23 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
|
||||
ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
|
||||
RandomString(&rnd, 380)));
|
||||
}
|
||||
|
||||
// Make sure that the compaction starts before the last bit of data is
|
||||
// flushed, so that the base level isn't raised to L1.
|
||||
rocksdb::SyncPoint::GetInstance()->LoadDependency({
|
||||
{"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
|
||||
});
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
ASSERT_OK(dbfull()->SetOptions({
|
||||
{"disable_auto_compactions", "false"},
|
||||
}));
|
||||
|
||||
TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
|
||||
Flush();
|
||||
dbfull()->TEST_WaitForCompact();
|
||||
ASSERT_TRUE(db_->GetIntProperty("rocksdb.base-level", &int_prop));
|
||||
ASSERT_EQ(2U, int_prop);
|
||||
|
||||
// A manual compaction will trigger the base level to become L2
|
||||
// Keep Writing data until base level changed 2->1. There will be L0->L2
|
||||
// compaction going on at the same time.
|
||||
rocksdb::SyncPoint::GetInstance()->DisableProcessing();
|
||||
rocksdb::SyncPoint::GetInstance()->ClearAllCallBacks();
|
||||
|
||||
// Write more data until the base level changes to L1. There will be
|
||||
// a manual compaction going on at the same time.
|
||||
rocksdb::SyncPoint::GetInstance()->LoadDependency({
|
||||
{"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:1"},
|
||||
{"DynamicLevelMaxBytesBase2:2", "CompactionJob::Run():End"},
|
||||
{"CompactionJob::Run():Start", "DynamicLevelMaxBytesBase2:0"},
|
||||
{"DynamicLevelMaxBytesBase2:1", "CompactionJob::Run():End"},
|
||||
{"DynamicLevelMaxBytesBase2:compact_range_finish",
|
||||
"FlushJob::WriteLevel0Table"},
|
||||
});
|
||||
@@ -254,12 +257,12 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBase2) {
|
||||
TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:compact_range_finish");
|
||||
});
|
||||
|
||||
TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
|
||||
TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:0");
|
||||
for (int i = 0; i < 2; i++) {
|
||||
ASSERT_OK(Put(Key(static_cast<int>(rnd.Uniform(kMaxKey))),
|
||||
RandomString(&rnd, 380)));
|
||||
}
|
||||
TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:2");
|
||||
TEST_SYNC_POINT("DynamicLevelMaxBytesBase2:1");
|
||||
|
||||
Flush();
|
||||
|
||||
@@ -375,7 +378,7 @@ TEST_F(DBTestDynamicLevel, DynamicLevelMaxBytesBaseInc) {
|
||||
int non_trivial = 0;
|
||||
rocksdb::SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::BackgroundCompaction:NonTrivial",
|
||||
[&](void* /*arg*/) { non_trivial++; });
|
||||
[&](void* arg) { non_trivial++; });
|
||||
rocksdb::SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Random rnd(301);
|
||||
@@ -498,8 +501,6 @@ int main(int argc, char** argv) {
|
||||
::testing::InitGoogleTest(&argc, argv);
|
||||
return RUN_ALL_TESTS();
|
||||
#else
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -40,7 +40,7 @@ TEST_F(DBEncryptionTest, CheckEncrypted) {
|
||||
continue;
|
||||
}
|
||||
auto filePath = dbname_ + "/" + *it;
|
||||
std::unique_ptr<SequentialFile> seqFile;
|
||||
unique_ptr<SequentialFile> seqFile;
|
||||
auto envOptions = EnvOptions(CurrentOptions());
|
||||
status = defaultEnv->NewSequentialFile(filePath, &seqFile, envOptions);
|
||||
ASSERT_OK(status);
|
||||
|
||||
+22
-30
@@ -44,7 +44,7 @@ Status DBImpl::EnableFileDeletions(bool force) {
|
||||
// Job id == 0 means that this is not our background process, but rather
|
||||
// user thread
|
||||
JobContext job_context(0);
|
||||
bool file_deletion_enabled = false;
|
||||
bool should_purge_files = false;
|
||||
{
|
||||
InstrumentedMutexLock l(&mutex_);
|
||||
if (force) {
|
||||
@@ -54,18 +54,19 @@ Status DBImpl::EnableFileDeletions(bool force) {
|
||||
--disable_delete_obsolete_files_;
|
||||
}
|
||||
if (disable_delete_obsolete_files_ == 0) {
|
||||
file_deletion_enabled = true;
|
||||
ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
|
||||
should_purge_files = true;
|
||||
FindObsoleteFiles(&job_context, true);
|
||||
bg_cv_.SignalAll();
|
||||
} else {
|
||||
ROCKS_LOG_WARN(
|
||||
immutable_db_options_.info_log,
|
||||
"File Deletions Enable, but not really enabled. Counter: %d",
|
||||
disable_delete_obsolete_files_);
|
||||
}
|
||||
}
|
||||
if (file_deletion_enabled) {
|
||||
ROCKS_LOG_INFO(immutable_db_options_.info_log, "File Deletions Enabled");
|
||||
if (should_purge_files) {
|
||||
PurgeObsoleteFiles(job_context);
|
||||
} else {
|
||||
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
||||
"File Deletions Enable, but not really enabled. Counter: %d",
|
||||
disable_delete_obsolete_files_);
|
||||
}
|
||||
job_context.Clean();
|
||||
LogFlush(immutable_db_options_.info_log);
|
||||
@@ -73,7 +74,7 @@ Status DBImpl::EnableFileDeletions(bool force) {
|
||||
}
|
||||
|
||||
int DBImpl::IsFileDeletionsEnabled() const {
|
||||
return !disable_delete_obsolete_files_;
|
||||
return disable_delete_obsolete_files_;
|
||||
}
|
||||
|
||||
Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||
@@ -86,28 +87,19 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||
if (flush_memtable) {
|
||||
// flush all dirty data to disk.
|
||||
Status status;
|
||||
if (immutable_db_options_.atomic_flush) {
|
||||
autovector<ColumnFamilyData*> cfds;
|
||||
SelectColumnFamiliesForAtomicFlush(&cfds);
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
if (cfd->IsDropped()) {
|
||||
continue;
|
||||
}
|
||||
cfd->Ref();
|
||||
mutex_.Unlock();
|
||||
status = AtomicFlushMemTables(cfds, FlushOptions(),
|
||||
FlushReason::kGetLiveFiles);
|
||||
status = FlushMemTable(cfd, FlushOptions());
|
||||
TEST_SYNC_POINT("DBImpl::GetLiveFiles:1");
|
||||
TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
|
||||
mutex_.Lock();
|
||||
} else {
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
if (cfd->IsDropped()) {
|
||||
continue;
|
||||
}
|
||||
cfd->Ref();
|
||||
mutex_.Unlock();
|
||||
status = FlushMemTable(cfd, FlushOptions(), FlushReason::kGetLiveFiles);
|
||||
TEST_SYNC_POINT("DBImpl::GetLiveFiles:1");
|
||||
TEST_SYNC_POINT("DBImpl::GetLiveFiles:2");
|
||||
mutex_.Lock();
|
||||
cfd->Unref();
|
||||
if (!status.ok()) {
|
||||
break;
|
||||
}
|
||||
cfd->Unref();
|
||||
if (!status.ok()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
|
||||
@@ -134,7 +126,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
|
||||
|
||||
// create names of the live files. The names are not absolute
|
||||
// paths, instead they are relative to dbname_;
|
||||
for (const auto& live_file : live) {
|
||||
for (auto live_file : live) {
|
||||
ret.push_back(MakeTableFileName("", live_file.GetNumber()));
|
||||
}
|
||||
|
||||
|
||||
+18
-323
@@ -25,12 +25,6 @@ class DBFlushDirectIOTest : public DBFlushTest,
|
||||
DBFlushDirectIOTest() : DBFlushTest() {}
|
||||
};
|
||||
|
||||
class DBAtomicFlushTest : public DBFlushTest,
|
||||
public ::testing::WithParamInterface<bool> {
|
||||
public:
|
||||
DBAtomicFlushTest() : DBFlushTest() {}
|
||||
};
|
||||
|
||||
// We had issue when two background threads trying to flush at the same time,
|
||||
// only one of them get committed. The test verifies the issue is fixed.
|
||||
TEST_F(DBFlushTest, FlushWhileWritingManifest) {
|
||||
@@ -41,12 +35,11 @@ TEST_F(DBFlushTest, FlushWhileWritingManifest) {
|
||||
Reopen(options);
|
||||
FlushOptions no_wait;
|
||||
no_wait.wait = false;
|
||||
no_wait.allow_write_stall=true;
|
||||
|
||||
SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"VersionSet::LogAndApply:WriteManifest",
|
||||
"DBFlushTest::FlushWhileWritingManifest:1"},
|
||||
{"MemTableList::TryInstallMemtableFlushResults:InProgress",
|
||||
{"MemTableList::InstallMemtableFlushResults:InProgress",
|
||||
"VersionSet::LogAndApply:WriteManifestDone"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
@@ -62,8 +55,6 @@ TEST_F(DBFlushTest, FlushWhileWritingManifest) {
|
||||
#endif // ROCKSDB_LITE
|
||||
}
|
||||
|
||||
// Disable this test temporarily on Travis as it fails intermittently.
|
||||
// Github issue: #4151
|
||||
TEST_F(DBFlushTest, SyncFail) {
|
||||
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
|
||||
new FaultInjectionTestEnv(env_));
|
||||
@@ -72,65 +63,29 @@ TEST_F(DBFlushTest, SyncFail) {
|
||||
options.env = fault_injection_env.get();
|
||||
|
||||
SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"DBFlushTest::SyncFail:GetVersionRefCount:1",
|
||||
"DBImpl::FlushMemTableToOutputFile:BeforePickMemtables"},
|
||||
{"DBImpl::FlushMemTableToOutputFile:AfterPickMemtables",
|
||||
"DBFlushTest::SyncFail:GetVersionRefCount:2"},
|
||||
{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedLogs:Start"},
|
||||
{{"DBFlushTest::SyncFail:1", "DBImpl::SyncClosedLogs:Start"},
|
||||
{"DBImpl::SyncClosedLogs:Failed", "DBFlushTest::SyncFail:2"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
CreateAndReopenWithCF({"pikachu"}, options);
|
||||
Put("key", "value");
|
||||
auto* cfd =
|
||||
reinterpret_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())
|
||||
->cfd();
|
||||
FlushOptions flush_options;
|
||||
flush_options.wait = false;
|
||||
ASSERT_OK(dbfull()->Flush(flush_options));
|
||||
// Flush installs a new super-version. Get the ref count after that.
|
||||
auto current_before = cfd->current();
|
||||
int refs_before = cfd->current()->TEST_refs();
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncFail:GetVersionRefCount:1");
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncFail:GetVersionRefCount:2");
|
||||
int refs_after_picking_memtables = cfd->current()->TEST_refs();
|
||||
ASSERT_EQ(refs_before + 1, refs_after_picking_memtables);
|
||||
fault_injection_env->SetFilesystemActive(false);
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncFail:1");
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
|
||||
fault_injection_env->SetFilesystemActive(true);
|
||||
// Now the background job will do the flush; wait for it.
|
||||
dbfull()->TEST_WaitForFlushMemTable();
|
||||
#ifndef ROCKSDB_LITE
|
||||
ASSERT_EQ("", FilesPerLevel()); // flush failed.
|
||||
#endif // ROCKSDB_LITE
|
||||
// Backgroun flush job should release ref count to current version.
|
||||
ASSERT_EQ(current_before, cfd->current());
|
||||
ASSERT_EQ(refs_before, cfd->current()->TEST_refs());
|
||||
Destroy(options);
|
||||
}
|
||||
|
||||
TEST_F(DBFlushTest, SyncSkip) {
|
||||
Options options = CurrentOptions();
|
||||
|
||||
SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"DBFlushTest::SyncSkip:1", "DBImpl::SyncClosedLogs:Skip"},
|
||||
{"DBImpl::SyncClosedLogs:Skip", "DBFlushTest::SyncSkip:2"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
Reopen(options);
|
||||
Put("key", "value");
|
||||
|
||||
auto* cfd =
|
||||
reinterpret_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())
|
||||
->cfd();
|
||||
int refs_before = cfd->current()->TEST_refs();
|
||||
FlushOptions flush_options;
|
||||
flush_options.wait = false;
|
||||
ASSERT_OK(dbfull()->Flush(flush_options));
|
||||
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncSkip:1");
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncSkip:2");
|
||||
|
||||
// Now the background job will do the flush; wait for it.
|
||||
fault_injection_env->SetFilesystemActive(false);
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncFail:1");
|
||||
TEST_SYNC_POINT("DBFlushTest::SyncFail:2");
|
||||
fault_injection_env->SetFilesystemActive(true);
|
||||
dbfull()->TEST_WaitForFlushMemTable();
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
ASSERT_EQ("", FilesPerLevel()); // flush failed.
|
||||
#endif // ROCKSDB_LITE
|
||||
// Flush job should release ref count to current version.
|
||||
ASSERT_EQ(refs_before, cfd->current()->TEST_refs());
|
||||
Destroy(options);
|
||||
}
|
||||
|
||||
@@ -146,7 +101,7 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
|
||||
std::thread::id tid;
|
||||
int num_flushes = 0, num_compactions = 0;
|
||||
SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::BGWorkFlush", [&](void* /*arg*/) {
|
||||
"DBImpl::BGWorkFlush", [&](void* arg) {
|
||||
if (tid == std::thread::id()) {
|
||||
tid = std::this_thread::get_id();
|
||||
} else {
|
||||
@@ -155,7 +110,7 @@ TEST_F(DBFlushTest, FlushInLowPriThreadPool) {
|
||||
++num_flushes;
|
||||
});
|
||||
SyncPoint::GetInstance()->SetCallBack(
|
||||
"DBImpl::BGWorkCompaction", [&](void* /*arg*/) {
|
||||
"DBImpl::BGWorkCompaction", [&](void* arg) {
|
||||
ASSERT_EQ(tid, std::this_thread::get_id());
|
||||
++num_compactions;
|
||||
});
|
||||
@@ -182,7 +137,7 @@ TEST_F(DBFlushTest, ManualFlushWithMinWriteBufferNumberToMerge) {
|
||||
{{"DBImpl::BGWorkFlush",
|
||||
"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:1"},
|
||||
{"DBFlushTest::ManualFlushWithMinWriteBufferNumberToMerge:2",
|
||||
"FlushJob::WriteLevel0Table"}});
|
||||
"DBImpl::FlushMemTableToOutputFile:BeforeInstallSV"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
ASSERT_OK(Put("key1", "value1"));
|
||||
@@ -230,269 +185,9 @@ TEST_P(DBFlushDirectIOTest, DirectIO) {
|
||||
delete options.env;
|
||||
}
|
||||
|
||||
TEST_F(DBFlushTest, FlushError) {
|
||||
Options options;
|
||||
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
|
||||
new FaultInjectionTestEnv(env_));
|
||||
options.write_buffer_size = 100;
|
||||
options.max_write_buffer_number = 4;
|
||||
options.min_write_buffer_number_to_merge = 3;
|
||||
options.disable_auto_compactions = true;
|
||||
options.env = fault_injection_env.get();
|
||||
Reopen(options);
|
||||
|
||||
ASSERT_OK(Put("key1", "value1"));
|
||||
ASSERT_OK(Put("key2", "value2"));
|
||||
fault_injection_env->SetFilesystemActive(false);
|
||||
Status s = dbfull()->TEST_SwitchMemtable();
|
||||
fault_injection_env->SetFilesystemActive(true);
|
||||
Destroy(options);
|
||||
ASSERT_NE(s, Status::OK());
|
||||
}
|
||||
|
||||
TEST_F(DBFlushTest, ManualFlushFailsInReadOnlyMode) {
|
||||
// Regression test for bug where manual flush hangs forever when the DB
|
||||
// is in read-only mode. Verify it now at least returns, despite failing.
|
||||
Options options;
|
||||
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
|
||||
new FaultInjectionTestEnv(env_));
|
||||
options.env = fault_injection_env.get();
|
||||
options.max_write_buffer_number = 2;
|
||||
Reopen(options);
|
||||
|
||||
// Trigger a first flush but don't let it run
|
||||
ASSERT_OK(db_->PauseBackgroundWork());
|
||||
ASSERT_OK(Put("key1", "value1"));
|
||||
FlushOptions flush_opts;
|
||||
flush_opts.wait = false;
|
||||
ASSERT_OK(db_->Flush(flush_opts));
|
||||
|
||||
// Write a key to the second memtable so we have something to flush later
|
||||
// after the DB is in read-only mode.
|
||||
ASSERT_OK(Put("key2", "value2"));
|
||||
|
||||
// Let the first flush continue, hit an error, and put the DB in read-only
|
||||
// mode.
|
||||
fault_injection_env->SetFilesystemActive(false);
|
||||
ASSERT_OK(db_->ContinueBackgroundWork());
|
||||
dbfull()->TEST_WaitForFlushMemTable();
|
||||
#ifndef ROCKSDB_LITE
|
||||
uint64_t num_bg_errors;
|
||||
ASSERT_TRUE(db_->GetIntProperty(DB::Properties::kBackgroundErrors,
|
||||
&num_bg_errors));
|
||||
ASSERT_GT(num_bg_errors, 0);
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
// In the bug scenario, triggering another flush would cause the second flush
|
||||
// to hang forever. After the fix we expect it to return an error.
|
||||
ASSERT_NOK(db_->Flush(FlushOptions()));
|
||||
|
||||
Close();
|
||||
}
|
||||
|
||||
TEST_P(DBAtomicFlushTest, ManualAtomicFlush) {
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.atomic_flush = GetParam();
|
||||
options.write_buffer_size = (static_cast<size_t>(64) << 20);
|
||||
|
||||
CreateAndReopenWithCF({"pikachu", "eevee"}, options);
|
||||
size_t num_cfs = handles_.size();
|
||||
ASSERT_EQ(3, num_cfs);
|
||||
WriteOptions wopts;
|
||||
wopts.disableWAL = true;
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
ASSERT_OK(Put(static_cast<int>(i) /*cf*/, "key", "value", wopts));
|
||||
}
|
||||
std::vector<int> cf_ids;
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
cf_ids.emplace_back(static_cast<int>(i));
|
||||
}
|
||||
ASSERT_OK(Flush(cf_ids));
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
|
||||
ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
|
||||
ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty());
|
||||
}
|
||||
}
|
||||
|
||||
TEST_P(DBAtomicFlushTest, AtomicFlushTriggeredByMemTableFull) {
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.atomic_flush = GetParam();
|
||||
// 4KB so that we can easily trigger auto flush.
|
||||
options.write_buffer_size = 4096;
|
||||
|
||||
SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"DBImpl::BackgroundCallFlush:FlushFinish:0",
|
||||
"DBAtomicFlushTest::AtomicFlushTriggeredByMemTableFull:BeforeCheck"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
CreateAndReopenWithCF({"pikachu", "eevee"}, options);
|
||||
size_t num_cfs = handles_.size();
|
||||
ASSERT_EQ(3, num_cfs);
|
||||
WriteOptions wopts;
|
||||
wopts.disableWAL = true;
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
ASSERT_OK(Put(static_cast<int>(i) /*cf*/, "key", "value", wopts));
|
||||
}
|
||||
// Keep writing to one of them column families to trigger auto flush.
|
||||
for (int i = 0; i != 4000; ++i) {
|
||||
ASSERT_OK(Put(static_cast<int>(num_cfs) - 1 /*cf*/,
|
||||
"key" + std::to_string(i), "value" + std::to_string(i),
|
||||
wopts));
|
||||
}
|
||||
|
||||
TEST_SYNC_POINT(
|
||||
"DBAtomicFlushTest::AtomicFlushTriggeredByMemTableFull:BeforeCheck");
|
||||
if (options.atomic_flush) {
|
||||
for (size_t i = 0; i != num_cfs - 1; ++i) {
|
||||
auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
|
||||
ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
|
||||
ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty());
|
||||
}
|
||||
} else {
|
||||
for (size_t i = 0; i != num_cfs - 1; ++i) {
|
||||
auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
|
||||
ASSERT_EQ(0, cfh->cfd()->imm()->NumNotFlushed());
|
||||
ASSERT_FALSE(cfh->cfd()->mem()->IsEmpty());
|
||||
}
|
||||
}
|
||||
SyncPoint::GetInstance()->DisableProcessing();
|
||||
}
|
||||
|
||||
TEST_P(DBAtomicFlushTest, AtomicFlushRollbackSomeJobs) {
|
||||
bool atomic_flush = GetParam();
|
||||
if (!atomic_flush) {
|
||||
return;
|
||||
}
|
||||
std::unique_ptr<FaultInjectionTestEnv> fault_injection_env(
|
||||
new FaultInjectionTestEnv(env_));
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.atomic_flush = atomic_flush;
|
||||
options.env = fault_injection_env.get();
|
||||
SyncPoint::GetInstance()->DisableProcessing();
|
||||
SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"DBImpl::AtomicFlushMemTablesToOutputFiles:SomeFlushJobsComplete:1",
|
||||
"DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:1"},
|
||||
{"DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:2",
|
||||
"DBImpl::AtomicFlushMemTablesToOutputFiles:SomeFlushJobsComplete:2"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
CreateAndReopenWithCF({"pikachu", "eevee"}, options);
|
||||
size_t num_cfs = handles_.size();
|
||||
ASSERT_EQ(3, num_cfs);
|
||||
WriteOptions wopts;
|
||||
wopts.disableWAL = true;
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
int cf_id = static_cast<int>(i);
|
||||
ASSERT_OK(Put(cf_id, "key", "value", wopts));
|
||||
}
|
||||
FlushOptions flush_opts;
|
||||
flush_opts.wait = false;
|
||||
ASSERT_OK(dbfull()->Flush(flush_opts, handles_));
|
||||
TEST_SYNC_POINT("DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:1");
|
||||
fault_injection_env->SetFilesystemActive(false);
|
||||
TEST_SYNC_POINT("DBAtomicFlushTest::AtomicFlushRollbackSomeJobs:2");
|
||||
for (auto* cfh : handles_) {
|
||||
dbfull()->TEST_WaitForFlushMemTable(cfh);
|
||||
}
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
auto cfh = static_cast<ColumnFamilyHandleImpl*>(handles_[i]);
|
||||
ASSERT_EQ(1, cfh->cfd()->imm()->NumNotFlushed());
|
||||
ASSERT_TRUE(cfh->cfd()->mem()->IsEmpty());
|
||||
}
|
||||
fault_injection_env->SetFilesystemActive(true);
|
||||
Destroy(options);
|
||||
}
|
||||
|
||||
TEST_P(DBAtomicFlushTest, FlushMultipleCFs_DropSomeBeforeRequestFlush) {
|
||||
bool atomic_flush = GetParam();
|
||||
if (!atomic_flush) {
|
||||
return;
|
||||
}
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.atomic_flush = atomic_flush;
|
||||
SyncPoint::GetInstance()->DisableProcessing();
|
||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
CreateAndReopenWithCF({"pikachu", "eevee"}, options);
|
||||
size_t num_cfs = handles_.size();
|
||||
ASSERT_EQ(3, num_cfs);
|
||||
WriteOptions wopts;
|
||||
wopts.disableWAL = true;
|
||||
std::vector<int> cf_ids;
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
int cf_id = static_cast<int>(i);
|
||||
ASSERT_OK(Put(cf_id, "key", "value", wopts));
|
||||
cf_ids.push_back(cf_id);
|
||||
}
|
||||
ASSERT_OK(dbfull()->DropColumnFamily(handles_[1]));
|
||||
ASSERT_TRUE(Flush(cf_ids).IsShutdownInProgress());
|
||||
Destroy(options);
|
||||
}
|
||||
|
||||
TEST_P(DBAtomicFlushTest,
|
||||
FlushMultipleCFs_DropSomeAfterScheduleFlushBeforeFlushJobRun) {
|
||||
bool atomic_flush = GetParam();
|
||||
if (!atomic_flush) {
|
||||
return;
|
||||
}
|
||||
Options options = CurrentOptions();
|
||||
options.create_if_missing = true;
|
||||
options.atomic_flush = atomic_flush;
|
||||
|
||||
CreateAndReopenWithCF({"pikachu", "eevee"}, options);
|
||||
|
||||
SyncPoint::GetInstance()->DisableProcessing();
|
||||
SyncPoint::GetInstance()->ClearAllCallBacks();
|
||||
SyncPoint::GetInstance()->LoadDependency(
|
||||
{{"DBImpl::AtomicFlushMemTables:AfterScheduleFlush",
|
||||
"DBAtomicFlushTest::BeforeDropCF"},
|
||||
{"DBAtomicFlushTest::AfterDropCF",
|
||||
"DBImpl::BackgroundCallFlush:start"}});
|
||||
SyncPoint::GetInstance()->EnableProcessing();
|
||||
|
||||
size_t num_cfs = handles_.size();
|
||||
ASSERT_EQ(3, num_cfs);
|
||||
WriteOptions wopts;
|
||||
wopts.disableWAL = true;
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
int cf_id = static_cast<int>(i);
|
||||
ASSERT_OK(Put(cf_id, "key", "value", wopts));
|
||||
}
|
||||
port::Thread user_thread([&]() {
|
||||
TEST_SYNC_POINT("DBAtomicFlushTest::BeforeDropCF");
|
||||
ASSERT_OK(dbfull()->DropColumnFamily(handles_[1]));
|
||||
TEST_SYNC_POINT("DBAtomicFlushTest::AfterDropCF");
|
||||
});
|
||||
FlushOptions flush_opts;
|
||||
flush_opts.wait = true;
|
||||
ASSERT_OK(dbfull()->Flush(flush_opts, handles_));
|
||||
user_thread.join();
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
int cf_id = static_cast<int>(i);
|
||||
ASSERT_EQ("value", Get(cf_id, "key"));
|
||||
}
|
||||
|
||||
ReopenWithColumnFamilies({kDefaultColumnFamilyName, "eevee"}, options);
|
||||
num_cfs = handles_.size();
|
||||
ASSERT_EQ(2, num_cfs);
|
||||
for (size_t i = 0; i != num_cfs; ++i) {
|
||||
int cf_id = static_cast<int>(i);
|
||||
ASSERT_EQ("value", Get(cf_id, "key"));
|
||||
}
|
||||
Destroy(options);
|
||||
}
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(DBFlushDirectIOTest, DBFlushDirectIOTest,
|
||||
testing::Bool());
|
||||
|
||||
INSTANTIATE_TEST_CASE_P(DBAtomicFlushTest, DBAtomicFlushTest, testing::Bool());
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
|
||||
+458
-1261
File diff suppressed because it is too large
Load Diff
+174
-528
File diff suppressed because it is too large
Load Diff
+360
-1382
File diff suppressed because it is too large
Load Diff
+16
-63
@@ -10,7 +10,6 @@
|
||||
#ifndef NDEBUG
|
||||
|
||||
#include "db/db_impl.h"
|
||||
#include "db/error_handler.h"
|
||||
#include "monitoring/thread_status_updater.h"
|
||||
|
||||
namespace rocksdb {
|
||||
@@ -26,18 +25,6 @@ void DBImpl::TEST_SwitchWAL() {
|
||||
SwitchWAL(&write_context);
|
||||
}
|
||||
|
||||
bool DBImpl::TEST_WALBufferIsEmpty(bool lock) {
|
||||
if (lock) {
|
||||
log_write_mutex_.Lock();
|
||||
}
|
||||
log::Writer* cur_log_writer = logs_.back().writer;
|
||||
auto res = cur_log_writer->TEST_BufferIsEmpty();
|
||||
if (lock) {
|
||||
log_write_mutex_.Unlock();
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
|
||||
ColumnFamilyHandle* column_family) {
|
||||
ColumnFamilyData* cfd;
|
||||
@@ -93,7 +80,7 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
|
||||
cfd->ioptions()->compaction_style == kCompactionStyleFIFO)
|
||||
? level
|
||||
: level + 1;
|
||||
return RunManualCompaction(cfd, level, output_level, 0, 0, begin, end, true,
|
||||
return RunManualCompaction(cfd, level, output_level, 0, begin, end, true,
|
||||
disallow_trivial_move);
|
||||
}
|
||||
|
||||
@@ -106,11 +93,9 @@ Status DBImpl::TEST_SwitchMemtable(ColumnFamilyData* cfd) {
|
||||
return SwitchMemtable(cfd, &write_context);
|
||||
}
|
||||
|
||||
Status DBImpl::TEST_FlushMemTable(bool wait, bool allow_write_stall,
|
||||
ColumnFamilyHandle* cfh) {
|
||||
Status DBImpl::TEST_FlushMemTable(bool wait, ColumnFamilyHandle* cfh) {
|
||||
FlushOptions fo;
|
||||
fo.wait = wait;
|
||||
fo.allow_write_stall = allow_write_stall;
|
||||
ColumnFamilyData* cfd;
|
||||
if (cfh == nullptr) {
|
||||
cfd = default_cf_handle_->cfd();
|
||||
@@ -118,7 +103,7 @@ Status DBImpl::TEST_FlushMemTable(bool wait, bool allow_write_stall,
|
||||
auto cfhi = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh);
|
||||
cfd = cfhi->cfd();
|
||||
}
|
||||
return FlushMemTable(cfd, fo, FlushReason::kTest);
|
||||
return FlushMemTable(cfd, fo);
|
||||
}
|
||||
|
||||
Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
|
||||
@@ -129,10 +114,10 @@ Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
|
||||
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
||||
cfd = cfh->cfd();
|
||||
}
|
||||
return WaitForFlushMemTable(cfd, nullptr, false);
|
||||
return WaitForFlushMemTable(cfd);
|
||||
}
|
||||
|
||||
Status DBImpl::TEST_WaitForCompact(bool wait_unscheduled) {
|
||||
Status DBImpl::TEST_WaitForCompact() {
|
||||
// Wait until the compaction completes
|
||||
|
||||
// TODO: a bug here. This function actually does not necessarily
|
||||
@@ -141,17 +126,20 @@ Status DBImpl::TEST_WaitForCompact(bool wait_unscheduled) {
|
||||
|
||||
InstrumentedMutexLock l(&mutex_);
|
||||
while ((bg_bottom_compaction_scheduled_ || bg_compaction_scheduled_ ||
|
||||
bg_flush_scheduled_ ||
|
||||
(wait_unscheduled && unscheduled_compactions_)) &&
|
||||
(error_handler_.GetBGError() == Status::OK())) {
|
||||
bg_flush_scheduled_) &&
|
||||
bg_error_.ok()) {
|
||||
bg_cv_.Wait();
|
||||
}
|
||||
return error_handler_.GetBGError();
|
||||
return bg_error_;
|
||||
}
|
||||
|
||||
void DBImpl::TEST_LockMutex() { mutex_.Lock(); }
|
||||
void DBImpl::TEST_LockMutex() {
|
||||
mutex_.Lock();
|
||||
}
|
||||
|
||||
void DBImpl::TEST_UnlockMutex() { mutex_.Unlock(); }
|
||||
void DBImpl::TEST_UnlockMutex() {
|
||||
mutex_.Unlock();
|
||||
}
|
||||
|
||||
void* DBImpl::TEST_BeginWrite() {
|
||||
auto w = new WriteThread::Writer();
|
||||
@@ -195,21 +183,11 @@ Status DBImpl::TEST_GetAllImmutableCFOptions(
|
||||
}
|
||||
|
||||
uint64_t DBImpl::TEST_FindMinLogContainingOutstandingPrep() {
|
||||
return logs_with_prep_tracker_.FindMinLogContainingOutstandingPrep();
|
||||
}
|
||||
|
||||
size_t DBImpl::TEST_PreparedSectionCompletedSize() {
|
||||
return logs_with_prep_tracker_.TEST_PreparedSectionCompletedSize();
|
||||
}
|
||||
|
||||
size_t DBImpl::TEST_LogsWithPrepSize() {
|
||||
return logs_with_prep_tracker_.TEST_LogsWithPrepSize();
|
||||
return FindMinLogContainingOutstandingPrep();
|
||||
}
|
||||
|
||||
uint64_t DBImpl::TEST_FindMinPrepLogReferencedByMemTable() {
|
||||
autovector<MemTable*> empty_list;
|
||||
return FindMinPrepLogReferencedByMemTable(versions_.get(), nullptr,
|
||||
empty_list);
|
||||
return FindMinPrepLogReferencedByMemTable();
|
||||
}
|
||||
|
||||
Status DBImpl::TEST_GetLatestMutableCFOptions(
|
||||
@@ -239,30 +217,5 @@ SequenceNumber DBImpl::TEST_GetLastVisibleSequence() const {
|
||||
}
|
||||
}
|
||||
|
||||
size_t DBImpl::TEST_GetWalPreallocateBlockSize(
|
||||
uint64_t write_buffer_size) const {
|
||||
InstrumentedMutexLock l(&mutex_);
|
||||
return GetWalPreallocateBlockSize(write_buffer_size);
|
||||
}
|
||||
|
||||
void DBImpl::TEST_WaitForDumpStatsRun(std::function<void()> callback) const {
|
||||
if (thread_dump_stats_ != nullptr) {
|
||||
thread_dump_stats_->TEST_WaitForRun(callback);
|
||||
}
|
||||
}
|
||||
|
||||
void DBImpl::TEST_WaitForPersistStatsRun(std::function<void()> callback) const {
|
||||
if (thread_persist_stats_ != nullptr) {
|
||||
thread_persist_stats_->TEST_WaitForRun(callback);
|
||||
}
|
||||
}
|
||||
|
||||
bool DBImpl::TEST_IsPersistentStatsEnabled() const {
|
||||
return thread_persist_stats_ && thread_persist_stats_->IsRunning();
|
||||
}
|
||||
|
||||
size_t DBImpl::TEST_EstiamteStatsHistorySize() const {
|
||||
return EstiamteStatsHistorySize();
|
||||
}
|
||||
} // namespace rocksdb
|
||||
#endif // NDEBUG
|
||||
|
||||
@@ -131,16 +131,16 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
|
||||
edit.DeleteFile(0, f->fd.GetNumber());
|
||||
edit.AddFile(target_level, f->fd.GetNumber(), f->fd.GetPathId(),
|
||||
f->fd.GetFileSize(), f->smallest, f->largest,
|
||||
f->fd.smallest_seqno, f->fd.largest_seqno,
|
||||
f->smallest_seqno, f->largest_seqno,
|
||||
f->marked_for_compaction);
|
||||
}
|
||||
|
||||
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
|
||||
&edit, &mutex_, directories_.GetDbDir());
|
||||
if (status.ok()) {
|
||||
InstallSuperVersionAndScheduleWork(cfd,
|
||||
&job_context.superversion_contexts[0],
|
||||
*cfd->GetLatestMutableCFOptions());
|
||||
InstallSuperVersionAndScheduleWork(
|
||||
cfd, &job_context.superversion_context,
|
||||
*cfd->GetLatestMutableCFOptions());
|
||||
}
|
||||
} // lock released here
|
||||
LogFlush(immutable_db_options_.info_log);
|
||||
|
||||
+142
-241
@@ -12,29 +12,122 @@
|
||||
#define __STDC_FORMAT_MACROS
|
||||
#endif
|
||||
#include <inttypes.h>
|
||||
#include <set>
|
||||
#include <unordered_set>
|
||||
#include "db/event_helpers.h"
|
||||
#include "db/memtable_list.h"
|
||||
#include "util/file_util.h"
|
||||
#include "util/sst_file_manager_impl.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
uint64_t DBImpl::MinLogNumberToKeep() {
|
||||
if (allow_2pc()) {
|
||||
return versions_->min_log_number_to_keep_2pc();
|
||||
} else {
|
||||
return versions_->MinLogNumberWithUnflushedData();
|
||||
namespace rocksdb {
|
||||
uint64_t DBImpl::FindMinPrepLogReferencedByMemTable() {
|
||||
if (!allow_2pc()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t min_log = 0;
|
||||
|
||||
// we must look through the memtables for two phase transactions
|
||||
// that have been committed but not yet flushed
|
||||
for (auto loop_cfd : *versions_->GetColumnFamilySet()) {
|
||||
if (loop_cfd->IsDropped()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto log = loop_cfd->imm()->GetMinLogContainingPrepSection();
|
||||
|
||||
if (log > 0 && (min_log == 0 || log < min_log)) {
|
||||
min_log = log;
|
||||
}
|
||||
|
||||
log = loop_cfd->mem()->GetMinLogContainingPrepSection();
|
||||
|
||||
if (log > 0 && (min_log == 0 || log < min_log)) {
|
||||
min_log = log;
|
||||
}
|
||||
}
|
||||
|
||||
return min_log;
|
||||
}
|
||||
|
||||
// TODO(myabandeh): Avoid using locks
|
||||
void DBImpl::MarkLogAsHavingPrepSectionFlushed(uint64_t log) {
|
||||
assert(log != 0);
|
||||
std::lock_guard<std::mutex> lock(prep_heap_mutex_);
|
||||
auto it = prepared_section_completed_.find(log);
|
||||
assert(it != prepared_section_completed_.end());
|
||||
it->second += 1;
|
||||
}
|
||||
|
||||
// TODO(myabandeh): Avoid using locks
|
||||
void DBImpl::MarkLogAsContainingPrepSection(uint64_t log) {
|
||||
assert(log != 0);
|
||||
std::lock_guard<std::mutex> lock(prep_heap_mutex_);
|
||||
min_log_with_prep_.push(log);
|
||||
auto it = prepared_section_completed_.find(log);
|
||||
if (it == prepared_section_completed_.end()) {
|
||||
prepared_section_completed_[log] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
uint64_t DBImpl::MinObsoleteSstNumberToKeep() {
|
||||
mutex_.AssertHeld();
|
||||
if (!pending_outputs_.empty()) {
|
||||
return *pending_outputs_.begin();
|
||||
uint64_t DBImpl::FindMinLogContainingOutstandingPrep() {
|
||||
|
||||
if (!allow_2pc()) {
|
||||
return 0;
|
||||
}
|
||||
return std::numeric_limits<uint64_t>::max();
|
||||
|
||||
std::lock_guard<std::mutex> lock(prep_heap_mutex_);
|
||||
uint64_t min_log = 0;
|
||||
|
||||
// first we look in the prepared heap where we keep
|
||||
// track of transactions that have been prepared (written to WAL)
|
||||
// but not yet committed.
|
||||
while (!min_log_with_prep_.empty()) {
|
||||
min_log = min_log_with_prep_.top();
|
||||
|
||||
auto it = prepared_section_completed_.find(min_log);
|
||||
|
||||
// value was marked as 'deleted' from heap
|
||||
if (it != prepared_section_completed_.end() && it->second > 0) {
|
||||
it->second -= 1;
|
||||
min_log_with_prep_.pop();
|
||||
|
||||
// back to squere one...
|
||||
min_log = 0;
|
||||
continue;
|
||||
} else {
|
||||
// found a valid value
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return min_log;
|
||||
}
|
||||
|
||||
uint64_t DBImpl::MinLogNumberToKeep() {
|
||||
uint64_t log_number = versions_->MinLogNumber();
|
||||
|
||||
if (allow_2pc()) {
|
||||
// if are 2pc we must consider logs containing prepared
|
||||
// sections of outstanding transactions.
|
||||
//
|
||||
// We must check min logs with outstanding prep before we check
|
||||
// logs referneces by memtables because a log referenced by the
|
||||
// first data structure could transition to the second under us.
|
||||
//
|
||||
// TODO(horuff): iterating over all column families under db mutex.
|
||||
// should find more optimial solution
|
||||
auto min_log_in_prep_heap = FindMinLogContainingOutstandingPrep();
|
||||
|
||||
if (min_log_in_prep_heap != 0 && min_log_in_prep_heap < log_number) {
|
||||
log_number = min_log_in_prep_heap;
|
||||
}
|
||||
|
||||
auto min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable();
|
||||
|
||||
if (min_log_refed_by_mem != 0 && min_log_refed_by_mem < log_number) {
|
||||
log_number = min_log_refed_by_mem;
|
||||
}
|
||||
}
|
||||
return log_number;
|
||||
}
|
||||
|
||||
// * Returns the list of live files in 'sst_live'
|
||||
@@ -57,7 +150,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||
|
||||
bool doing_the_full_scan = false;
|
||||
|
||||
// logic for figuring out if we're doing the full scan
|
||||
// logic for figurint out if we're doing the full scan
|
||||
if (no_full_scan) {
|
||||
doing_the_full_scan = false;
|
||||
} else if (force ||
|
||||
@@ -77,7 +170,7 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||
// threads
|
||||
// Since job_context->min_pending_output is set, until file scan finishes,
|
||||
// mutex_ cannot be released. Otherwise, we might see no min_pending_output
|
||||
// here but later find newer generated unfinalized files while scanning.
|
||||
// here but later find newer generated unfinalized files while scannint.
|
||||
if (!pending_outputs_.empty()) {
|
||||
job_context->min_pending_output = *pending_outputs_.begin();
|
||||
} else {
|
||||
@@ -91,68 +184,27 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||
&job_context->manifest_delete_files,
|
||||
job_context->min_pending_output);
|
||||
|
||||
// Mark the elements in job_context->sst_delete_files as grabbedForPurge
|
||||
// so that other threads calling FindObsoleteFiles with full_scan=true
|
||||
// will not add these files to candidate list for purge.
|
||||
for (const auto& sst_to_del : job_context->sst_delete_files) {
|
||||
MarkAsGrabbedForPurge(sst_to_del.metadata->fd.GetNumber());
|
||||
}
|
||||
|
||||
// store the current filenum, lognum, etc
|
||||
job_context->manifest_file_number = versions_->manifest_file_number();
|
||||
job_context->pending_manifest_file_number =
|
||||
versions_->pending_manifest_file_number();
|
||||
job_context->log_number = MinLogNumberToKeep();
|
||||
|
||||
job_context->prev_log_number = versions_->prev_log_number();
|
||||
|
||||
versions_->AddLiveFiles(&job_context->sst_live);
|
||||
if (doing_the_full_scan) {
|
||||
InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
|
||||
dbname_);
|
||||
std::set<std::string> paths;
|
||||
for (size_t path_id = 0; path_id < immutable_db_options_.db_paths.size();
|
||||
path_id++) {
|
||||
paths.insert(immutable_db_options_.db_paths[path_id].path);
|
||||
}
|
||||
|
||||
// Note that if cf_paths is not specified in the ColumnFamilyOptions
|
||||
// of a particular column family, we use db_paths as the cf_paths
|
||||
// setting. Hence, there can be multiple duplicates of files from db_paths
|
||||
// in the following code. The duplicate are removed while identifying
|
||||
// unique files in PurgeObsoleteFiles.
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
for (size_t path_id = 0; path_id < cfd->ioptions()->cf_paths.size();
|
||||
path_id++) {
|
||||
auto& path = cfd->ioptions()->cf_paths[path_id].path;
|
||||
|
||||
if (paths.find(path) == paths.end()) {
|
||||
paths.insert(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (auto& path : paths) {
|
||||
// set of all files in the directory. We'll exclude files that are still
|
||||
// alive in the subsequent processings.
|
||||
std::vector<std::string> files;
|
||||
env_->GetChildren(path, &files); // Ignore errors
|
||||
for (const std::string& file : files) {
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
// 1. If we cannot parse the file name, we skip;
|
||||
// 2. If the file with file_number equals number has already been
|
||||
// grabbed for purge by another compaction job, or it has already been
|
||||
// schedule for purge, we also skip it if we
|
||||
// are doing full scan in order to avoid double deletion of the same
|
||||
// file under race conditions. See
|
||||
// https://github.com/facebook/rocksdb/issues/3573
|
||||
if (!ParseFileName(file, &number, info_log_prefix.prefix, &type) ||
|
||||
!ShouldPurge(number)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
env_->GetChildren(immutable_db_options_.db_paths[path_id].path,
|
||||
&files); // Ignore errors
|
||||
for (std::string file : files) {
|
||||
// TODO(icanadi) clean up this mess to avoid having one-off "/" prefixes
|
||||
job_context->full_scan_candidate_files.emplace_back("/" + file, path);
|
||||
job_context->full_scan_candidate_files.emplace_back(
|
||||
"/" + file, static_cast<uint32_t>(path_id));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -161,9 +213,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||
std::vector<std::string> log_files;
|
||||
env_->GetChildren(immutable_db_options_.wal_dir,
|
||||
&log_files); // Ignore errors
|
||||
for (const std::string& log_file : log_files) {
|
||||
job_context->full_scan_candidate_files.emplace_back(
|
||||
log_file, immutable_db_options_.wal_dir);
|
||||
for (std::string log_file : log_files) {
|
||||
job_context->full_scan_candidate_files.emplace_back(log_file, 0);
|
||||
}
|
||||
}
|
||||
// Add info log files in db_log_dir
|
||||
@@ -172,9 +223,8 @@ void DBImpl::FindObsoleteFiles(JobContext* job_context, bool force,
|
||||
std::vector<std::string> info_log_files;
|
||||
// Ignore errors
|
||||
env_->GetChildren(immutable_db_options_.db_log_dir, &info_log_files);
|
||||
for (std::string& log_file : info_log_files) {
|
||||
job_context->full_scan_candidate_files.emplace_back(
|
||||
log_file, immutable_db_options_.db_log_dir);
|
||||
for (std::string log_file : info_log_files) {
|
||||
job_context->full_scan_candidate_files.emplace_back(log_file, 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -249,24 +299,22 @@ bool CompareCandidateFile(const JobContext::CandidateFileInfo& first,
|
||||
} else if (first.file_name < second.file_name) {
|
||||
return false;
|
||||
} else {
|
||||
return (first.file_path > second.file_path);
|
||||
return (first.path_id > second.path_id);
|
||||
}
|
||||
}
|
||||
}; // namespace
|
||||
|
||||
// Delete obsolete files and log status and information of file deletion
|
||||
void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
|
||||
const std::string& path_to_sync,
|
||||
FileType type, uint64_t number) {
|
||||
FileType type, uint64_t number,
|
||||
uint32_t path_id) {
|
||||
Status file_deletion_status;
|
||||
if (type == kTableFile || type == kLogFile) {
|
||||
if (type == kTableFile) {
|
||||
file_deletion_status =
|
||||
DeleteDBFile(&immutable_db_options_, fname, path_to_sync);
|
||||
DeleteSSTFile(&immutable_db_options_, fname, path_id);
|
||||
} else {
|
||||
file_deletion_status = env_->DeleteFile(fname);
|
||||
}
|
||||
TEST_SYNC_POINT_CALLBACK("DBImpl::DeleteObsoleteFileImpl:AfterDeletion",
|
||||
&file_deletion_status);
|
||||
if (file_deletion_status.ok()) {
|
||||
ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
|
||||
"[JOB %d] Delete %s type=%d #%" PRIu64 " -- %s\n", job_id,
|
||||
@@ -293,10 +341,10 @@ void DBImpl::DeleteObsoleteFileImpl(int job_id, const std::string& fname,
|
||||
}
|
||||
|
||||
// Diffs the files listed in filenames and those that do not
|
||||
// belong to live files are possibly removed. Also, removes all the
|
||||
// belong to live files are posibly removed. Also, removes all the
|
||||
// files in sst_delete_files and log_delete_files.
|
||||
// It is not necessary to hold the mutex when invoking this method.
|
||||
void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
void DBImpl::PurgeObsoleteFiles(const JobContext& state, bool schedule_only) {
|
||||
TEST_SYNC_POINT("DBImpl::PurgeObsoleteFiles:Begin");
|
||||
// we'd better have sth to delete
|
||||
assert(state.HaveSomethingToDelete());
|
||||
@@ -319,24 +367,23 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
state.log_delete_files.size() + state.manifest_delete_files.size());
|
||||
// We may ignore the dbname when generating the file names.
|
||||
const char* kDumbDbName = "";
|
||||
for (auto& file : state.sst_delete_files) {
|
||||
for (auto file : state.sst_delete_files) {
|
||||
candidate_files.emplace_back(
|
||||
MakeTableFileName(kDumbDbName, file.metadata->fd.GetNumber()),
|
||||
file.path);
|
||||
if (file.metadata->table_reader_handle) {
|
||||
table_cache_->Release(file.metadata->table_reader_handle);
|
||||
MakeTableFileName(kDumbDbName, file->fd.GetNumber()),
|
||||
file->fd.GetPathId());
|
||||
if (file->table_reader_handle) {
|
||||
table_cache_->Release(file->table_reader_handle);
|
||||
}
|
||||
file.DeleteMetadata();
|
||||
delete file;
|
||||
}
|
||||
|
||||
for (auto file_num : state.log_delete_files) {
|
||||
if (file_num > 0) {
|
||||
candidate_files.emplace_back(LogFileName(kDumbDbName, file_num),
|
||||
immutable_db_options_.wal_dir);
|
||||
candidate_files.emplace_back(LogFileName(kDumbDbName, file_num), 0);
|
||||
}
|
||||
}
|
||||
for (const auto& filename : state.manifest_delete_files) {
|
||||
candidate_files.emplace_back(filename, dbname_);
|
||||
candidate_files.emplace_back(filename, 0);
|
||||
}
|
||||
|
||||
// dedup state.candidate_files so we don't try to delete the same
|
||||
@@ -359,32 +406,9 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
std::vector<std::string> old_info_log_files;
|
||||
InfoLogPrefix info_log_prefix(!immutable_db_options_.db_log_dir.empty(),
|
||||
dbname_);
|
||||
|
||||
// File numbers of most recent two OPTIONS file in candidate_files (found in
|
||||
// previos FindObsoleteFiles(full_scan=true))
|
||||
// At this point, there must not be any duplicate file numbers in
|
||||
// candidate_files.
|
||||
uint64_t optsfile_num1 = std::numeric_limits<uint64_t>::min();
|
||||
uint64_t optsfile_num2 = std::numeric_limits<uint64_t>::min();
|
||||
for (const auto& candidate_file : candidate_files) {
|
||||
const std::string& fname = candidate_file.file_name;
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
if (!ParseFileName(fname, &number, info_log_prefix.prefix, &type) ||
|
||||
type != kOptionsFile) {
|
||||
continue;
|
||||
}
|
||||
if (number > optsfile_num1) {
|
||||
optsfile_num2 = optsfile_num1;
|
||||
optsfile_num1 = number;
|
||||
} else if (number > optsfile_num2) {
|
||||
optsfile_num2 = number;
|
||||
}
|
||||
}
|
||||
|
||||
std::unordered_set<uint64_t> files_to_del;
|
||||
for (const auto& candidate_file : candidate_files) {
|
||||
const std::string& to_delete = candidate_file.file_name;
|
||||
std::string to_delete = candidate_file.file_name;
|
||||
uint32_t path_id = candidate_file.path_id;
|
||||
uint64_t number;
|
||||
FileType type;
|
||||
// Ignore file if we cannot recognize it.
|
||||
@@ -410,9 +434,6 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
// DontDeletePendingOutputs fail
|
||||
keep = (sst_live_map.find(number) != sst_live_map.end()) ||
|
||||
number >= state.min_pending_output;
|
||||
if (!keep) {
|
||||
files_to_del.insert(number);
|
||||
}
|
||||
break;
|
||||
case kTempFile:
|
||||
// Any temp files that are currently being written to must
|
||||
@@ -433,19 +454,11 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
old_info_log_files.push_back(to_delete);
|
||||
}
|
||||
break;
|
||||
case kOptionsFile:
|
||||
keep = (number >= optsfile_num2);
|
||||
TEST_SYNC_POINT_CALLBACK(
|
||||
"DBImpl::PurgeObsoleteFiles:CheckOptionsFiles:1",
|
||||
reinterpret_cast<void*>(&number));
|
||||
TEST_SYNC_POINT_CALLBACK(
|
||||
"DBImpl::PurgeObsoleteFiles:CheckOptionsFiles:2",
|
||||
reinterpret_cast<void*>(&keep));
|
||||
break;
|
||||
case kCurrentFile:
|
||||
case kDBLockFile:
|
||||
case kIdentityFile:
|
||||
case kMetaDatabase:
|
||||
case kOptionsFile:
|
||||
case kBlobFile:
|
||||
keep = true;
|
||||
break;
|
||||
@@ -456,21 +469,13 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
}
|
||||
|
||||
std::string fname;
|
||||
std::string dir_to_sync;
|
||||
if (type == kTableFile) {
|
||||
// evict from cache
|
||||
TableCache::Evict(table_cache_.get(), number);
|
||||
fname = MakeTableFileName(candidate_file.file_path, number);
|
||||
dir_to_sync = candidate_file.file_path;
|
||||
fname = TableFileName(immutable_db_options_.db_paths, number, path_id);
|
||||
} else {
|
||||
dir_to_sync =
|
||||
(type == kLogFile) ? immutable_db_options_.wal_dir : dbname_;
|
||||
fname = dir_to_sync +
|
||||
((!dir_to_sync.empty() && dir_to_sync.back() == '/') ||
|
||||
(!to_delete.empty() && to_delete.front() == '/')
|
||||
? ""
|
||||
: "/") +
|
||||
to_delete;
|
||||
fname = ((type == kLogFile) ? immutable_db_options_.wal_dir : dbname_) +
|
||||
"/" + to_delete;
|
||||
}
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
@@ -484,25 +489,12 @@ void DBImpl::PurgeObsoleteFiles(JobContext& state, bool schedule_only) {
|
||||
Status file_deletion_status;
|
||||
if (schedule_only) {
|
||||
InstrumentedMutexLock guard_lock(&mutex_);
|
||||
SchedulePendingPurge(fname, dir_to_sync, type, number, state.job_id);
|
||||
SchedulePendingPurge(fname, type, number, path_id, state.job_id);
|
||||
} else {
|
||||
DeleteObsoleteFileImpl(state.job_id, fname, dir_to_sync, type, number);
|
||||
DeleteObsoleteFileImpl(state.job_id, fname, type, number, path_id);
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
// After purging obsolete files, remove them from files_grabbed_for_purge_.
|
||||
// Use a temporary vector to perform bulk deletion via swap.
|
||||
InstrumentedMutexLock guard_lock(&mutex_);
|
||||
std::vector<uint64_t> tmp;
|
||||
for (auto fn : files_grabbed_for_purge_) {
|
||||
if (files_to_del.count(fn) == 0) {
|
||||
tmp.emplace_back(fn);
|
||||
}
|
||||
}
|
||||
files_grabbed_for_purge_.swap(tmp);
|
||||
}
|
||||
|
||||
// Delete old info log files.
|
||||
size_t old_info_log_file_count = old_info_log_files.size();
|
||||
if (old_info_log_file_count != 0 &&
|
||||
@@ -562,95 +554,4 @@ void DBImpl::DeleteObsoleteFiles() {
|
||||
job_context.Clean();
|
||||
mutex_.Lock();
|
||||
}
|
||||
|
||||
uint64_t FindMinPrepLogReferencedByMemTable(
|
||||
VersionSet* vset, const ColumnFamilyData* cfd_to_flush,
|
||||
const autovector<MemTable*>& memtables_to_flush) {
|
||||
uint64_t min_log = 0;
|
||||
|
||||
// we must look through the memtables for two phase transactions
|
||||
// that have been committed but not yet flushed
|
||||
for (auto loop_cfd : *vset->GetColumnFamilySet()) {
|
||||
if (loop_cfd->IsDropped() || loop_cfd == cfd_to_flush) {
|
||||
continue;
|
||||
}
|
||||
|
||||
auto log = loop_cfd->imm()->PrecomputeMinLogContainingPrepSection(
|
||||
memtables_to_flush);
|
||||
|
||||
if (log > 0 && (min_log == 0 || log < min_log)) {
|
||||
min_log = log;
|
||||
}
|
||||
|
||||
log = loop_cfd->mem()->GetMinLogContainingPrepSection();
|
||||
|
||||
if (log > 0 && (min_log == 0 || log < min_log)) {
|
||||
min_log = log;
|
||||
}
|
||||
}
|
||||
|
||||
return min_log;
|
||||
}
|
||||
|
||||
uint64_t PrecomputeMinLogNumberToKeep(
|
||||
VersionSet* vset, const ColumnFamilyData& cfd_to_flush,
|
||||
autovector<VersionEdit*> edit_list,
|
||||
const autovector<MemTable*>& memtables_to_flush,
|
||||
LogsWithPrepTracker* prep_tracker) {
|
||||
assert(vset != nullptr);
|
||||
assert(prep_tracker != nullptr);
|
||||
// Calculate updated min_log_number_to_keep
|
||||
// Since the function should only be called in 2pc mode, log number in
|
||||
// the version edit should be sufficient.
|
||||
|
||||
// Precompute the min log number containing unflushed data for the column
|
||||
// family being flushed (`cfd_to_flush`).
|
||||
uint64_t cf_min_log_number_to_keep = 0;
|
||||
for (auto& e : edit_list) {
|
||||
if (e->has_log_number()) {
|
||||
cf_min_log_number_to_keep =
|
||||
std::max(cf_min_log_number_to_keep, e->log_number());
|
||||
}
|
||||
}
|
||||
if (cf_min_log_number_to_keep == 0) {
|
||||
// No version edit contains information on log number. The log number
|
||||
// for this column family should stay the same as it is.
|
||||
cf_min_log_number_to_keep = cfd_to_flush.GetLogNumber();
|
||||
}
|
||||
|
||||
// Get min log number containing unflushed data for other column families.
|
||||
uint64_t min_log_number_to_keep =
|
||||
vset->PreComputeMinLogNumberWithUnflushedData(&cfd_to_flush);
|
||||
if (cf_min_log_number_to_keep != 0) {
|
||||
min_log_number_to_keep =
|
||||
std::min(cf_min_log_number_to_keep, min_log_number_to_keep);
|
||||
}
|
||||
|
||||
// if are 2pc we must consider logs containing prepared
|
||||
// sections of outstanding transactions.
|
||||
//
|
||||
// We must check min logs with outstanding prep before we check
|
||||
// logs references by memtables because a log referenced by the
|
||||
// first data structure could transition to the second under us.
|
||||
//
|
||||
// TODO: iterating over all column families under db mutex.
|
||||
// should find more optimal solution
|
||||
auto min_log_in_prep_heap =
|
||||
prep_tracker->FindMinLogContainingOutstandingPrep();
|
||||
|
||||
if (min_log_in_prep_heap != 0 &&
|
||||
min_log_in_prep_heap < min_log_number_to_keep) {
|
||||
min_log_number_to_keep = min_log_in_prep_heap;
|
||||
}
|
||||
|
||||
uint64_t min_log_refed_by_mem = FindMinPrepLogReferencedByMemTable(
|
||||
vset, &cfd_to_flush, memtables_to_flush);
|
||||
|
||||
if (min_log_refed_by_mem != 0 &&
|
||||
min_log_refed_by_mem < min_log_number_to_keep) {
|
||||
min_log_number_to_keep = min_log_refed_by_mem;
|
||||
}
|
||||
return min_log_number_to_keep;
|
||||
}
|
||||
|
||||
} // namespace rocksdb
|
||||
|
||||
+102
-264
@@ -14,7 +14,6 @@
|
||||
#include <inttypes.h>
|
||||
|
||||
#include "db/builder.h"
|
||||
#include "db/error_handler.h"
|
||||
#include "options/options_helper.h"
|
||||
#include "rocksdb/wal_filter.h"
|
||||
#include "table/block_based_table_factory.h"
|
||||
@@ -23,7 +22,8 @@
|
||||
#include "util/sync_point.h"
|
||||
|
||||
namespace rocksdb {
|
||||
Options SanitizeOptions(const std::string& dbname, const Options& src) {
|
||||
Options SanitizeOptions(const std::string& dbname,
|
||||
const Options& src) {
|
||||
auto db_options = SanitizeOptions(dbname, DBOptions(src));
|
||||
ImmutableDBOptions immutable_db_options(db_options);
|
||||
auto cf_options =
|
||||
@@ -41,8 +41,6 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
|
||||
max_max_open_files = 0x400000;
|
||||
}
|
||||
ClipToRange(&result.max_open_files, 20, max_max_open_files);
|
||||
TEST_SYNC_POINT_CALLBACK("SanitizeOptions::AfterChangeMaxOpenFiles",
|
||||
&result.max_open_files);
|
||||
}
|
||||
|
||||
if (result.info_log == nullptr) {
|
||||
@@ -57,9 +55,10 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
|
||||
result.write_buffer_manager.reset(
|
||||
new WriteBufferManager(result.db_write_buffer_size));
|
||||
}
|
||||
auto bg_job_limits = DBImpl::GetBGJobLimits(
|
||||
result.max_background_flushes, result.max_background_compactions,
|
||||
result.max_background_jobs, true /* parallelize_compactions */);
|
||||
auto bg_job_limits = DBImpl::GetBGJobLimits(result.max_background_flushes,
|
||||
result.max_background_compactions,
|
||||
result.max_background_jobs,
|
||||
true /* parallelize_compactions */);
|
||||
result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_compactions,
|
||||
Env::Priority::LOW);
|
||||
result.env->IncBackgroundThreadsIfNeeded(bg_job_limits.max_flushes,
|
||||
@@ -107,12 +106,14 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
|
||||
result.db_paths.emplace_back(dbname, std::numeric_limits<uint64_t>::max());
|
||||
}
|
||||
|
||||
if (result.use_direct_reads && result.compaction_readahead_size == 0) {
|
||||
if (result.use_direct_io_for_flush_and_compaction &&
|
||||
result.compaction_readahead_size == 0) {
|
||||
TEST_SYNC_POINT_CALLBACK("SanitizeOptions:direct_io", nullptr);
|
||||
result.compaction_readahead_size = 1024 * 1024 * 2;
|
||||
}
|
||||
|
||||
if (result.compaction_readahead_size > 0 || result.use_direct_reads) {
|
||||
if (result.compaction_readahead_size > 0 ||
|
||||
result.use_direct_io_for_flush_and_compaction) {
|
||||
result.new_table_reader_for_compaction_inputs = true;
|
||||
}
|
||||
|
||||
@@ -132,15 +133,8 @@ DBOptions SanitizeOptions(const std::string& dbname, const DBOptions& src) {
|
||||
for (size_t i = 0; i < result.db_paths.size(); i++) {
|
||||
DeleteScheduler::CleanupDirectory(result.env, sfm, result.db_paths[i].path);
|
||||
}
|
||||
|
||||
// Create a default SstFileManager for purposes of tracking compaction size
|
||||
// and facilitating recovery from out of space errors.
|
||||
if (result.sst_file_manager.get() == nullptr) {
|
||||
std::shared_ptr<SstFileManager> sst_file_manager(
|
||||
NewSstFileManager(result.env, result.info_log));
|
||||
result.sst_file_manager = sst_file_manager;
|
||||
}
|
||||
#endif
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -169,23 +163,28 @@ static Status ValidateOptions(
|
||||
if (s.ok() && db_options.allow_concurrent_memtable_write) {
|
||||
s = CheckConcurrentWritesSupported(cfd.options);
|
||||
}
|
||||
if (s.ok()) {
|
||||
s = CheckCFPathsSupported(db_options, cfd.options);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
if (cfd.options.ttl > 0) {
|
||||
if (db_options.db_paths.size() > 1) {
|
||||
if ((cfd.options.compaction_style != kCompactionStyleUniversal) &&
|
||||
(cfd.options.compaction_style != kCompactionStyleLevel)) {
|
||||
return Status::NotSupported(
|
||||
"More than one DB paths are only supported in "
|
||||
"universal and level compaction styles. ");
|
||||
}
|
||||
}
|
||||
if (cfd.options.compaction_options_fifo.ttl > 0) {
|
||||
if (db_options.max_open_files != -1) {
|
||||
return Status::NotSupported(
|
||||
"TTL is only supported when files are always "
|
||||
"FIFO Compaction with TTL is only supported when files are always "
|
||||
"kept open (set max_open_files = -1). ");
|
||||
}
|
||||
if (cfd.options.table_factory->Name() !=
|
||||
BlockBasedTableFactory().Name()) {
|
||||
return Status::NotSupported(
|
||||
"TTL is only supported in Block-Based Table format. ");
|
||||
"FIFO Compaction with TTL is only supported in "
|
||||
"Block-Based Table format. ");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -216,7 +215,7 @@ static Status ValidateOptions(
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
} // namespace
|
||||
} // namespace
|
||||
Status DBImpl::NewDB() {
|
||||
VersionEdit new_db;
|
||||
new_db.SetLogNumber(0);
|
||||
@@ -228,7 +227,7 @@ Status DBImpl::NewDB() {
|
||||
ROCKS_LOG_INFO(immutable_db_options_.info_log, "Creating manifest 1 \n");
|
||||
const std::string manifest = DescriptorFileName(dbname_, 1);
|
||||
{
|
||||
std::unique_ptr<WritableFile> file;
|
||||
unique_ptr<WritableFile> file;
|
||||
EnvOptions env_options = env_->OptimizeForManifestWrite(env_options_);
|
||||
s = NewWritableFile(env_, manifest, &file, env_options);
|
||||
if (!s.ok()) {
|
||||
@@ -236,9 +235,8 @@ Status DBImpl::NewDB() {
|
||||
}
|
||||
file->SetPreallocationBlockSize(
|
||||
immutable_db_options_.manifest_preallocation_size);
|
||||
std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
|
||||
std::move(file), manifest, env_options, env_, nullptr /* stats */,
|
||||
immutable_db_options_.listeners));
|
||||
unique_ptr<WritableFileWriter> file_writer(
|
||||
new WritableFileWriter(std::move(file), env_options));
|
||||
log::Writer log(std::move(file_writer), 0, false);
|
||||
std::string record;
|
||||
new_db.EncodeTo(&record);
|
||||
@@ -256,8 +254,9 @@ Status DBImpl::NewDB() {
|
||||
return s;
|
||||
}
|
||||
|
||||
Status DBImpl::CreateAndNewDirectory(Env* env, const std::string& dirname,
|
||||
std::unique_ptr<Directory>* directory) {
|
||||
Status DBImpl::Directories::CreateAndNewDirectory(
|
||||
Env* env, const std::string& dirname,
|
||||
std::unique_ptr<Directory>* directory) const {
|
||||
// We call CreateDirIfMissing() as the directory may already exist (if we
|
||||
// are reopening a DB), when this happens we don't want creating the
|
||||
// directory to cause an error. However, we need to check if creating the
|
||||
@@ -275,12 +274,12 @@ Status DBImpl::CreateAndNewDirectory(Env* env, const std::string& dirname,
|
||||
Status DBImpl::Directories::SetDirectories(
|
||||
Env* env, const std::string& dbname, const std::string& wal_dir,
|
||||
const std::vector<DbPath>& data_paths) {
|
||||
Status s = DBImpl::CreateAndNewDirectory(env, dbname, &db_dir_);
|
||||
Status s = CreateAndNewDirectory(env, dbname, &db_dir_);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
if (!wal_dir.empty() && dbname != wal_dir) {
|
||||
s = DBImpl::CreateAndNewDirectory(env, wal_dir, &wal_dir_);
|
||||
s = CreateAndNewDirectory(env, wal_dir, &wal_dir_);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
@@ -293,7 +292,7 @@ Status DBImpl::Directories::SetDirectories(
|
||||
data_dirs_.emplace_back(nullptr);
|
||||
} else {
|
||||
std::unique_ptr<Directory> path_directory;
|
||||
s = DBImpl::CreateAndNewDirectory(env, db_path, &path_directory);
|
||||
s = CreateAndNewDirectory(env, db_path, &path_directory);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
@@ -338,8 +337,8 @@ Status DBImpl::Recover(
|
||||
}
|
||||
} else if (s.ok()) {
|
||||
if (immutable_db_options_.error_if_exists) {
|
||||
return Status::InvalidArgument(dbname_,
|
||||
"exists (error_if_exists is true)");
|
||||
return Status::InvalidArgument(
|
||||
dbname_, "exists (error_if_exists is true)");
|
||||
}
|
||||
} else {
|
||||
// Unexpected error reading file
|
||||
@@ -357,53 +356,12 @@ Status DBImpl::Recover(
|
||||
assert(s.IsIOError());
|
||||
return s;
|
||||
}
|
||||
// Verify compatibility of env_options_ and filesystem
|
||||
{
|
||||
std::unique_ptr<RandomAccessFile> idfile;
|
||||
EnvOptions customized_env(env_options_);
|
||||
customized_env.use_direct_reads |=
|
||||
immutable_db_options_.use_direct_io_for_flush_and_compaction;
|
||||
s = env_->NewRandomAccessFile(IdentityFileName(dbname_), &idfile,
|
||||
customized_env);
|
||||
if (!s.ok()) {
|
||||
std::string error_str = s.ToString();
|
||||
// Check if unsupported Direct I/O is the root cause
|
||||
customized_env.use_direct_reads = false;
|
||||
s = env_->NewRandomAccessFile(IdentityFileName(dbname_), &idfile,
|
||||
customized_env);
|
||||
if (s.ok()) {
|
||||
return Status::InvalidArgument(
|
||||
"Direct I/O is not supported by the specified DB.");
|
||||
} else {
|
||||
return Status::InvalidArgument(
|
||||
"Found options incompatible with filesystem", error_str.c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Status s = versions_->Recover(column_families, read_only);
|
||||
if (immutable_db_options_.paranoid_checks && s.ok()) {
|
||||
s = CheckConsistency();
|
||||
}
|
||||
if (s.ok() && !read_only) {
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
s = cfd->AddDirectories();
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Initial max_total_in_memory_state_ before recovery logs. Log recovery
|
||||
// may check this value to decide whether to flush.
|
||||
max_total_in_memory_state_ = 0;
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
|
||||
max_total_in_memory_state_ += mutable_cf_options->write_buffer_size *
|
||||
mutable_cf_options->max_write_buffer_number;
|
||||
}
|
||||
|
||||
if (s.ok()) {
|
||||
SequenceNumber next_sequence(kMaxSequenceNumber);
|
||||
default_cf_handle_ = new ColumnFamilyHandleImpl(
|
||||
@@ -421,10 +379,7 @@ Status DBImpl::Recover(
|
||||
// produced by an older version of rocksdb.
|
||||
std::vector<std::string> filenames;
|
||||
s = env_->GetChildren(immutable_db_options_.wal_dir, &filenames);
|
||||
if (s.IsNotFound()) {
|
||||
return Status::InvalidArgument("wal_dir not found",
|
||||
immutable_db_options_.wal_dir);
|
||||
} else if (!s.ok()) {
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
|
||||
@@ -479,26 +434,12 @@ Status DBImpl::Recover(
|
||||
}
|
||||
}
|
||||
|
||||
if (read_only) {
|
||||
// If we are opening as read-only, we need to update options_file_number_
|
||||
// to reflect the most recent OPTIONS file. It does not matter for regular
|
||||
// read-write db instance because options_file_number_ will later be
|
||||
// updated to versions_->NewFileNumber() in RenameTempFileToOptionsFile.
|
||||
std::vector<std::string> file_names;
|
||||
if (s.ok()) {
|
||||
s = env_->GetChildren(GetName(), &file_names);
|
||||
}
|
||||
if (s.ok()) {
|
||||
uint64_t number = 0;
|
||||
uint64_t options_file_number = 0;
|
||||
FileType type;
|
||||
for (const auto& fname : file_names) {
|
||||
if (ParseFileName(fname, &number, &type) && type == kOptionsFile) {
|
||||
options_file_number = std::max(number, options_file_number);
|
||||
}
|
||||
}
|
||||
versions_->options_file_number_ = options_file_number;
|
||||
}
|
||||
// Initial value
|
||||
max_total_in_memory_state_ = 0;
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
|
||||
max_total_in_memory_state_ += mutable_cf_options->write_buffer_size *
|
||||
mutable_cf_options->max_write_buffer_number;
|
||||
}
|
||||
|
||||
return s;
|
||||
@@ -512,7 +453,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
Logger* info_log;
|
||||
const char* fname;
|
||||
Status* status; // nullptr if immutable_db_options_.paranoid_checks==false
|
||||
void Corruption(size_t bytes, const Status& s) override {
|
||||
virtual void Corruption(size_t bytes, const Status& s) override {
|
||||
ROCKS_LOG_WARN(info_log, "%s%s: dropping %d bytes; %s",
|
||||
(this->status == nullptr ? "(ignoring error) " : ""),
|
||||
fname, static_cast<int>(bytes), s.ToString().c_str());
|
||||
@@ -549,9 +490,10 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
std::map<std::string, uint32_t> cf_name_id_map;
|
||||
std::map<uint32_t, uint64_t> cf_lognumber_map;
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
cf_name_id_map.insert(std::make_pair(cfd->GetName(), cfd->GetID()));
|
||||
cf_name_id_map.insert(
|
||||
std::make_pair(cfd->GetName(), cfd->GetID()));
|
||||
cf_lognumber_map.insert(
|
||||
std::make_pair(cfd->GetID(), cfd->GetLogNumber()));
|
||||
std::make_pair(cfd->GetID(), cfd->GetLogNumber()));
|
||||
}
|
||||
|
||||
immutable_db_options_.wal_filter->ColumnFamilyLogNumberMap(cf_lognumber_map,
|
||||
@@ -564,13 +506,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
bool flushed = false;
|
||||
uint64_t corrupted_log_number = kMaxSequenceNumber;
|
||||
for (auto log_number : log_numbers) {
|
||||
if (log_number < versions_->min_log_number_to_keep_2pc()) {
|
||||
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
||||
"Skipping log #%" PRIu64
|
||||
" since it is older than min log to keep #%" PRIu64,
|
||||
log_number, versions_->min_log_number_to_keep_2pc());
|
||||
continue;
|
||||
}
|
||||
// The previous incarnation may not have written any MANIFEST
|
||||
// records after allocating this log number. So we manually
|
||||
// update the file number allocation counter in VersionSet.
|
||||
@@ -580,7 +515,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
|
||||
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
||||
"Recovering log #%" PRIu64 " mode %d", log_number,
|
||||
static_cast<int>(immutable_db_options_.wal_recovery_mode));
|
||||
immutable_db_options_.wal_recovery_mode);
|
||||
auto logFileDropped = [this, &fname]() {
|
||||
uint64_t bytes;
|
||||
if (env_->GetFileSize(fname, &bytes).ok()) {
|
||||
@@ -594,9 +529,9 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
continue;
|
||||
}
|
||||
|
||||
std::unique_ptr<SequentialFileReader> file_reader;
|
||||
unique_ptr<SequentialFileReader> file_reader;
|
||||
{
|
||||
std::unique_ptr<SequentialFile> file;
|
||||
unique_ptr<SequentialFile> file;
|
||||
status = env_->NewSequentialFile(fname, &file,
|
||||
env_->OptimizeForLogRead(env_options_));
|
||||
if (!status.ok()) {
|
||||
@@ -609,7 +544,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
continue;
|
||||
}
|
||||
}
|
||||
file_reader.reset(new SequentialFileReader(std::move(file), fname));
|
||||
file_reader.reset(new SequentialFileReader(std::move(file)));
|
||||
}
|
||||
|
||||
// Create the log reader.
|
||||
@@ -629,7 +564,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
// to be skipped instead of propagating bad information (like overly
|
||||
// large sequence numbers).
|
||||
log::Reader reader(immutable_db_options_.info_log, std::move(file_reader),
|
||||
&reporter, true /*checksum*/, log_number);
|
||||
&reporter, true /*checksum*/, 0 /*initial_offset*/,
|
||||
log_number);
|
||||
|
||||
// Determine if we should tolerate incomplete records at the tail end of the
|
||||
// Read all the records and add to a memtable
|
||||
@@ -723,8 +659,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
" mode %d log filter %s returned "
|
||||
"more records (%d) than original (%d) which is not allowed. "
|
||||
"Aborting recovery.",
|
||||
log_number,
|
||||
static_cast<int>(immutable_db_options_.wal_recovery_mode),
|
||||
log_number, immutable_db_options_.wal_recovery_mode,
|
||||
immutable_db_options_.wal_filter->Name(), new_count,
|
||||
original_count);
|
||||
status = Status::NotSupported(
|
||||
@@ -751,7 +686,7 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
status = WriteBatchInternal::InsertInto(
|
||||
&batch, column_family_memtables_.get(), &flush_scheduler_, true,
|
||||
log_number, this, false /* concurrent_memtable_writes */,
|
||||
next_sequence, &has_valid_writes, seq_per_batch_, batch_per_txn_);
|
||||
next_sequence, &has_valid_writes, seq_per_batch_);
|
||||
MaybeIgnoreError(&status);
|
||||
if (!status.ok()) {
|
||||
// We are treating this as a failure while reading since we read valid
|
||||
@@ -901,8 +836,8 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
// VersionSet::next_file_number_ always to be strictly greater than any
|
||||
// log number
|
||||
versions_->MarkFileNumberUsed(max_log_number + 1);
|
||||
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),
|
||||
edit, &mutex_);
|
||||
status = versions_->LogAndApply(
|
||||
cfd, *cfd->GetLatestMutableCFOptions(), edit, &mutex_);
|
||||
if (!status.ok()) {
|
||||
// Recovery failed
|
||||
break;
|
||||
@@ -910,8 +845,18 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
}
|
||||
}
|
||||
|
||||
if (status.ok() && data_seen && !flushed) {
|
||||
status = RestoreAliveLogFiles(log_numbers);
|
||||
if (data_seen && !flushed) {
|
||||
// Mark these as alive so they'll be considered for deletion later by
|
||||
// FindObsoleteFiles()
|
||||
if (two_write_queues_) {
|
||||
log_write_mutex_.Lock();
|
||||
}
|
||||
for (auto log_number : log_numbers) {
|
||||
alive_log_files_.push_back(LogFileNumberSize(log_number));
|
||||
}
|
||||
if (two_write_queues_) {
|
||||
log_write_mutex_.Unlock();
|
||||
}
|
||||
}
|
||||
|
||||
event_logger_.Log() << "job" << job_id << "event"
|
||||
@@ -920,60 +865,6 @@ Status DBImpl::RecoverLogFiles(const std::vector<uint64_t>& log_numbers,
|
||||
return status;
|
||||
}
|
||||
|
||||
Status DBImpl::RestoreAliveLogFiles(const std::vector<uint64_t>& log_numbers) {
|
||||
if (log_numbers.empty()) {
|
||||
return Status::OK();
|
||||
}
|
||||
Status s;
|
||||
mutex_.AssertHeld();
|
||||
assert(immutable_db_options_.avoid_flush_during_recovery);
|
||||
if (two_write_queues_) {
|
||||
log_write_mutex_.Lock();
|
||||
}
|
||||
// Mark these as alive so they'll be considered for deletion later by
|
||||
// FindObsoleteFiles()
|
||||
total_log_size_ = 0;
|
||||
log_empty_ = false;
|
||||
for (auto log_number : log_numbers) {
|
||||
LogFileNumberSize log(log_number);
|
||||
std::string fname = LogFileName(immutable_db_options_.wal_dir, log_number);
|
||||
// This gets the appear size of the logs, not including preallocated space.
|
||||
s = env_->GetFileSize(fname, &log.size);
|
||||
if (!s.ok()) {
|
||||
break;
|
||||
}
|
||||
total_log_size_ += log.size;
|
||||
alive_log_files_.push_back(log);
|
||||
// We preallocate space for logs, but then after a crash and restart, those
|
||||
// preallocated space are not needed anymore. It is likely only the last
|
||||
// log has such preallocated space, so we only truncate for the last log.
|
||||
if (log_number == log_numbers.back()) {
|
||||
std::unique_ptr<WritableFile> last_log;
|
||||
Status truncate_status = env_->ReopenWritableFile(
|
||||
fname, &last_log,
|
||||
env_->OptimizeForLogWrite(
|
||||
env_options_,
|
||||
BuildDBOptions(immutable_db_options_, mutable_db_options_)));
|
||||
if (truncate_status.ok()) {
|
||||
truncate_status = last_log->Truncate(log.size);
|
||||
}
|
||||
if (truncate_status.ok()) {
|
||||
truncate_status = last_log->Close();
|
||||
}
|
||||
// Not a critical error if fail to truncate.
|
||||
if (!truncate_status.ok()) {
|
||||
ROCKS_LOG_WARN(immutable_db_options_.info_log,
|
||||
"Failed to truncate log #%" PRIu64 ": %s", log_number,
|
||||
truncate_status.ToString().c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
if (two_write_queues_) {
|
||||
log_write_mutex_.Unlock();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
|
||||
MemTable* mem, VersionEdit* edit) {
|
||||
mutex_.AssertHeld();
|
||||
@@ -1015,21 +906,14 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
|
||||
if (use_custom_gc_ && snapshot_checker == nullptr) {
|
||||
snapshot_checker = DisableGCSnapshotChecker::Instance();
|
||||
}
|
||||
std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
|
||||
range_del_iters;
|
||||
auto range_del_iter =
|
||||
mem->NewRangeTombstoneIterator(ro, kMaxSequenceNumber);
|
||||
if (range_del_iter != nullptr) {
|
||||
range_del_iters.emplace_back(range_del_iter);
|
||||
}
|
||||
s = BuildTable(
|
||||
dbname_, env_, *cfd->ioptions(), mutable_cf_options,
|
||||
env_options_for_compaction_, cfd->table_cache(), iter.get(),
|
||||
std::move(range_del_iters), &meta, cfd->internal_comparator(),
|
||||
std::unique_ptr<InternalIterator>(mem->NewRangeTombstoneIterator(ro)),
|
||||
&meta, cfd->internal_comparator(),
|
||||
cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
|
||||
snapshot_seqs, earliest_write_conflict_snapshot, snapshot_checker,
|
||||
GetCompressionFlush(*cfd->ioptions(), mutable_cf_options),
|
||||
mutable_cf_options.sample_for_compression,
|
||||
cfd->ioptions()->compression_opts, paranoid_file_checks,
|
||||
cfd->internal_stats(), TableFileCreationReason::kRecovery,
|
||||
&event_logger_, job_id, Env::IO_HIGH, nullptr /* table_properties */,
|
||||
@@ -1051,17 +935,17 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
|
||||
if (s.ok() && meta.fd.GetFileSize() > 0) {
|
||||
edit->AddFile(level, meta.fd.GetNumber(), meta.fd.GetPathId(),
|
||||
meta.fd.GetFileSize(), meta.smallest, meta.largest,
|
||||
meta.fd.smallest_seqno, meta.fd.largest_seqno,
|
||||
meta.smallest_seqno, meta.largest_seqno,
|
||||
meta.marked_for_compaction);
|
||||
}
|
||||
|
||||
InternalStats::CompactionStats stats(CompactionReason::kFlush, 1);
|
||||
InternalStats::CompactionStats stats(1);
|
||||
stats.micros = env_->NowMicros() - start_micros;
|
||||
stats.bytes_written = meta.fd.GetFileSize();
|
||||
stats.num_output_files = 1;
|
||||
cfd->internal_stats()->AddCompactionStats(level, Env::Priority::USER, stats);
|
||||
cfd->internal_stats()->AddCFStats(InternalStats::BYTES_FLUSHED,
|
||||
meta.fd.GetFileSize());
|
||||
cfd->internal_stats()->AddCompactionStats(level, stats);
|
||||
cfd->internal_stats()->AddCFStats(
|
||||
InternalStats::BYTES_FLUSHED, meta.fd.GetFileSize());
|
||||
RecordTick(stats_, COMPACT_WRITE_BYTES, meta.fd.GetFileSize());
|
||||
return s;
|
||||
}
|
||||
@@ -1086,16 +970,15 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
|
||||
Status DB::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
const std::vector<ColumnFamilyDescriptor>& column_families,
|
||||
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
|
||||
const bool kSeqPerBatch = true;
|
||||
const bool kBatchPerTxn = true;
|
||||
const bool seq_per_batch = true;
|
||||
return DBImpl::Open(db_options, dbname, column_families, handles, dbptr,
|
||||
!kSeqPerBatch, kBatchPerTxn);
|
||||
!seq_per_batch);
|
||||
}
|
||||
|
||||
Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
const std::vector<ColumnFamilyDescriptor>& column_families,
|
||||
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
|
||||
const bool seq_per_batch, const bool batch_per_txn) {
|
||||
const bool seq_per_batch) {
|
||||
Status s = SanitizeOptionsByTable(db_options, column_families);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
@@ -1115,30 +998,15 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
std::max(max_write_buffer_size, cf.options.write_buffer_size);
|
||||
}
|
||||
|
||||
DBImpl* impl = new DBImpl(db_options, dbname, seq_per_batch, batch_per_txn);
|
||||
DBImpl* impl = new DBImpl(db_options, dbname, seq_per_batch);
|
||||
s = impl->env_->CreateDirIfMissing(impl->immutable_db_options_.wal_dir);
|
||||
if (s.ok()) {
|
||||
std::vector<std::string> paths;
|
||||
for (auto& db_path : impl->immutable_db_options_.db_paths) {
|
||||
paths.emplace_back(db_path.path);
|
||||
}
|
||||
for (auto& cf : column_families) {
|
||||
for (auto& cf_path : cf.options.cf_paths) {
|
||||
paths.emplace_back(cf_path.path);
|
||||
}
|
||||
}
|
||||
for (auto& path : paths) {
|
||||
s = impl->env_->CreateDirIfMissing(path);
|
||||
for (auto db_path : impl->immutable_db_options_.db_paths) {
|
||||
s = impl->env_->CreateDirIfMissing(db_path.path);
|
||||
if (!s.ok()) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// For recovery from NoSpace() error, we can only handle
|
||||
// the case where the database is stored in a single path
|
||||
if (paths.size() <= 1) {
|
||||
impl->error_handler_.EnableAutoRecovery();
|
||||
}
|
||||
}
|
||||
|
||||
if (!s.ok()) {
|
||||
@@ -1157,16 +1025,16 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
s = impl->Recover(column_families);
|
||||
if (s.ok()) {
|
||||
uint64_t new_log_number = impl->versions_->NewFileNumber();
|
||||
std::unique_ptr<WritableFile> lfile;
|
||||
unique_ptr<WritableFile> lfile;
|
||||
EnvOptions soptions(db_options);
|
||||
EnvOptions opt_env_options =
|
||||
impl->immutable_db_options_.env->OptimizeForLogWrite(
|
||||
soptions, BuildDBOptions(impl->immutable_db_options_,
|
||||
impl->mutable_db_options_));
|
||||
std::string log_fname =
|
||||
LogFileName(impl->immutable_db_options_.wal_dir, new_log_number);
|
||||
s = NewWritableFile(impl->immutable_db_options_.env, log_fname, &lfile,
|
||||
opt_env_options);
|
||||
s = NewWritableFile(
|
||||
impl->immutable_db_options_.env,
|
||||
LogFileName(impl->immutable_db_options_.wal_dir, new_log_number),
|
||||
&lfile, opt_env_options);
|
||||
if (s.ok()) {
|
||||
lfile->SetWriteLifeTimeHint(write_hint);
|
||||
lfile->SetPreallocationBlockSize(
|
||||
@@ -1174,16 +1042,13 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
{
|
||||
InstrumentedMutexLock wl(&impl->log_write_mutex_);
|
||||
impl->logfile_number_ = new_log_number;
|
||||
const auto& listeners = impl->immutable_db_options_.listeners;
|
||||
std::unique_ptr<WritableFileWriter> file_writer(
|
||||
new WritableFileWriter(std::move(lfile), log_fname, opt_env_options,
|
||||
impl->env_, nullptr /* stats */, listeners));
|
||||
unique_ptr<WritableFileWriter> file_writer(
|
||||
new WritableFileWriter(std::move(lfile), opt_env_options));
|
||||
impl->logs_.emplace_back(
|
||||
new_log_number,
|
||||
new log::Writer(
|
||||
std::move(file_writer), new_log_number,
|
||||
impl->immutable_db_options_.recycle_log_file_num > 0,
|
||||
impl->immutable_db_options_.manual_wal_flush));
|
||||
impl->immutable_db_options_.recycle_log_file_num > 0));
|
||||
}
|
||||
|
||||
// set column family handles
|
||||
@@ -1254,8 +1119,7 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
!cfd->mem()->IsMergeOperatorSupported()) {
|
||||
s = Status::InvalidArgument(
|
||||
"The memtable of column family %s does not support merge operator "
|
||||
"its options.merge_operator is non-null",
|
||||
cfd->GetName().c_str());
|
||||
"its options.merge_operator is non-null", cfd->GetName().c_str());
|
||||
}
|
||||
if (!s.ok()) {
|
||||
break;
|
||||
@@ -1281,57 +1145,31 @@ Status DBImpl::Open(const DBOptions& db_options, const std::string& dbname,
|
||||
impl->immutable_db_options_.sst_file_manager.get());
|
||||
if (s.ok() && sfm) {
|
||||
// Notify SstFileManager about all sst files that already exist in
|
||||
// db_paths[0] and cf_paths[0] when the DB is opened.
|
||||
std::vector<std::string> paths;
|
||||
paths.emplace_back(impl->immutable_db_options_.db_paths[0].path);
|
||||
for (auto& cf : column_families) {
|
||||
if (!cf.options.cf_paths.empty()) {
|
||||
paths.emplace_back(cf.options.cf_paths[0].path);
|
||||
// db_paths[0] when the DB is opened.
|
||||
auto& db_path = impl->immutable_db_options_.db_paths[0];
|
||||
std::vector<std::string> existing_files;
|
||||
impl->immutable_db_options_.env->GetChildren(db_path.path, &existing_files);
|
||||
for (auto& file_name : existing_files) {
|
||||
uint64_t file_number;
|
||||
FileType file_type;
|
||||
std::string file_path = db_path.path + "/" + file_name;
|
||||
if (ParseFileName(file_name, &file_number, &file_type) &&
|
||||
file_type == kTableFile) {
|
||||
sfm->OnAddFile(file_path);
|
||||
}
|
||||
}
|
||||
// Remove duplicate paths.
|
||||
std::sort(paths.begin(), paths.end());
|
||||
paths.erase(std::unique(paths.begin(), paths.end()), paths.end());
|
||||
for (auto& path : paths) {
|
||||
std::vector<std::string> existing_files;
|
||||
impl->immutable_db_options_.env->GetChildren(path, &existing_files);
|
||||
for (auto& file_name : existing_files) {
|
||||
uint64_t file_number;
|
||||
FileType file_type;
|
||||
std::string file_path = path + "/" + file_name;
|
||||
if (ParseFileName(file_name, &file_number, &file_type) &&
|
||||
file_type == kTableFile) {
|
||||
sfm->OnAddFile(file_path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reserve some disk buffer space. This is a heuristic - when we run out
|
||||
// of disk space, this ensures that there is atleast write_buffer_size
|
||||
// amount of free space before we resume DB writes. In low disk space
|
||||
// conditions, we want to avoid a lot of small L0 files due to frequent
|
||||
// WAL write failures and resultant forced flushes
|
||||
sfm->ReserveDiskBuffer(max_write_buffer_size,
|
||||
impl->immutable_db_options_.db_paths[0].path);
|
||||
}
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
if (s.ok()) {
|
||||
ROCKS_LOG_HEADER(impl->immutable_db_options_.info_log, "DB pointer %p",
|
||||
impl);
|
||||
ROCKS_LOG_INFO(impl->immutable_db_options_.info_log, "DB pointer %p", impl);
|
||||
LogFlush(impl->immutable_db_options_.info_log);
|
||||
assert(impl->TEST_WALBufferIsEmpty());
|
||||
// If the assert above fails then we need to FlushWAL before returning
|
||||
// control back to the user.
|
||||
if (!persist_options_status.ok()) {
|
||||
s = Status::IOError(
|
||||
"DB::Open() failed --- Unable to persist Options file",
|
||||
persist_options_status.ToString());
|
||||
}
|
||||
}
|
||||
if (s.ok()) {
|
||||
impl->StartTimedTasks();
|
||||
}
|
||||
if (!s.ok()) {
|
||||
for (auto* h : *handles) {
|
||||
delete h;
|
||||
|
||||
+27
-45
@@ -9,6 +9,7 @@
|
||||
#include "db/db_impl.h"
|
||||
#include "db/db_iter.h"
|
||||
#include "db/merge_context.h"
|
||||
#include "db/range_del_aggregator.h"
|
||||
#include "monitoring/perf_context_imp.h"
|
||||
|
||||
namespace rocksdb {
|
||||
@@ -23,45 +24,30 @@ DBImplReadOnly::DBImplReadOnly(const DBOptions& db_options,
|
||||
LogFlush(immutable_db_options_.info_log);
|
||||
}
|
||||
|
||||
DBImplReadOnly::~DBImplReadOnly() {}
|
||||
DBImplReadOnly::~DBImplReadOnly() {
|
||||
}
|
||||
|
||||
// Implementations of the DB interface
|
||||
Status DBImplReadOnly::Get(const ReadOptions& read_options,
|
||||
ColumnFamilyHandle* column_family, const Slice& key,
|
||||
PinnableSlice* pinnable_val) {
|
||||
assert(pinnable_val != nullptr);
|
||||
// TODO: stopwatch DB_GET needed?, perf timer needed?
|
||||
PERF_TIMER_GUARD(get_snapshot_time);
|
||||
Status s;
|
||||
SequenceNumber snapshot = versions_->LastSequence();
|
||||
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
||||
auto cfd = cfh->cfd();
|
||||
if (tracer_) {
|
||||
InstrumentedMutexLock lock(&trace_mutex_);
|
||||
if (tracer_) {
|
||||
tracer_->Get(column_family, key);
|
||||
}
|
||||
}
|
||||
SuperVersion* super_version = cfd->GetSuperVersion();
|
||||
MergeContext merge_context;
|
||||
SequenceNumber max_covering_tombstone_seq = 0;
|
||||
RangeDelAggregator range_del_agg(cfd->internal_comparator(), snapshot);
|
||||
LookupKey lkey(key, snapshot);
|
||||
PERF_TIMER_STOP(get_snapshot_time);
|
||||
if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
|
||||
&max_covering_tombstone_seq, read_options)) {
|
||||
&range_del_agg, read_options)) {
|
||||
pinnable_val->PinSelf();
|
||||
RecordTick(stats_, MEMTABLE_HIT);
|
||||
} else {
|
||||
PERF_TIMER_GUARD(get_from_output_files_time);
|
||||
super_version->current->Get(read_options, lkey, pinnable_val, &s,
|
||||
&merge_context, &max_covering_tombstone_seq);
|
||||
RecordTick(stats_, MEMTABLE_MISS);
|
||||
&merge_context, &range_del_agg);
|
||||
}
|
||||
RecordTick(stats_, NUMBER_KEYS_READ);
|
||||
size_t size = pinnable_val->size();
|
||||
RecordTick(stats_, BYTES_READ, size);
|
||||
RecordInHistogram(stats_, BYTES_PER_READ, size);
|
||||
PERF_COUNTER_ADD(get_read_bytes, size);
|
||||
return s;
|
||||
}
|
||||
|
||||
@@ -71,20 +57,18 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
|
||||
auto cfd = cfh->cfd();
|
||||
SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
|
||||
SequenceNumber latest_snapshot = versions_->LastSequence();
|
||||
SequenceNumber read_seq =
|
||||
read_options.snapshot != nullptr
|
||||
? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
|
||||
->number_
|
||||
: latest_snapshot;
|
||||
ReadCallback* read_callback = nullptr; // No read callback provided.
|
||||
auto db_iter = NewArenaWrappedDbIterator(
|
||||
env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
|
||||
read_seq,
|
||||
env_, read_options, *cfd->ioptions(),
|
||||
(read_options.snapshot != nullptr
|
||||
? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
|
||||
->number_
|
||||
: latest_snapshot),
|
||||
super_version->mutable_cf_options.max_sequential_skip_in_iterations,
|
||||
super_version->version_number, read_callback);
|
||||
auto internal_iter =
|
||||
NewInternalIterator(read_options, cfd, super_version, db_iter->GetArena(),
|
||||
db_iter->GetRangeDelAggregator(), read_seq);
|
||||
db_iter->GetRangeDelAggregator());
|
||||
db_iter->SetIterUnderDBIter(internal_iter);
|
||||
return db_iter;
|
||||
}
|
||||
@@ -100,22 +84,21 @@ Status DBImplReadOnly::NewIterators(
|
||||
iterators->clear();
|
||||
iterators->reserve(column_families.size());
|
||||
SequenceNumber latest_snapshot = versions_->LastSequence();
|
||||
SequenceNumber read_seq =
|
||||
read_options.snapshot != nullptr
|
||||
? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
|
||||
->number_
|
||||
: latest_snapshot;
|
||||
|
||||
for (auto cfh : column_families) {
|
||||
auto* cfd = reinterpret_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
|
||||
auto* sv = cfd->GetSuperVersion()->Ref();
|
||||
auto* db_iter = NewArenaWrappedDbIterator(
|
||||
env_, read_options, *cfd->ioptions(), sv->mutable_cf_options, read_seq,
|
||||
env_, read_options, *cfd->ioptions(),
|
||||
(read_options.snapshot != nullptr
|
||||
? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
|
||||
->number_
|
||||
: latest_snapshot),
|
||||
sv->mutable_cf_options.max_sequential_skip_in_iterations,
|
||||
sv->version_number, read_callback);
|
||||
auto* internal_iter =
|
||||
NewInternalIterator(read_options, cfd, sv, db_iter->GetArena(),
|
||||
db_iter->GetRangeDelAggregator(), read_seq);
|
||||
db_iter->GetRangeDelAggregator());
|
||||
db_iter->SetIterUnderDBIter(internal_iter);
|
||||
iterators->push_back(db_iter);
|
||||
}
|
||||
@@ -124,7 +107,7 @@ Status DBImplReadOnly::NewIterators(
|
||||
}
|
||||
|
||||
Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
|
||||
DB** dbptr, bool /*error_if_log_file_exist*/) {
|
||||
DB** dbptr, bool error_if_log_file_exist) {
|
||||
*dbptr = nullptr;
|
||||
|
||||
// Try to first open DB as fully compacted DB
|
||||
@@ -200,21 +183,20 @@ Status DB::OpenForReadOnly(
|
||||
return s;
|
||||
}
|
||||
|
||||
#else // !ROCKSDB_LITE
|
||||
#else // !ROCKSDB_LITE
|
||||
|
||||
Status DB::OpenForReadOnly(const Options& /*options*/,
|
||||
const std::string& /*dbname*/, DB** /*dbptr*/,
|
||||
bool /*error_if_log_file_exist*/) {
|
||||
Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
|
||||
DB** dbptr, bool error_if_log_file_exist) {
|
||||
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
|
||||
}
|
||||
|
||||
Status DB::OpenForReadOnly(
|
||||
const DBOptions& /*db_options*/, const std::string& /*dbname*/,
|
||||
const std::vector<ColumnFamilyDescriptor>& /*column_families*/,
|
||||
std::vector<ColumnFamilyHandle*>* /*handles*/, DB** /*dbptr*/,
|
||||
bool /*error_if_log_file_exist*/) {
|
||||
const DBOptions& db_options, const std::string& dbname,
|
||||
const std::vector<ColumnFamilyDescriptor>& column_families,
|
||||
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
|
||||
bool error_if_log_file_exist) {
|
||||
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
|
||||
}
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
} // namespace rocksdb
|
||||
|
||||
+33
-37
@@ -7,9 +7,9 @@
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "db/db_impl.h"
|
||||
#include <vector>
|
||||
#include <string>
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
@@ -36,49 +36,46 @@ class DBImplReadOnly : public DBImpl {
|
||||
std::vector<Iterator*>* iterators) override;
|
||||
|
||||
using DBImpl::Put;
|
||||
virtual Status Put(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/, const Slice& /*value*/) override {
|
||||
virtual Status Put(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family, const Slice& key,
|
||||
const Slice& value) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
using DBImpl::Merge;
|
||||
virtual Status Merge(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/, const Slice& /*value*/) override {
|
||||
virtual Status Merge(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family, const Slice& key,
|
||||
const Slice& value) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
using DBImpl::Delete;
|
||||
virtual Status Delete(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/) override {
|
||||
virtual Status Delete(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const Slice& key) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
using DBImpl::SingleDelete;
|
||||
virtual Status SingleDelete(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/) override {
|
||||
virtual Status SingleDelete(const WriteOptions& options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const Slice& key) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status Write(const WriteOptions& /*options*/,
|
||||
WriteBatch* /*updates*/) override {
|
||||
virtual Status Write(const WriteOptions& options,
|
||||
WriteBatch* updates) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
using DBImpl::CompactRange;
|
||||
virtual Status CompactRange(const CompactRangeOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice* /*begin*/,
|
||||
const Slice* /*end*/) override {
|
||||
virtual Status CompactRange(const CompactRangeOptions& options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const Slice* begin, const Slice* end) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::CompactFiles;
|
||||
virtual Status CompactFiles(
|
||||
const CompactionOptions& /*compact_options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const std::vector<std::string>& /*input_file_names*/,
|
||||
const int /*output_level*/, const int /*output_path_id*/ = -1,
|
||||
std::vector<std::string>* const /*output_file_names*/ = nullptr,
|
||||
CompactionJobInfo* /*compaction_job_info*/ = nullptr) override {
|
||||
const CompactionOptions& compact_options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const std::vector<std::string>& input_file_names,
|
||||
const int output_level, const int output_path_id = -1) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
@@ -86,19 +83,18 @@ class DBImplReadOnly : public DBImpl {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
virtual Status EnableFileDeletions(bool /*force*/) override {
|
||||
virtual Status EnableFileDeletions(bool force) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
virtual Status GetLiveFiles(std::vector<std::string>& ret,
|
||||
virtual Status GetLiveFiles(std::vector<std::string>&,
|
||||
uint64_t* manifest_file_size,
|
||||
bool /*flush_memtable*/) override {
|
||||
return DBImpl::GetLiveFiles(ret, manifest_file_size,
|
||||
false /* flush_memtable */);
|
||||
bool flush_memtable = true) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::Flush;
|
||||
virtual Status Flush(const FlushOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/) override {
|
||||
virtual Status Flush(const FlushOptions& options,
|
||||
ColumnFamilyHandle* column_family) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
@@ -109,9 +105,9 @@ class DBImplReadOnly : public DBImpl {
|
||||
|
||||
using DB::IngestExternalFile;
|
||||
virtual Status IngestExternalFile(
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const std::vector<std::string>& /*external_files*/,
|
||||
const IngestExternalFileOptions& /*ingestion_options*/) override {
|
||||
ColumnFamilyHandle* column_family,
|
||||
const std::vector<std::string>& external_files,
|
||||
const IngestExternalFileOptions& ingestion_options) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
@@ -122,6 +118,6 @@ class DBImplReadOnly : public DBImpl {
|
||||
DBImplReadOnly(const DBImplReadOnly&);
|
||||
void operator=(const DBImplReadOnly&);
|
||||
};
|
||||
} // namespace rocksdb
|
||||
}
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
@@ -1,356 +0,0 @@
|
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#include "db/db_impl_secondary.h"
|
||||
#include "db/db_iter.h"
|
||||
#include "db/merge_context.h"
|
||||
#include "monitoring/perf_context_imp.h"
|
||||
#include "util/auto_roll_logger.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
DBImplSecondary::DBImplSecondary(const DBOptions& db_options,
|
||||
const std::string& dbname)
|
||||
: DBImpl(db_options, dbname) {
|
||||
ROCKS_LOG_INFO(immutable_db_options_.info_log,
|
||||
"Opening the db in secondary mode");
|
||||
LogFlush(immutable_db_options_.info_log);
|
||||
}
|
||||
|
||||
DBImplSecondary::~DBImplSecondary() {}
|
||||
|
||||
Status DBImplSecondary::Recover(
|
||||
const std::vector<ColumnFamilyDescriptor>& column_families,
|
||||
bool /*readonly*/, bool /*error_if_log_file_exist*/,
|
||||
bool /*error_if_data_exists_in_logs*/) {
|
||||
mutex_.AssertHeld();
|
||||
|
||||
Status s;
|
||||
s = static_cast<ReactiveVersionSet*>(versions_.get())
|
||||
->Recover(column_families, &manifest_reader_, &manifest_reporter_,
|
||||
&manifest_reader_status_);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
if (immutable_db_options_.paranoid_checks && s.ok()) {
|
||||
s = CheckConsistency();
|
||||
}
|
||||
// Initial max_total_in_memory_state_ before recovery logs.
|
||||
max_total_in_memory_state_ = 0;
|
||||
for (auto cfd : *versions_->GetColumnFamilySet()) {
|
||||
auto* mutable_cf_options = cfd->GetLatestMutableCFOptions();
|
||||
max_total_in_memory_state_ += mutable_cf_options->write_buffer_size *
|
||||
mutable_cf_options->max_write_buffer_number;
|
||||
}
|
||||
if (s.ok()) {
|
||||
default_cf_handle_ = new ColumnFamilyHandleImpl(
|
||||
versions_->GetColumnFamilySet()->GetDefault(), this, &mutex_);
|
||||
default_cf_internal_stats_ = default_cf_handle_->cfd()->internal_stats();
|
||||
single_column_family_mode_ =
|
||||
versions_->GetColumnFamilySet()->NumberOfColumnFamilies() == 1;
|
||||
}
|
||||
|
||||
// TODO: attempt to recover from WAL files.
|
||||
return s;
|
||||
}
|
||||
|
||||
// Implementation of the DB interface
|
||||
Status DBImplSecondary::Get(const ReadOptions& read_options,
|
||||
ColumnFamilyHandle* column_family, const Slice& key,
|
||||
PinnableSlice* value) {
|
||||
return GetImpl(read_options, column_family, key, value);
|
||||
}
|
||||
|
||||
Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
|
||||
ColumnFamilyHandle* column_family,
|
||||
const Slice& key, PinnableSlice* pinnable_val) {
|
||||
assert(pinnable_val != nullptr);
|
||||
PERF_CPU_TIMER_GUARD(get_cpu_nanos, env_);
|
||||
StopWatch sw(env_, stats_, DB_GET);
|
||||
PERF_TIMER_GUARD(get_snapshot_time);
|
||||
|
||||
auto cfh = static_cast<ColumnFamilyHandleImpl*>(column_family);
|
||||
ColumnFamilyData* cfd = cfh->cfd();
|
||||
if (tracer_) {
|
||||
InstrumentedMutexLock lock(&trace_mutex_);
|
||||
if (tracer_) {
|
||||
tracer_->Get(column_family, key);
|
||||
}
|
||||
}
|
||||
// Acquire SuperVersion
|
||||
SuperVersion* super_version = GetAndRefSuperVersion(cfd);
|
||||
SequenceNumber snapshot = versions_->LastSequence();
|
||||
MergeContext merge_context;
|
||||
SequenceNumber max_covering_tombstone_seq = 0;
|
||||
Status s;
|
||||
LookupKey lkey(key, snapshot);
|
||||
PERF_TIMER_STOP(get_snapshot_time);
|
||||
|
||||
bool done = false;
|
||||
if (super_version->mem->Get(lkey, pinnable_val->GetSelf(), &s, &merge_context,
|
||||
&max_covering_tombstone_seq, read_options)) {
|
||||
done = true;
|
||||
pinnable_val->PinSelf();
|
||||
RecordTick(stats_, MEMTABLE_HIT);
|
||||
} else if ((s.ok() || s.IsMergeInProgress()) &&
|
||||
super_version->imm->Get(
|
||||
lkey, pinnable_val->GetSelf(), &s, &merge_context,
|
||||
&max_covering_tombstone_seq, read_options)) {
|
||||
done = true;
|
||||
pinnable_val->PinSelf();
|
||||
RecordTick(stats_, MEMTABLE_HIT);
|
||||
}
|
||||
if (!done && !s.ok() && !s.IsMergeInProgress()) {
|
||||
ReturnAndCleanupSuperVersion(cfd, super_version);
|
||||
return s;
|
||||
}
|
||||
if (!done) {
|
||||
PERF_TIMER_GUARD(get_from_output_files_time);
|
||||
super_version->current->Get(read_options, lkey, pinnable_val, &s,
|
||||
&merge_context, &max_covering_tombstone_seq);
|
||||
RecordTick(stats_, MEMTABLE_MISS);
|
||||
}
|
||||
{
|
||||
PERF_TIMER_GUARD(get_post_process_time);
|
||||
ReturnAndCleanupSuperVersion(cfd, super_version);
|
||||
RecordTick(stats_, NUMBER_KEYS_READ);
|
||||
size_t size = pinnable_val->size();
|
||||
RecordTick(stats_, BYTES_READ, size);
|
||||
RecordTimeToHistogram(stats_, BYTES_PER_READ, size);
|
||||
PERF_COUNTER_ADD(get_read_bytes, size);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options,
|
||||
ColumnFamilyHandle* column_family) {
|
||||
if (read_options.managed) {
|
||||
return NewErrorIterator(
|
||||
Status::NotSupported("Managed iterator is not supported anymore."));
|
||||
}
|
||||
if (read_options.read_tier == kPersistedTier) {
|
||||
return NewErrorIterator(Status::NotSupported(
|
||||
"ReadTier::kPersistedData is not yet supported in iterators."));
|
||||
}
|
||||
Iterator* result = nullptr;
|
||||
auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
|
||||
auto cfd = cfh->cfd();
|
||||
ReadCallback* read_callback = nullptr; // No read callback provided.
|
||||
if (read_options.tailing) {
|
||||
return NewErrorIterator(Status::NotSupported(
|
||||
"tailing iterator not supported in secondary mode"));
|
||||
} else if (read_options.snapshot != nullptr) {
|
||||
// TODO (yanqin) support snapshot.
|
||||
return NewErrorIterator(
|
||||
Status::NotSupported("snapshot not supported in secondary mode"));
|
||||
} else {
|
||||
auto snapshot = versions_->LastSequence();
|
||||
result = NewIteratorImpl(read_options, cfd, snapshot, read_callback);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
ArenaWrappedDBIter* DBImplSecondary::NewIteratorImpl(
|
||||
const ReadOptions& read_options, ColumnFamilyData* cfd,
|
||||
SequenceNumber snapshot, ReadCallback* read_callback) {
|
||||
assert(nullptr != cfd);
|
||||
SuperVersion* super_version = cfd->GetReferencedSuperVersion(&mutex_);
|
||||
auto db_iter = NewArenaWrappedDbIterator(
|
||||
env_, read_options, *cfd->ioptions(), super_version->mutable_cf_options,
|
||||
snapshot,
|
||||
super_version->mutable_cf_options.max_sequential_skip_in_iterations,
|
||||
super_version->version_number, read_callback);
|
||||
auto internal_iter =
|
||||
NewInternalIterator(read_options, cfd, super_version, db_iter->GetArena(),
|
||||
db_iter->GetRangeDelAggregator(), snapshot);
|
||||
db_iter->SetIterUnderDBIter(internal_iter);
|
||||
return db_iter;
|
||||
}
|
||||
|
||||
Status DBImplSecondary::NewIterators(
|
||||
const ReadOptions& read_options,
|
||||
const std::vector<ColumnFamilyHandle*>& column_families,
|
||||
std::vector<Iterator*>* iterators) {
|
||||
if (read_options.managed) {
|
||||
return Status::NotSupported("Managed iterator is not supported anymore.");
|
||||
}
|
||||
if (read_options.read_tier == kPersistedTier) {
|
||||
return Status::NotSupported(
|
||||
"ReadTier::kPersistedData is not yet supported in iterators.");
|
||||
}
|
||||
ReadCallback* read_callback = nullptr; // No read callback provided.
|
||||
if (iterators == nullptr) {
|
||||
return Status::InvalidArgument("iterators not allowed to be nullptr");
|
||||
}
|
||||
iterators->clear();
|
||||
iterators->reserve(column_families.size());
|
||||
if (read_options.tailing) {
|
||||
return Status::NotSupported(
|
||||
"tailing iterator not supported in secondary mode");
|
||||
} else if (read_options.snapshot != nullptr) {
|
||||
// TODO (yanqin) support snapshot.
|
||||
return Status::NotSupported("snapshot not supported in secondary mode");
|
||||
} else {
|
||||
SequenceNumber read_seq = versions_->LastSequence();
|
||||
for (auto cfh : column_families) {
|
||||
ColumnFamilyData* cfd = static_cast<ColumnFamilyHandleImpl*>(cfh)->cfd();
|
||||
iterators->push_back(
|
||||
NewIteratorImpl(read_options, cfd, read_seq, read_callback));
|
||||
}
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
Status DBImplSecondary::TryCatchUpWithPrimary() {
|
||||
assert(versions_.get() != nullptr);
|
||||
assert(manifest_reader_.get() != nullptr);
|
||||
Status s;
|
||||
std::unordered_set<ColumnFamilyData*> cfds_changed;
|
||||
InstrumentedMutexLock lock_guard(&mutex_);
|
||||
s = static_cast<ReactiveVersionSet*>(versions_.get())
|
||||
->ReadAndApply(&mutex_, &manifest_reader_, &cfds_changed);
|
||||
if (s.ok()) {
|
||||
SuperVersionContext sv_context(true /* create_superversion */);
|
||||
for (auto cfd : cfds_changed) {
|
||||
sv_context.NewSuperVersion();
|
||||
cfd->InstallSuperVersion(&sv_context, &mutex_);
|
||||
}
|
||||
sv_context.Clean();
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status DB::OpenAsSecondary(const Options& options, const std::string& dbname,
|
||||
const std::string& secondary_path, DB** dbptr) {
|
||||
*dbptr = nullptr;
|
||||
|
||||
DBOptions db_options(options);
|
||||
ColumnFamilyOptions cf_options(options);
|
||||
std::vector<ColumnFamilyDescriptor> column_families;
|
||||
column_families.emplace_back(kDefaultColumnFamilyName, cf_options);
|
||||
std::vector<ColumnFamilyHandle*> handles;
|
||||
|
||||
Status s = DB::OpenAsSecondary(db_options, dbname, secondary_path,
|
||||
column_families, &handles, dbptr);
|
||||
if (s.ok()) {
|
||||
assert(handles.size() == 1);
|
||||
delete handles[0];
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
Status DB::OpenAsSecondary(
|
||||
const DBOptions& db_options, const std::string& dbname,
|
||||
const std::string& secondary_path,
|
||||
const std::vector<ColumnFamilyDescriptor>& column_families,
|
||||
std::vector<ColumnFamilyHandle*>* handles, DB** dbptr) {
|
||||
*dbptr = nullptr;
|
||||
if (db_options.max_open_files != -1) {
|
||||
// TODO (yanqin) maybe support max_open_files != -1 by creating hard links
|
||||
// on SST files so that db secondary can still have access to old SSTs
|
||||
// while primary instance may delete original.
|
||||
return Status::InvalidArgument("require max_open_files to be -1");
|
||||
}
|
||||
|
||||
DBOptions tmp_opts(db_options);
|
||||
if (nullptr == tmp_opts.info_log) {
|
||||
Env* env = tmp_opts.env;
|
||||
assert(env != nullptr);
|
||||
std::string secondary_abs_path;
|
||||
env->GetAbsolutePath(secondary_path, &secondary_abs_path);
|
||||
std::string fname = InfoLogFileName(secondary_path, secondary_abs_path,
|
||||
tmp_opts.db_log_dir);
|
||||
|
||||
env->CreateDirIfMissing(secondary_path);
|
||||
if (tmp_opts.log_file_time_to_roll > 0 || tmp_opts.max_log_file_size > 0) {
|
||||
AutoRollLogger* result = new AutoRollLogger(
|
||||
env, secondary_path, tmp_opts.db_log_dir, tmp_opts.max_log_file_size,
|
||||
tmp_opts.log_file_time_to_roll, tmp_opts.info_log_level);
|
||||
Status s = result->GetStatus();
|
||||
if (!s.ok()) {
|
||||
delete result;
|
||||
} else {
|
||||
tmp_opts.info_log.reset(result);
|
||||
}
|
||||
}
|
||||
if (nullptr == tmp_opts.info_log) {
|
||||
env->RenameFile(
|
||||
fname, OldInfoLogFileName(secondary_path, env->NowMicros(),
|
||||
secondary_abs_path, tmp_opts.db_log_dir));
|
||||
Status s = env->NewLogger(fname, &(tmp_opts.info_log));
|
||||
if (tmp_opts.info_log != nullptr) {
|
||||
tmp_opts.info_log->SetInfoLogLevel(tmp_opts.info_log_level);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert(tmp_opts.info_log != nullptr);
|
||||
|
||||
handles->clear();
|
||||
DBImplSecondary* impl = new DBImplSecondary(tmp_opts, dbname);
|
||||
impl->versions_.reset(new ReactiveVersionSet(
|
||||
dbname, &impl->immutable_db_options_, impl->env_options_,
|
||||
impl->table_cache_.get(), impl->write_buffer_manager_,
|
||||
&impl->write_controller_));
|
||||
impl->column_family_memtables_.reset(
|
||||
new ColumnFamilyMemTablesImpl(impl->versions_->GetColumnFamilySet()));
|
||||
impl->mutex_.Lock();
|
||||
Status s = impl->Recover(column_families, true, false, false);
|
||||
if (s.ok()) {
|
||||
for (auto cf : column_families) {
|
||||
auto cfd =
|
||||
impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
|
||||
if (nullptr == cfd) {
|
||||
s = Status::InvalidArgument("Column family not found: ", cf.name);
|
||||
break;
|
||||
}
|
||||
handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_));
|
||||
}
|
||||
}
|
||||
SuperVersionContext sv_context(true /* create_superversion */);
|
||||
if (s.ok()) {
|
||||
for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
|
||||
sv_context.NewSuperVersion();
|
||||
cfd->InstallSuperVersion(&sv_context, &impl->mutex_);
|
||||
}
|
||||
}
|
||||
impl->mutex_.Unlock();
|
||||
sv_context.Clean();
|
||||
if (s.ok()) {
|
||||
*dbptr = impl;
|
||||
for (auto h : *handles) {
|
||||
impl->NewThreadStatusCfInfo(
|
||||
reinterpret_cast<ColumnFamilyHandleImpl*>(h)->cfd());
|
||||
}
|
||||
} else {
|
||||
for (auto h : *handles) {
|
||||
delete h;
|
||||
}
|
||||
handles->clear();
|
||||
delete impl;
|
||||
}
|
||||
return s;
|
||||
}
|
||||
#else // !ROCKSDB_LITE
|
||||
|
||||
Status DB::OpenAsSecondary(const Options& /*options*/,
|
||||
const std::string& /*name*/,
|
||||
const std::string& /*secondary_path*/,
|
||||
DB** /*dbptr*/) {
|
||||
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
|
||||
}
|
||||
|
||||
Status DB::OpenAsSecondary(
|
||||
const DBOptions& /*db_options*/, const std::string& /*dbname*/,
|
||||
const std::string& /*secondary_path*/,
|
||||
const std::vector<ColumnFamilyDescriptor>& /*column_families*/,
|
||||
std::vector<ColumnFamilyHandle*>* /*handles*/, DB** /*dbptr*/) {
|
||||
return Status::NotSupported("Not supported in ROCKSDB_LITE.");
|
||||
}
|
||||
#endif // !ROCKSDB_LITE
|
||||
|
||||
} // namespace rocksdb
|
||||
@@ -1,151 +0,0 @@
|
||||
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||
// This source code is licensed under both the GPLv2 (found in the
|
||||
// COPYING file in the root directory) and Apache 2.0 License
|
||||
// (found in the LICENSE.Apache file in the root directory).
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifndef ROCKSDB_LITE
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include "db/db_impl.h"
|
||||
|
||||
namespace rocksdb {
|
||||
|
||||
class DBImplSecondary : public DBImpl {
|
||||
public:
|
||||
DBImplSecondary(const DBOptions& options, const std::string& dbname);
|
||||
~DBImplSecondary() override;
|
||||
|
||||
Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
|
||||
bool read_only, bool error_if_log_file_exist,
|
||||
bool error_if_data_exists_in_logs) override;
|
||||
|
||||
// Implementations of the DB interface
|
||||
using DB::Get;
|
||||
Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family,
|
||||
const Slice& key, PinnableSlice* value) override;
|
||||
|
||||
Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
|
||||
const Slice& key, PinnableSlice* value);
|
||||
|
||||
using DBImpl::NewIterator;
|
||||
Iterator* NewIterator(const ReadOptions&,
|
||||
ColumnFamilyHandle* column_family) override;
|
||||
|
||||
ArenaWrappedDBIter* NewIteratorImpl(const ReadOptions& read_options,
|
||||
ColumnFamilyData* cfd,
|
||||
SequenceNumber snapshot,
|
||||
ReadCallback* read_callback);
|
||||
|
||||
Status NewIterators(const ReadOptions& options,
|
||||
const std::vector<ColumnFamilyHandle*>& column_families,
|
||||
std::vector<Iterator*>* iterators) override;
|
||||
|
||||
using DBImpl::Put;
|
||||
Status Put(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
|
||||
const Slice& /*value*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::Merge;
|
||||
Status Merge(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/, const Slice& /*key*/,
|
||||
const Slice& /*value*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::Delete;
|
||||
Status Delete(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::SingleDelete;
|
||||
Status SingleDelete(const WriteOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice& /*key*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
Status Write(const WriteOptions& /*options*/,
|
||||
WriteBatch* /*updates*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::CompactRange;
|
||||
Status CompactRange(const CompactRangeOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const Slice* /*begin*/, const Slice* /*end*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::CompactFiles;
|
||||
Status CompactFiles(
|
||||
const CompactionOptions& /*compact_options*/,
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const std::vector<std::string>& /*input_file_names*/,
|
||||
const int /*output_level*/, const int /*output_path_id*/ = -1,
|
||||
std::vector<std::string>* const /*output_file_names*/ = nullptr,
|
||||
CompactionJobInfo* /*compaction_job_info*/ = nullptr) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
Status DisableFileDeletions() override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
Status EnableFileDeletions(bool /*force*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
Status GetLiveFiles(std::vector<std::string>&,
|
||||
uint64_t* /*manifest_file_size*/,
|
||||
bool /*flush_memtable*/ = true) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::Flush;
|
||||
Status Flush(const FlushOptions& /*options*/,
|
||||
ColumnFamilyHandle* /*column_family*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DBImpl::SyncWAL;
|
||||
Status SyncWAL() override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
using DB::IngestExternalFile;
|
||||
Status IngestExternalFile(
|
||||
ColumnFamilyHandle* /*column_family*/,
|
||||
const std::vector<std::string>& /*external_files*/,
|
||||
const IngestExternalFileOptions& /*ingestion_options*/) override {
|
||||
return Status::NotSupported("Not supported operation in read only mode.");
|
||||
}
|
||||
|
||||
// Try to catch up with the primary by reading as much as possible from the
|
||||
// log files until there is nothing more to read or encounters an error. If
|
||||
// the amount of information in the log files to process is huge, this
|
||||
// method can take long time due to all the I/O and CPU costs.
|
||||
Status TryCatchUpWithPrimary() override;
|
||||
|
||||
private:
|
||||
friend class DB;
|
||||
|
||||
// No copying allowed
|
||||
DBImplSecondary(const DBImplSecondary&);
|
||||
void operator=(const DBImplSecondary&);
|
||||
|
||||
using DBImpl::Recover;
|
||||
|
||||
std::unique_ptr<log::FragmentBufferedReader> manifest_reader_;
|
||||
std::unique_ptr<log::Reader::Reporter> manifest_reporter_;
|
||||
std::unique_ptr<Status> manifest_reader_status_;
|
||||
};
|
||||
} // namespace rocksdb
|
||||
|
||||
#endif // !ROCKSDB_LITE
|
||||
+190
-408
File diff suppressed because it is too large
Load Diff
@@ -42,7 +42,7 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
||||
"Error when reading %s dir\n", dbname.c_str());
|
||||
}
|
||||
std::sort(files.begin(), files.end());
|
||||
for (const std::string& file : files) {
|
||||
for (std::string file : files) {
|
||||
if (!ParseFileName(file, &number, &type)) {
|
||||
continue;
|
||||
}
|
||||
@@ -85,7 +85,7 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
||||
continue;
|
||||
}
|
||||
std::sort(files.begin(), files.end());
|
||||
for (const std::string& file : files) {
|
||||
for (std::string file : files) {
|
||||
if (ParseFileName(file, &number, &type)) {
|
||||
if (type == kTableFile && ++file_num < 10) {
|
||||
file_info.append(file).append(" ");
|
||||
@@ -109,7 +109,7 @@ void DumpDBFileSummary(const ImmutableDBOptions& options,
|
||||
return;
|
||||
}
|
||||
wal_info.clear();
|
||||
for (const std::string& file : files) {
|
||||
for (std::string file : files) {
|
||||
if (ParseFileName(file, &number, &type)) {
|
||||
if (type == kLogFile) {
|
||||
env->GetFileSize(options.wal_dir + "/" + file, &file_size);
|
||||
|
||||
@@ -88,6 +88,7 @@ TEST_F(DBIOFailureTest, DropWritesFlush) {
|
||||
env_->drop_writes_.store(false, std::memory_order_release);
|
||||
} while (ChangeCompactOptions());
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
// Check that CompactRange() returns failure if there is not enough space left
|
||||
// on device
|
||||
@@ -115,7 +116,6 @@ TEST_F(DBIOFailureTest, NoSpaceCompactRange) {
|
||||
env_->no_space_.store(false, std::memory_order_release);
|
||||
} while (ChangeCompactOptions());
|
||||
}
|
||||
#endif // ROCKSDB_LITE
|
||||
|
||||
TEST_F(DBIOFailureTest, NonWritableFileSystem) {
|
||||
do {
|
||||
|
||||
+354
-494
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user