Disable error as warning

Add one more #include<functional>
Add some include<functional>
2026-07-07 14:47:40 +08:00 · 2019-11-05 11:03:59 -08:00 · 2019-11-05 11:03:58 -08:00 · 2019-10-31 14:29:25 -07:00 · 2019-10-31 14:29:24 -07:00 · 2017-07-17 22:22:06 -07:00
714 changed files with 21198 additions and 13124 deletions
@@ -42,6 +42,7 @@ unity.a
 tags
 rocksdb_dump
 rocksdb_undump
+db_test2

 java/out
 java/target
@@ -34,7 +34,7 @@ before_script:
 # as EnvPosixTest::AllocateTest expects within the Travis OpenVZ environment.
 script:
  - if [[ "${TRAVIS_OS_NAME}" == 'linux' ]]; then OPT=-DTRAVIS CLANG_FORMAT_DIFF=/tmp/clang-format-diff.py make format || true; fi
-  - OPT=-DTRAVIS V=1 make -j4 check && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest && make clean && OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 check
+  - OPT=-DTRAVIS V=1 make -j4 check && OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest && make clean && OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib

 notifications:
    email:
@@ -13,8 +13,8 @@
 #        cd build
 # 3. Run cmake to generate project files for Windows, add more options to enable required third-party libraries.
 #    See thirdparty.inc for more information.
-#        sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
-#        OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 ..
+#        sample command: cmake -G "Visual Studio 12 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
+#        OR for VS Studio 15 cmake -G "Visual Studio 14 Win64" -DGFLAGS=1 -DSNAPPY=1 -DJEMALLOC=1 -DJNI=1 ..
 # 4. Then build the project in debug mode (you may want to add /m[:<N>] flag to run msbuild in <N> parallel threads
 #                                          or simply /m ot use all avail cores)
 #        msbuild rocksdb.sln
@@ -30,10 +30,9 @@ cmake_minimum_required(VERSION 2.6)
 project(rocksdb)

 include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
-
-execute_process(COMMAND $ENV{COMSPEC} " /C date /T" OUTPUT_VARIABLE DATE)
-execute_process(COMMAND $ENV{COMSPEC} " /C time /T" OUTPUT_VARIABLE TIME)
-string(REGEX REPLACE "(..)/(..)/..(..).*" "\\1/\\2/\\3" DATE ${DATE})
+execute_process(COMMAND powershell -Command "Get-Date -format MM_dd_yyyy" OUTPUT_VARIABLE DATE)
+execute_process(COMMAND powershell -Command "Get-Date -format HH:mm:ss" OUTPUT_VARIABLE TIME)
+string(REGEX REPLACE "(..)_(..)_..(..).*" "\\1/\\2/\\3" DATE ${DATE})
 string(REGEX REPLACE "(..):(.....).*" " \\1:\\2" TIME ${TIME})
 string(CONCAT GIT_DATE_TIME ${DATE} ${TIME})

@@ -99,6 +98,7 @@ add_subdirectory(third-party/gtest-1.7.0/fused-src/gtest)

 # Main library source code
 set(SOURCES
+        db/auto_roll_logger.cc
        db/builder.cc
        db/c.cc
        db/column_family.cc
@@ -114,6 +114,7 @@ set(SOURCES
        db/db_impl_debug.cc
        db/db_impl_experimental.cc
        db/db_impl_readonly.cc
+        db/db_info_dumper.cc
        db/db_iter.cc
        db/event_helpers.cc
        db/experimental.cc
@@ -145,9 +146,12 @@ set(SOURCES
        db/write_batch_base.cc
        db/write_controller.cc
        db/write_thread.cc
+        db/xfunc_test_points.cc
        memtable/hash_cuckoo_rep.cc
        memtable/hash_linklist_rep.cc
        memtable/hash_skiplist_rep.cc
+        memtable/skiplistrep.cc
+        memtable/vectorrep.cc
        port/stack_trace.cc
        port/win/env_win.cc
        port/win/port_win.cc
@@ -173,7 +177,6 @@ set(SOURCES
        table/merger.cc
        table/sst_file_writer.cc
        table/meta_blocks.cc
-        table/mock_table.cc
        table/plain_table_builder.cc
        table/plain_table_factory.cc
        table/plain_table_index.cc
@@ -182,9 +185,9 @@ set(SOURCES
        table/table_properties.cc
        table/two_level_iterator.cc
        tools/sst_dump_tool.cc
+        tools/db_bench_tool.cc
        tools/dump/db_dump_tool.cc
        util/arena.cc
-        util/auto_roll_logger.cc
        util/bloom.cc
        util/build_version.cc
        util/cache.cc
@@ -193,17 +196,18 @@ set(SOURCES
        util/comparator.cc
        util/concurrent_arena.cc
        util/crc32c.cc
-        util/db_info_dumper.cc
-        util/delete_scheduler_impl.cc
+        util/delete_scheduler.cc
        util/dynamic_bloom.cc
        util/env.cc
        util/env_hdfs.cc
        util/event_logger.cc
        util/file_util.cc
        util/file_reader_writer.cc
+        util/sst_file_manager_impl.cc
        util/filter_policy.cc
        util/hash.cc
        util/histogram.cc
+        util/histogram_windowing.cc
        util/instrumented_mutex.cc
        util/iostats_context.cc
        tools/ldb_cmd.cc
@@ -211,7 +215,6 @@ set(SOURCES
        util/logging.cc
        util/log_buffer.cc
        util/memenv.cc
-        util/mock_env.cc
        util/murmurhash.cc
        util/mutable_cf_options.cc
        util/options.cc
@@ -223,7 +226,6 @@ set(SOURCES
        util/perf_level.cc
        util/random.cc
        util/rate_limiter.cc
-        util/skiplistrep.cc
        util/slice.cc
        util/statistics.cc
        util/status.cc
@@ -237,11 +239,12 @@ set(SOURCES
        util/thread_status_updater.cc
        util/thread_status_util.cc
        util/thread_status_util_debug.cc
-        util/vectorrep.cc
+        util/transaction_test_util.cc
        util/xfunc.cc
        util/xxhash.cc
        utilities/backupable/backupable_db.cc
        utilities/checkpoint/checkpoint.cc
+        utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
        utilities/document/document_db.cc
        utilities/document/json_document.cc
        utilities/document/json_document_builder.cc
@@ -275,6 +278,8 @@ set(SOURCES
 # and linked to tests. Add test only code that is not #ifdefed for Release here.
 set(TESTUTIL_SOURCE
    db/db_test_util.cc
+    table/mock_table.cc
+    util/mock_env.cc
    util/thread_status_updater_debug.cc
 )

@@ -287,8 +292,19 @@ set_target_properties(rocksdb${ARTIFACT_SUFFIX} PROPERTIES COMPILE_FLAGS "-DROCK
 add_dependencies(rocksdb${ARTIFACT_SUFFIX} GenerateBuildVersion)
 target_link_libraries(rocksdb${ARTIFACT_SUFFIX} ${LIBS})

+if (DEFINED JNI)
+  if (${JNI} EQUAL 1)
+    message(STATUS "JNI library is enabled")
+    add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
+  else()
+    message(STATUS "JNI library is disabled")
+  endif()
+else()
+  message(STATUS "JNI library is disabled")
+endif()
+
 set(APPS
-        db/db_bench.cc
+        tools/db_bench.cc
        db/memtablerep_bench.cc
        table/table_reader_bench.cc
        tools/db_stress.cc
@@ -303,6 +319,7 @@ set(APPS
 set(C_TESTS db/c_test.c)

 set(TESTS
+        db/auto_roll_logger_test.cc
        db/column_family_test.cc
        db/compact_files_test.cc
        db/compaction_iterator_test.cc
@@ -312,17 +329,20 @@ set(TESTS
        db/comparator_db_test.cc
        db/corruption_test.cc
        db/cuckoo_table_db_test.cc
-        db/db_iter_test.cc
-        db/db_test.cc
        db/db_compaction_filter_test.cc
        db/db_compaction_test.cc
        db/db_dynamic_level_test.cc
        db/db_inplace_update_test.cc
+        db/db_iter_test.cc
        db/db_log_iter_test.cc
+        db/db_properties_test.cc
+        db/db_table_properties_test.cc
+        db/db_tailing_iter_test.cc
+        db/db_test.cc
+        db/db_test2.cc
+        db/db_block_cache_test.cc
        db/db_universal_compaction_test.cc
        db/db_wal_test.cc
-        db/db_tailing_iter_test.cc
-        db/db_table_properties_test.cc
        db/dbformat_test.cc
        db/deletefile_test.cc
        db/fault_injection_test.cc
@@ -340,6 +360,7 @@ set(TESTS
        db/perf_context_test.cc
        db/plain_table_db_test.cc
        db/prefix_test.cc
+        db/repair_test.cc
        db/skiplist_test.cc
        db/table_properties_collector_test.cc
        db/version_builder_test.cc
@@ -363,7 +384,6 @@ set(TESTS
        tools/sst_dump_test.cc
        util/arena_test.cc
        util/autovector_test.cc
-        util/auto_roll_logger_test.cc
        util/bloom_test.cc
        util/cache_test.cc
        util/coding_test.cc
@@ -376,8 +396,10 @@ set(TESTS
        util/file_reader_writer_test.cc
        util/heap_test.cc
        util/histogram_test.cc
+        util/iostats_context_test.cc
        util/memenv_test.cc
        util/mock_env_test.cc
+        util/options_settable_test.cc
        util/options_test.cc
        util/rate_limiter_test.cc
        util/slice_transform_test.cc
@@ -0,0 +1,9 @@
+# RocksDB default options change log
+## 4.7.0 (4/8/2016)
+* options.write_buffer_size changes from 4MB to 64MB
+* options.target_file_size_base changes from 2MB to 64MB
+* options.max_bytes_for_level_base changes from 10MB to 256MB
+* options.soft_pending_compaction_bytes_limit changes from 0 (disabled) to 64GB
+* options.hard_pending_compaction_bytes_limit changes from 0 (disabled) to 256GB
+* table_cache_numshardbits changes from 4 to 6
+* max_file_opening_threads changes from 1 to 16 
@@ -1,4 +1,29 @@
 # Rocksdb Change Log
+## 4.7.0 (4/8/2016)
+### Public API Change
+* rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
+* Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
+
+## 4.6.0 (3/10/2016)
+### Public API Changes
+* Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
+* Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signiture of Cache::Insert() is updated accordingly.
+* Tickers [NUMBER_DB_NEXT, NUMBER_DB_PREV, NUMBER_DB_NEXT_FOUND, NUMBER_DB_PREV_FOUND, ITER_BYTES_READ] are not updated immediately. The are updated when the Iterator is deleted.  
+* Add monotonically increasing counter (DB property "rocksdb.current-super-version-number") that increments upon any change to the LSM tree.
+### New Features
+* Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
+* Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
+
+## 4.5.0 (2/5/2016)
+### Public API Changes
+* Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
+* Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
+* DBOptions::delete_scheduler and NewDeleteScheduler() are removed, please use DBOptions::sst_file_manager and NewSstFileManager() instead
+
+### New Features
+* ldb tool now supports operations to non-default column families.
+* Add kPersistedTier to ReadTier.  This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
+* Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.  

 ## 4.4.0 (1/14/2016)
 ### Public API Changes
@@ -21,7 +21,7 @@ depend on gflags. You will need to have gflags installed to run `make all`. This
 use binaries compiled by `make all` in production.

 * By default the binary we produce is optimized for the platform you're compiling on
-(-march=native). If you want to build a portable binary, add 'PORTABLE=1' before
+(-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
 your make commands, like this: `PORTABLE=1 make static_lib`

 ## Dependencies
@@ -0,0 +1,11 @@
+This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
+
+* Java - https://github.com/facebook/rocksdb/tree/master/java
+* Python - http://pyrocksdb.readthedocs.org/en/latest/
+* Perl - https://metacpan.org/pod/RocksDB
+* Node.js - https://npmjs.org/package/rocksdb
+* Go - https://github.com/tecbot/gorocksdb
+* Ruby - http://rubygems.org/gems/rocksdb-ruby
+* Haskell - https://hackage.haskell.org/package/rocksdb-haskell
+* PHP - https://github.com/Photonios/rocksdb-php
+* C# - https://github.com/warrenfalk/rocksdb-sharp
@@ -2,7 +2,7 @@ BSD License

 For rocksdb software

-Copyright (c) 2014, Facebook, Inc.
+Copyright (c) 2011-present, Facebook, Inc.
 All rights reserved.
 ---------------------------------------------------------------------

@@ -84,7 +84,8 @@ endif
 # compile with -O2 if debug level is not 2
 ifneq ($(DEBUG_LEVEL), 2)
 OPT += -O2 -fno-omit-frame-pointer
-ifneq ($(MACHINE),ppc64) # ppc64 doesn't support -momit-leaf-frame-pointer
+# Skip for archs that don't support -momit-leaf-frame-pointer
+ifeq (,$(shell $(CXX) -fsyntax-only -momit-leaf-frame-pointer -xc /dev/null 2>&1))
 OPT += -momit-leaf-frame-pointer
 endif
 endif
@@ -143,6 +144,9 @@ else
 OPT += -DNDEBUG
 endif

+ifeq ($(PLATFORM), OS_SOLARIS)
+	PLATFORM_CXXFLAGS += -D _GLIBCXX_USE_C99
+endif
 ifneq ($(filter -DROCKSDB_LITE,$(OPT)),)
 	# found
 	CFLAGS += -fno-exceptions
@@ -170,6 +174,15 @@ else
 	pg = -pg
 endif

+# USAN doesn't work well with jemalloc. If we're compiling with USAN, we should use regular malloc.
+ifdef COMPILE_WITH_UBSAN
+	DISABLE_JEMALLOC=1
+	EXEC_LDFLAGS += -fsanitize=undefined
+	PLATFORM_CCFLAGS += -fsanitize=undefined
+	PLATFORM_CXXFLAGS += -fsanitize=undefined
+endif
+
+
 ifndef DISABLE_JEMALLOC
 	EXEC_LDFLAGS := $(JEMALLOC_LIB) $(EXEC_LDFLAGS)
 	PLATFORM_CXXFLAGS += $(JEMALLOC_INCLUDE)
@@ -187,10 +200,6 @@ default: all
 WARNING_FLAGS = -W -Wextra -Wall -Wsign-compare -Wshadow \
  -Wno-unused-parameter

-ifndef DISABLE_WARNING_AS_ERROR
-	WARNING_FLAGS += -Werror
-endif
-
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers

@@ -237,8 +246,12 @@ VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)

 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full

+BENCHTOOLOBJECTS = $(BENCH_SOURCES:.cc=.o) $(LIBOBJECTS) $(TESTUTIL)
+
 TESTS = \
 	db_test \
+	db_test2 \
+	db_block_cache_test \
 	db_iter_test \
 	db_log_iter_test \
 	db_compaction_filter_test \
@@ -248,6 +261,7 @@ TESTS = \
 	db_tailing_iter_test \
 	db_universal_compaction_test \
 	db_wal_test \
+	db_properties_test \
 	db_table_properties_test \
 	block_hash_index_test \
 	autovector_test \
@@ -312,6 +326,7 @@ TESTS = \
 	rate_limiter_test \
 	delete_scheduler_test \
 	options_test \
+	options_settable_test \
 	options_util_test \
 	event_logger_test \
 	cuckoo_table_builder_test \
@@ -332,7 +347,8 @@ TESTS = \
 	compact_on_deletion_collector_test \
 	compaction_job_stats_test \
 	transaction_test \
-	ldb_cmd_test
+	ldb_cmd_test \
+	iostats_context_test

 SUBSET :=  $(shell echo $(TESTS) |sed s/^.*$(ROCKSDBTESTS_START)/$(ROCKSDBTESTS_START)/)

@@ -460,7 +476,7 @@ test_names = \
      -e '/^(\s*)(\S+)/; !$$1 and do {$$p=$$2; break};'			\
      -e 'print qq! $$p$$2!'

-ifeq ($(MAKECMDGOALS),check)
+ifneq (,$(filter check parallel_check,$(MAKECMDGOALS)),)
 # Use /dev/shm if it has the sticky bit set (otherwise, /tmp),
 # and create a randomly-named rocksdb.XXXX directory therein.
 # We'll use that directory in the "make check" rules.
@@ -590,7 +606,7 @@ ldb_tests: ldb
 crash_test: whitebox_crash_test blackbox_crash_test

 blackbox_crash_test: db_stress
-	python -u tools/db_crashtest.py --simple blackbox 
+	python -u tools/db_crashtest.py --simple blackbox
 	python -u tools/db_crashtest.py blackbox

 whitebox_crash_test: db_stress
@@ -607,8 +623,18 @@ asan_crash_test:
 	COMPILE_WITH_ASAN=1 $(MAKE) crash_test
 	$(MAKE) clean

+ubsan_check:
+	$(MAKE) clean
+	COMPILE_WITH_UBSAN=1 $(MAKE) check -j32
+	$(MAKE) clean
+
+ubsan_crash_test:
+	$(MAKE) clean
+	COMPILE_WITH_UBSAN=1 $(MAKE) crash_test
+	$(MAKE) clean
+
 valgrind_check: $(TESTS)
-	for t in $(filter-out %skiplist_test,$(TESTS)); do \
+	for t in $(filter-out %skiplist_test options_settable_test,$(TESTS)); do \
 		$(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \
 		ret_code=$$?; \
 		if [ $$ret_code -ne 0 ]; then \
@@ -616,6 +642,46 @@ valgrind_check: $(TESTS)
 		fi; \
 	done

+
+ifneq ($(PAR_TEST),)
+parloop:
+	ret_bad=0;							\
+	for t in $(PAR_TEST); do		\
+		echo "===== Running $$t in parallel $(NUM_PAR)";\
+		if [ $(db_test) -eq 1 ]; then \
+			seq $(J) | v="$$t" parallel --gnu 's=$(TMPD)/rdb-{};  export TEST_TMPDIR=$$s;' \
+				'timeout 2m ./db_test --gtest_filter=$$v >> $$s/log-{} 2>1'; \
+		else\
+			seq $(J) | v="./$$t" parallel --gnu 's=$(TMPD)/rdb-{};' \
+			     'export TEST_TMPDIR=$$s; timeout 10m $$v >> $$s/log-{} 2>1'; \
+		fi; \
+		ret_code=$$?; \
+		if [ $$ret_code -ne 0 ]; then \
+			ret_bad=$$ret_code; \
+			echo $$t exited with $$ret_code; \
+		fi; \
+	done; \
+	exit $$ret_bad;
+endif
+
+parallel_check: $(TESTS)
+	$(AM_V_GEN)if test "$(J)" > 1                                  \
+	    && (parallel --gnu --help 2>/dev/null) |                    \
+	        grep -q 'GNU Parallel';                                 \
+	then                                                            \
+	    echo Running in parallel $(J);			\
+	else                                                            \
+	    echo "Need to have GNU Parallel and J > 1"; exit 1;		\
+	fi;								\
+	ret_bad=0;							\
+	echo $(J);\
+	echo Test Dir: $(TMPD); \
+        seq $(J) | parallel --gnu 's=$(TMPD)/rdb-{}; rm -rf $$s; mkdir $$s'; \
+	$(MAKE)  PAR_TEST="$(shell $(test_names))" TMPD=$(TMPD) \
+		J=$(J) db_test=1 parloop; \
+	$(MAKE) PAR_TEST="$(filter-out db_test, $(TESTS))" \
+		TMPD=$(TMPD) J=$(J) db_test=0 parloop;
+
 analyze: clean
 	$(CLANG_SCAN_BUILD) --use-analyzer=$(CLANG_ANALYZER) \
 		--use-c++=$(CXX) --use-cc=$(CC) --status-bugs \
@@ -652,7 +718,7 @@ clean:

 tags:
 	ctags * -R
-	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
+	cscope -b `find . -name '*.cc'` `find . -name '*.h'` `find . -name '*.c'`

 format:
 	build_tools/format-diff.sh
@@ -667,7 +733,7 @@ $(LIBRARY): $(LIBOBJECTS)
 	$(AM_V_AR)rm -f $@
 	$(AM_V_at)$(AR) $(ARFLAGS) $@ $(LIBOBJECTS)

-db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
+db_bench: tools/db_bench.o $(BENCHTOOLOBJECTS)
 	$(AM_LINK)

 cache_bench: util/cache_bench.o $(LIBOBJECTS) $(TESTUTIL)
@@ -742,6 +808,12 @@ slice_transform_test: util/slice_transform_test.o $(LIBOBJECTS) $(TESTHARNESS)
 db_test: db/db_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+db_test2: db/db_test2.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
+db_block_cache_test: db/db_block_cache_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 db_log_iter_test: db/db_log_iter_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -769,6 +841,9 @@ db_universal_compaction_test: db/db_universal_compaction_test.o db/db_test_util.
 db_wal_test: db/db_wal_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+db_properties_test: db/db_properties_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 db_table_properties_test: db/db_table_properties_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -946,6 +1021,9 @@ compact_files_test: db/compact_files_test.o $(LIBOBJECTS) $(TESTHARNESS)
 options_test: util/options_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

+options_settable_test: util/options_settable_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 options_util_test: utilities/options/options_util_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

@@ -970,7 +1048,7 @@ manual_compaction_test: db/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

-auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
+auto_roll_logger_test: db/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 memtable_list_test: db/memtable_list_test.o $(LIBOBJECTS) $(TESTHARNESS)
@@ -988,12 +1066,18 @@ transaction_test: utilities/transactions/transaction_test.o $(LIBOBJECTS) $(TEST
 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
 	$(AM_LINK)

+repair_test: db/repair_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 ldb_cmd_test: tools/ldb_cmd_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)

 ldb: tools/ldb.o $(LIBOBJECTS)
 	$(AM_LINK)

+iostats_context_test: util/iostats_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_V_CCLD)$(CXX) $^ $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+
 #-------------------------------------------------
 # make install related stuff
 INSTALL_PATH ?= /usr/local
@@ -1036,7 +1120,11 @@ install: install-static
 # ---------------------------------------------------------------------------

 JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
-ARCH := $(shell getconf LONG_BIT)
+ifeq ($(PLATFORM), OS_SOLARIS)
+	ARCH := $(shell isainfo -b)
+else
+	ARCH := $(shell getconf LONG_BIT)
+endif
 ROCKSDBJNILIB = librocksdbjni-linux$(ARCH).so
 ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-linux$(ARCH).jar
 ROCKSDB_JAR_ALL = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH).jar
@@ -1044,14 +1132,19 @@ ROCKSDB_JAVADOCS_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PA
 ROCKSDB_SOURCES_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-sources.jar

 ifeq ($(PLATFORM), OS_MACOSX)
-ROCKSDBJNILIB = librocksdbjni-osx.jnilib
-ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
+	ROCKSDBJNILIB = librocksdbjni-osx.jnilib
+	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-osx.jar
 ifneq ("$(wildcard $(JAVA_HOME)/include/darwin)","")
 	JAVA_INCLUDE = -I$(JAVA_HOME)/include -I $(JAVA_HOME)/include/darwin
 else
 	JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
 endif
 endif
+ifeq ($(PLATFORM), OS_SOLARIS)
+	ROCKSDBJNILIB = librocksdbjni-solaris$(ARCH).so
+	ROCKSDB_JAR = rocksdbjni-$(ROCKSDB_MAJOR).$(ROCKSDB_MINOR).$(ROCKSDB_PATCH)-solaris$(ARCH).jar
+	JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/solaris
+endif

 libz.a:
 	-rm -rf zlib-1.2.8
@@ -1145,11 +1238,10 @@ jtest: rocksdbjava
 jdb_bench:
 	cd java;$(MAKE) db_bench;

-commit-prereq:
-	$(MAKE) clean && $(MAKE) all check;
+commit_prereq: build_tools/rocksdb-lego-determinator \
+               build_tools/precommit_checker.py
+	J=$(J) build_tools/precommit_checker.py unit unit_481 clang_unit tsan asan ubsan lite
 	$(MAKE) clean && $(MAKE) jclean && $(MAKE) rocksdbjava;
-	$(MAKE) clean && USE_CLANG=1 $(MAKE) all;
-	$(MAKE) clean && OPT=-DROCKSDB_LITE $(MAKE) static_lib;

 xfunc:
 	for xftest in $(XFUNC_TESTS); do \
@@ -45,4 +45,14 @@ Airbnb is using RocksDB as a storage engine for their personalized search servic
 Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo

 ## Smyte
-[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services.
+[Smyte](https://www.smyte.com/) uses RocksDB as the storage layer for their core key-value storage, high-performance counters and time-windowed HyperLogLog services.
+
+## Rakuten Marketing
+[Rakuten Marketing](https://marketing.rakuten.com/) uses RocksDB as the disk cache layer for the real-time bidding service in their Performance DSP.
+
+## VWO, Wingify
+[VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
+
+## quasardb
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
+quasardb uses a heavily tuned RocksDB as its persistence layer.
@@ -10,13 +10,184 @@ class FacebookArcanistConfiguration extends ArcanistConfiguration {
                                 ArcanistBaseWorkflow $workflow,
                                 $error_code) {
    if ($command == 'diff' && !$workflow->isRawDiffSource()) {
-      $this->maybePushToJenkins($workflow);
+      $this->startTestsInJenkins($workflow);
+      $this->startTestsInSandcastle($workflow);
    }
  }

+  //////////////////////////////////////////////////////////////////////
+  /*  Run tests in sandcastle */
+  function postURL($diffID, $url) {
+    $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
+           . '"name":"click here for sandcastle tests for D' . $diffID . '", '
+           . '"link":"' . $url . '"}\' | '
+           . 'http_proxy=fwdproxy.any.facebook.com:8080 '
+           . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit '
+           . 'differential.updateunitresults';
+    shell_exec($cmd);
+  }
+
+  function updateTestCommand($diffID, $test, $status) {
+    $cmd = 'echo \'{"diff_id": "' . $diffID . '", '
+           . '"name":"' . $test . '", '
+           . '"result":"' . $status . '"}\' | '
+           . 'http_proxy=fwdproxy.any.facebook.com:8080 '
+           . 'https_proxy=fwdproxy.any.facebook.com:8080 arc call-conduit '
+           . 'differential.updateunitresults';
+    return $cmd;
+  }
+
+  function updateTest($diffID, $test) {
+    shell_exec($this->updateTestCommand($diffID, $test, "waiting"));
+  }
+
+  function getSteps($diffID, $username, $test) {
+    $arcrc_content = exec("cat ~/.arcrc | gzip -f | base64 -w0");
+
+    // Sandcastle machines don't have arc setup. We copy the user certificate
+    // and authenticate using that in sandcastle
+    $setup = array(
+      "name" => "Setup arcrc",
+      "shell" => "echo " . $arcrc_content . " | base64 --decode"
+                 . " | gzip -d > ~/.arcrc",
+      "user" => "root"
+    );
+
+    // arc demands certain permission on its config
+    $fix_permission = array(
+      "name" => "Fix environment",
+      "shell" => "chmod 600 ~/.arcrc",
+      "user" => "root"
+    );
+
+    // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
+    // git thinks it is uncommited change
+    $fix_git_ignore = array(
+      "name" => "Fix git ignore",
+      "shell" => "echo fbcode >> .git/info/exclude",
+      "user" => "root"
+    );
+
+    // Patch the code (keep your fingures crossed)
+    $patch = array(
+      "name" => "Patch " . $diffID,
+      "shell" => "HTTPS_PROXY=fwdproxy:8080 arc --arcrc-file ~/.arcrc "
+                  . "patch --diff " . $diffID,
+      "user" => "root"
+    );
+
+    // Clean up the user arc config we are using
+    $cleanup = array(
+      "name" => "Arc cleanup",
+      "shell" => "rm -f ~/.arcrc",
+      "user" => "root"
+    );
+
+    // Construct the steps in the order of execution
+    $steps[] = $setup;
+    $steps[] = $fix_permission;
+    $steps[] = $fix_git_ignore;
+    $steps[] = $patch;
+
+    // Run the actual command
+    $this->updateTest($diffID, $test);
+    $cmd = $this->updateTestCommand($diffID, $test, "running") . ";"
+           . "(./build_tools/precommit_checker.py " . $test
+           . "&& "
+           . $this->updateTestCommand($diffID, $test, "pass") . ")"
+           . "|| " . $this->updateTestCommand($diffID, $test, "fail")
+           . "; cat /tmp/precommit-check.log"
+           . "; for f in `ls t/log-*`; do echo \$f; cat \$f; done";
+
+    $run_test = array(
+      "name" => "Run " . $test,
+      "shell" => $cmd,
+      "user" => "root",
+    );
+
+    $steps[] = $run_test;
+    $steps[] = $cleanup;
+
+    return $steps;
+  }
+
+  function startTestsInSandcastle($workflow) {
+    // extract information we need from workflow or CLI
+    $diffID = $workflow->getDiffId();
+    $username = exec("whoami");
+
+    if ($diffID == null || $username == null) {
+      // there is no diff and we can't extract username
+      // we cannot schedule sandcasstle job
+      return;
+    }
+
+    if (strcmp(getenv("ROCKSDB_CHECK_ALL"), 1) == 0) {
+      // extract all tests from the CI definition
+      $output = file_get_contents("build_tools/rocksdb-lego-determinator");
+      preg_match_all('/[ ]{2}([a-zA-Z0-9_]+)[\)]{1}/', $output, $matches);
+      $tests = $matches[1];
+    } else {
+      // manually list of tests we want to run in sandcastle
+      $tests = array(
+        "unit", "unit_481", "clang_unit", "tsan", "asan", "lite", "valgrind"
+      );
+    }
+
+    // construct a job definition for each test and add it to the master plan
+    foreach ($tests as $test) {
+      $arg[] = array(
+        "name" => "RocksDB diff " . $diffID . " test " . $test,
+        "steps" => $this->getSteps($diffID, $username, $test)
+      );
+    }
+
+    // we cannot submit the parallel execution master plan to sandcastle
+    // we need supply the job plan as a determinator
+    // so we construct a small job that will spit out the master job plan
+    // which sandcastle will parse and execute
+    // Why compress ? Otherwise we run over the max string size.
+    $cmd = "echo " . base64_encode(json_encode($arg))
+           . " | gzip -f | base64 -w0";
+    $arg_encoded = shell_exec($cmd);
+
+    $command = array(
+      "name" => "Run diff " . $diffID . "for user " . $username,
+      "steps" => array()
+    );
+
+    $command["steps"][] = array(
+      "name" => "Generate determinator",
+      "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
+                 . " | base64 --decode",
+      "determinator" => true,
+      "user" => "root"
+    );
+
+    // submit to sandcastle
+    $url = 'https://interngraph.intern.facebook.com/sandcastle/generate?'
+            .'command=SandcastleUniversalCommand'
+            .'&vcs=rocksdb-git&revision=origin%2Fmaster&type=lego'
+            .'&user=krad&alias=rocksdb-precommit'
+            .'&command-args=' . urlencode(json_encode($command));
+
+    $cmd = 'https_proxy= HTTPS_PROXY= curl -s -k -F app=659387027470559 '
+            . '-F token=AeO_3f2Ya3TujjnxGD4 "' . $url . '"';
+
+    $output = shell_exec($cmd);
+
+    // extract sandcastle URL from the response
+    preg_match('/url": "(.+)"/', $output, $sandcastle_url);
+
+    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
+
+    // Ask phabricator to display it on the diff UI
+    $this->postURL($diffID, $sandcastle_url[1]);
+  }
+
  //////////////////////////////////////////////////////////////////////
  /* Send off builds to jenkins */
-  function maybePushToJenkins($workflow) {
+  function startTestsInJenkins($workflow) {
    $diffID = $workflow->getDiffID();
    if ($diffID === null) {
      return;
@@ -31,5 +202,4 @@ class FacebookArcanistConfiguration extends ArcanistConfiguration {
               ."buildWithParameters?token=AUTH&DIFF_ID=$diffID";
    system("curl --noproxy '*' \"$url\" > /dev/null 2>&1");
  }
-
 }
@@ -1,5 +1,5 @@
 #!/usr/bin/python
-# Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+# Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 # This source code is licensed under the BSD-style license found in the
 # LICENSE file in the root directory of this source tree. An additional grant
 # of patent rights can be found in the PATENTS file in the same directory.
@@ -52,12 +52,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
    FBCODE_BUILD="true"
    # If we're compiling with TSAN we need pic build
    PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    else
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    fi
+    source "$PWD/build_tools/fbcode_config.sh"
 fi

 # Delete existing output, if it exists
@@ -189,17 +184,19 @@ if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
    # Also don't need any compilation tests if compiling on fbcode
    true
 else
-    # Test whether fallocate is available
-    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-      #include <fcntl.h>
-      #include <linux/falloc.h>
-      int main() {
-	int fd = open("/dev/null", 0);
-  fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
-      }
+    if ! test $ROCKSDB_DISABLE_FALLOCATE; then
+        # Test whether fallocate is available
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <fcntl.h>
+          #include <linux/falloc.h>
+          int main() {
+      int fd = open("/dev/null", 0);
+      fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
+          }
 EOF
-    if [ "$?" = 0 ]; then
-        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_FALLOCATE_PRESENT"
+        fi
    fi

    # Test whether Snappy library is installed
@@ -359,7 +356,15 @@ if test "$USE_SSE"; then
  # if Intel SSE instruction set is supported, set USE_SSE=1
  COMMON_FLAGS="$COMMON_FLAGS -msse -msse4.2 "
 elif test -z "$PORTABLE"; then
-  COMMON_FLAGS="$COMMON_FLAGS -march=native "
+  if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
+    # Tune for this POWER processor, treating '+' models as base models
+    POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
+    COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
+  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
+    COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
+  else
+    COMMON_FLAGS="$COMMON_FLAGS -march=native "
+  fi
 fi

 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
@@ -0,0 +1,19 @@
+# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
+NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832
@@ -0,0 +1,16 @@
+GCC_BASE=/mnt/gvfs/third-party2/gcc/ebc96bc2fb751b5a0300b8d91a95bdf24ac1d88b/4.8.1/centos6-native/cc6c9dc
+CLANG_BASE=/mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/9d9ecb9/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/f97108c12512b3b0789ac4515d836bdb1eae1142/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f5484f168c0e4d19823d41df052c5870c6e575a4/2.17/gcc-4.8.1-glibc-2.17/99df8fc
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/cbf6f1f209e5bd160bdc5d971744e039f36b1566/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/6d39cb54708049f527e713ad19f2aadb9d3667e8/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/2ddd45f0853bfc8bb1c27f0f447236a1a26c338a/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.8.1-glibc-2.17/c3f970a
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/cb6c4880fcb4fee471574ba6af63a3882155a16a/0.5.1/gcc-4.8.1-glibc-2.17/c3f970a
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/c7275a4ceae0aca0929e56964a31dafc53c1ee96/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9acac8a9a2c9378eb696e22ffa8bd0e97d9206b/master/gcc-4.8.1-glibc-2.17/8d31e51
+NUMA_BASE=/mnt/gvfs/third-party2/numa/ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/41bfa3759ce52c071f5fd547ec9ecd2522929f0a/trunk/gcc-4.8.1-glibc-2.17/675d945
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/71454c53fffcb716a0beb9a90047aff7fb5c984a/2.26/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/af85c56f424cd5edfc2c97588299b44ecdec96bb/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
@@ -6,128 +6,147 @@
 # Environment variables that change the behavior of this script:
 # PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included

+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies.sh"
+
 CFLAGS=""

-# location of libgcc
-LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/0473c80518a10d6efcbe24c5eeca3fb4ec9b519c/4.9.x/gcc-4.9-glibc-2.20/e1a7e4e"
-LIBGCC_INCLUDE="$LIBGCC_BASE/include"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"
+# libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"

-# location of glibc
-GLIBC_REV=7397bed99280af5d9543439cdb7d018af7542720
-GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/include"
-GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.20/gcc-4.9-glibc-2.20/99df8fc/lib"
-
-SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/include/"
+# glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"

+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
 if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy.a"
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
 else
-  SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/b0f269b3ca47770121aa159b99e1d8d2ab260e1f/1.0.3/gcc-4.9-glibc-2.20/c32916f/lib/libsnappy_pic.a"
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
 fi
-
 CFLAGS+=" -DSNAPPY"

 if test -z $PIC_BUILD; then
  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/include/"
-  ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/feb983d9667f4cf5e9da07ce75abc824764b67a1/1.2.8/gcc-4.9-glibc-2.20/4230243/lib/libz.a"
+  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
  CFLAGS+=" -DZLIB"

  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/include/"
-  BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/af004cceebb2dfd173ca29933ea5915e727aad2f/1.0.6/gcc-4.9-glibc-2.20/4230243/lib/libbz2.a"
+  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
  CFLAGS+=" -DBZIP2"

-  LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/include/"
-  LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/6858fac689e0f92e584224d91bdb0e39f6c8320d/r131/gcc-4.9-glibc-2.20/e9936bf/lib/liblz4.a"
+  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
  CFLAGS+=" -DLZ4"
-
-  ZSTD_REV=810b81b4705def5243e998b54701f3c504e4009e
-  ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/include"
-  ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
-  CFLAGS+=" -DZSTD"
 fi

+ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
+if test -z $PIC_BUILD; then
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+else
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
+fi
+CFLAGS+=" -DZSTD"
+
 # location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/include/"
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
 if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags.a"
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
 else
-  GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/0fa60e2b88de3e469db6c482d6e6dac72f5d65f9/1.6/gcc-4.9-glibc-2.20/4230243/lib/libgflags_pic.a"
+  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
 fi
-CFLAGS+=" -DGFLAGS=google"
+CFLAGS+=" -DGFLAGS=gflags"

 # location of jemalloc
-JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/include/"
-JEMALLOC_LIB=" /mnt/gvfs/third-party2/jemalloc/bcd68e5e419efa4e61b9486d6854564d6d75a0b5/3.6.0/gcc-4.9-glibc-2.20/2aafc78/lib/libjemalloc.a"
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
+JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"

 if test -z $PIC_BUILD; then
  # location of numa
-  NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/include/"
-  NUMA_LIB=" /mnt/gvfs/third-party2/numa/bbefc39ecbf31d0ca184168eb613ef8d397790ee/2.0.8/gcc-4.9-glibc-2.20/4230243/lib/libnuma.a"
+  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
  CFLAGS+=" -DNUMA"

  # location of libunwind
-  LIBUNWIND="/mnt/gvfs/third-party2/libunwind/1de3b75e0afedfe5585b231bbb340ec7a1542335/1.1/gcc-4.9-glibc-2.20/34235e8/lib/libunwind.a"
+  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
 fi

+# location of TBB
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+if test -z $PIC_BUILD; then
+  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+else
+  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
+fi
+CFLAGS+=" -DTBB"
+
 # use Intel SSE support for checksum calculations
 export USE_SSE=1

-BINUTILS="/mnt/gvfs/third-party2/binutils/0b6ad0c88ddd903333a48ae8bff134efac468e4a/2.25/centos6-native/da39a3e/bin"
+BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"

-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"

-GCC_BASE="/mnt/gvfs/third-party2/gcc/1c67a0b88f64d4d9ced0382d141c76aaa7d62fba/4.9.x/centos6-native/1317bc4"
 STDLIBS="-L $GCC_BASE/lib64"

-CLANG_BASE="/mnt/gvfs/third-party2/clang/d81444dd214df3d2466734de45bb264a0486acc3/dev"
-CLANG_BIN="$CLANG_BASE/centos6-native/af4b1a0/bin"
+CLANG_BIN="$CLANG_BASE/bin"
+CLANG_LIB="$CLANG_BASE/lib"
+CLANG_SRC="$CLANG_BASE/../../src"
+
 CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_BASE/src/clang/tools/scan-build/scan-build"
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"

 if [ -z "$USE_CLANG" ]; then
  # gcc
  CC="$GCC_BASE/bin/gcc"
  CXX="$GCC_BASE/bin/g++"
-  
+
  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  JEMALLOC=1
 else
-  # clang 
-  CLANG_INCLUDE="$CLANG_BASE/gcc-4.9-glibc-2.20/74c386f/lib/clang/dev/include/"
+  # clang
+  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
  CC="$CLANG_BIN/clang"
  CXX="$CLANG_BIN/clang++"

-  KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/ffd14f660a43c4b92717986b1bba66722ef089d0/3.2.18_70_fbk11_00129_gc8882d0/gcc-4.9-glibc-2.20/da39a3e/include"
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"

  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.9.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
  CFLAGS+=" -isystem $CLANG_INCLUDE"
  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+  CFLAGS+=" -Wno-expansion-to-defined "
  CXXFLAGS="-nostdinc++"
 fi

 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"

-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.9-glibc-2.20/lib/ld.so"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -B$BINUTILS/gold"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.9-glibc-2.20/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"

 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"

-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS"

-VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/6c45ef049cbf11c2df593addb712cd891049e737/3.10.0/gcc-4.9-glibc-2.20/4230243/bin/"
+VALGRIND_VER="$VALGRIND_BASE/bin/"

-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 
@@ -4,85 +4,77 @@
 # fbcode settings.  It uses the latest g++ compiler and also
 # uses jemalloc

+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies_4.8.1.sh"
+
 # location of libgcc
-LIBGCC_BASE="/mnt/gvfs/third-party2/libgcc/d00277f4559e261ed0a81f30f23c0ce5564e359e/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc"
 LIBGCC_INCLUDE="$LIBGCC_BASE/include"
 LIBGCC_LIBS=" -L $LIBGCC_BASE/libs"

 # location of glibc
-GLIBC_REV=0600c95b31226b5e535614c590677d87c62d8016
-GLIBC_INCLUDE="/mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/include"
-GLIBC_LIBS=" -L /mnt/gvfs/third-party2/glibc/$GLIBC_REV/2.17/gcc-4.8.1-glibc-2.17/99df8fc/lib"
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"

 # location of snappy headers and libraries
-SNAPPY_REV=cbf6f1f209e5bd160bdc5d971744e039f36b1566
-SNAPPY_INCLUDE=" -I /mnt/gvfs/third-party2/snappy/$SNAPPY_REV/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a/include"
-SNAPPY_LIBS=" /mnt/gvfs/third-party2/snappy/$SNAPPY_REV/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a/lib/libsnappy.a"
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
+SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"

 # location of zlib headers and libraries
-ZLIB_REV=6d39cb54708049f527e713ad19f2aadb9d3667e8
-ZLIB_INCLUDE=" -I /mnt/gvfs/third-party2/zlib/$ZLIB_REV/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a/include"
-ZLIB_LIBS=" /mnt/gvfs/third-party2/zlib/$ZLIB_REV/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libz.a"
+ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
+ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"

 # location of bzip headers and libraries
-BZIP_REV=d6c789bfc2ec4c51a63d66df2878926b8158cde8
-BZIP_INCLUDE=" -I /mnt/gvfs/third-party2/bzip2/$BZIP_REV/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/include/"
-BZIP_LIBS=" /mnt/gvfs/third-party2/bzip2/$BZIP_REV/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a/lib/libbz2.a"
+BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
+BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"

-LZ4_REV=6858fac689e0f92e584224d91bdb0e39f6c8320d
-LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r131/gcc-4.8.1-glibc-2.17/c3f970a/include"
-LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r131/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
+LZ4_INCLUDE=" -I $LZ4_BASE/include"
+LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"

-ZSTD_REV=810b81b4705def5243e998b54701f3c504e4009e
-ZSTD_INCLUDE=" -I /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/include"
-ZSTD_LIBS=" /mnt/gvfs/third-party2/zstd/$ZSTD_REV/0.4.2/gcc-4.8.1-glibc-2.17/c3f970a/lib/libzstd.a"
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
+ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"

 # location of gflags headers and libraries
-GFLAGS_REV=c7275a4ceae0aca0929e56964a31dafc53c1ee96
-GFLAGS_INCLUDE=" -I /mnt/gvfs/third-party2/gflags/$GFLAGS_REV/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a/include/"
-GFLAGS_LIBS=" /mnt/gvfs/third-party2/gflags/$GFLAGS_REV/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a/lib/libgflags.a"
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"

 # location of jemalloc
-JEMALLOC_REV=c370265e58c4b6602e798df23335a1e9913dae52
-JEMALLOC_INCLUDE=" -I /mnt/gvfs/third-party2/jemalloc/$JEMALLOC_REV/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51/include"
-JEMALLOC_LIB="/mnt/gvfs/third-party2/jemalloc/$JEMALLOC_REV/4.0.3/gcc-4.8.1-glibc-2.17/8d31e51/lib/libjemalloc.a"
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
+JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"

 # location of numa
-NUMA_REV=ae54a5ed22cdabb1c6446dce4e8ffae5b4446d73
-NUMA_INCLUDE=" -I /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/include/"
-NUMA_LIB=" /mnt/gvfs/third-party2/numa/$NUMA_REV/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a/lib/libnuma.a"
+NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"

 # location of libunwind
-LIBUNWIND_REV=121f1a75c4414683aea8c70b761bfaf187f7c1a3
-LIBUNWIND="/mnt/gvfs/third-party2/libunwind/$LIBUNWIND_REV/trunk/gcc-4.8.1-glibc-2.17/675d945/lib/libunwind.a"
+LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"

 # use Intel SSE support for checksum calculations
 export USE_SSE=1

-BINUTILS="/mnt/gvfs/third-party2/binutils/75670d0d8ef4891fd1ec2a7513ef01cd002c823b/2.25/centos6-native/da39a3e/bin"
+BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"

-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE"

-GCC_BASE="/mnt/gvfs/third-party2/gcc/c0064002d2609ab649603f769f0bd110bbe48029/4.8.1/centos6-native/cc6c9dc"
 STDLIBS="-L $GCC_BASE/lib64"

 if [ -z "$USE_CLANG" ]; then
  # gcc
  CC="$GCC_BASE/bin/gcc"
  CXX="$GCC_BASE/bin/g++"
-  
+
  CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
  CFLAGS+=" -isystem $GLIBC_INCLUDE"
  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
 else
-  # clang 
-  CLANG_BASE="/mnt/gvfs/third-party2/clang/ab054e9a490a8fd4537c0b6ec56e5c91c0f81c91/3.7"
-  CLANG_INCLUDE="$CLANG_BASE/gcc-4.8.1-glibc-2.17/ee9b060/lib/clang/3.7/include"
-  CC="$CLANG_BASE/centos6-native/b2feaee/bin/clang"
-  CXX="$CLANG_BASE/centos6-native/b2feaee/bin/clang++"
+  # clang
+  CLANG_BIN="$CLANG_BASE/bin"
+  CLANG_LIB="$CLANG_BASE/lib"
+  CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
+  CC="$CLANG_BIN/clang"
+  CXX="$CLANG_BIN/clang++"

-  KERNEL_HEADERS_INCLUDE="/mnt/gvfs/third-party2/kernel-headers/1a48835975c66d30e47770ec419758ed3b9ba010/3.10.62-62_fbk17_03959_ge29cc63/gcc-4.8.1-glibc-2.17/da39a3e/include/"
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"

  CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
@@ -100,16 +92,15 @@ CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PR
 CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA"
 CXXFLAGS+=" $CFLAGS"

-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB"
 EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
 EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"

 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"

-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"

-VALGRIND_REV=af85c56f424cd5edfc2c97588299b44ecdec96bb
-VALGRIND_VER="/mnt/gvfs/third-party2/valgrind/$VALGRIND_REV/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a/bin/"
+VALGRIND_VER="$VALGRIND_BASE/bin/"

 export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE
@@ -1,5 +1,5 @@
 #!/bin/bash
-#  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 #  This source code is licensed under the BSD-style license found in the
 #  LICENSE file in the root directory of this source tree. An additional grant
 #  of patent rights can be found in the PATENTS file in the same directory.
@@ -10,7 +10,7 @@ then
  GIT="git"
 fi

-# Print out the colored progress info so that it can be brainlessly 
+# Print out the colored progress info so that it can be brainlessly
 # distinguished by users.
 function title() {
  echo -e "\033[1;32m$*\033[0m"
@@ -0,0 +1,198 @@
+#!/usr/local/fbcode/gcc-4.8.1-glibc-2.17-fb/bin/python2.7
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import argparse
+import commands
+import subprocess
+import sys
+import re
+import os
+import time
+
+#
+# Simple logger
+#
+
+
+class Log:
+
+    LOG_FILE = "/tmp/precommit-check.log"
+
+    def __init__(self):
+        self.filename = Log.LOG_FILE
+        self.f = open(self.filename, 'w+', 0)
+
+    def caption(self, str):
+        line = "\n##### %s #####\n" % str
+        if self.f:
+            self.f.write("%s \n" % line)
+        else:
+            print(line)
+
+    def error(self, str):
+        data = "\n\n##### ERROR ##### %s" % str
+        if self.f:
+            self.f.write("%s \n" % data)
+        else:
+            print(data)
+
+    def log(self, str):
+        if self.f:
+            self.f.write("%s \n" % str)
+        else:
+            print(str)
+
+#
+# Shell Environment
+#
+
+
+class Env(object):
+
+    def __init__(self, tests):
+        self.tests = tests
+        self.log = Log()
+
+    def shell(self, cmd, path=os.getcwd()):
+        if path:
+            os.chdir(path)
+
+        self.log.log("==== shell session ===========================")
+        self.log.log("%s> %s" % (path, cmd))
+        status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
+                                 stdout=self.log.f, stderr=self.log.f)
+        self.log.log("status = %s" % status)
+        self.log.log("============================================== \n\n")
+        return status
+
+    def GetOutput(self, cmd, path=os.getcwd()):
+        if path:
+            os.chdir(path)
+
+        self.log.log("==== shell session ===========================")
+        self.log.log("%s> %s" % (path, cmd))
+        status, out = commands.getstatusoutput(cmd)
+        self.log.log("status = %s" % status)
+        self.log.log("out = %s" % out)
+        self.log.log("============================================== \n\n")
+        return status, out
+
+#
+# Pre-commit checker
+#
+
+
+class PreCommitChecker(Env):
+
+    def __init__(self, tests):
+        Env.__init__(self, tests)
+
+    #
+    #   Get commands for a given job from the determinator file
+    #
+    def get_commands(self, test):
+        status, out = self.GetOutput(
+            "build_tools/rocksdb-lego-determinator %s" % test, ".")
+        return status, out
+
+    #
+    # Run a specific CI job
+    #
+    def run_test(self, test):
+        self.log.caption("Running test %s locally" % test)
+
+        # get commands for the CI job determinator
+        status, cmds = self.get_commands(test)
+        if status != 0:
+            self.log.error("Error getting commands for test %s" % test)
+            return False
+
+        # Parse the JSON to extract the commands to run
+        cmds = re.findall("'shell':'([^\']*)'", cmds)
+
+        if len(cmds) == 0:
+            self.log.log("No commands found")
+            return False
+
+        # Run commands
+        for cmd in cmds:
+            # Replace J=<..> with the local environment variable
+            if "J" in os.environ:
+                cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
+                cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
+            # Run the command
+            status = self.shell(cmd, ".")
+            if status != 0:
+                self.log.error("Error running command %s for test %s"
+                               % (cmd, test))
+                return False
+
+        return True
+
+    #
+    # Run specified CI jobs
+    #
+    def run_tests(self):
+        if not self.tests:
+            self.log.error("Invalid args. Please provide tests")
+            return False
+
+        self.print_separator()
+        self.print_row("TEST", "RESULT")
+        self.print_separator()
+
+        for test in self.tests:
+            start_time = time.time()
+            self.print_test(test)
+            result = self.run_test(test)
+            elapsed_min = (time.time() - start_time) / 60
+            if not result:
+                self.log.error("Error running test %s" % test)
+                self.print_result("FAIL (%dm)" % elapsed_min)
+                return False
+            self.print_result("PASS (%dm)" % elapsed_min)
+
+        self.print_separator()
+        return True
+
+    #
+    # Print a line
+    #
+    def print_separator(self):
+        print("".ljust(60, "-"))
+
+    #
+    # Print two colums
+    #
+    def print_row(self, c0, c1):
+        print("%s%s" % (c0.ljust(40), c1.ljust(20)))
+
+    def print_test(self, test):
+        print(test.ljust(40), end="")
+        sys.stdout.flush()
+
+    def print_result(self, result):
+        print(result.ljust(20))
+
+#
+# Main
+#
+parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
+
+# <test ....>
+parser.add_argument('test', nargs='+',
+                    help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
+
+print("Please follow log %s" % Log.LOG_FILE)
+
+args = parser.parse_args()
+checker = PreCommitChecker(args.test)
+
+if not checker.run_tests():
+    print("Error running tests. Please check log file %s" % Log.LOG_FILE)
+    sys.exit(1)
+
+sys.exit(0)
@@ -67,6 +67,7 @@ ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
 LITE="OPT=-DROCKSDB_LITE"
 TSAN="COMPILE_WITH_TSAN=1"
+UBSAN="COMPILE_WITH_UBSAN=1"
 DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
 PARSER="'parser':'egrep \'Failure|^#|Abort|Expected|Actual|GoogleTestFailure|^==\''"

@@ -376,6 +377,48 @@ ASAN_CRASH_TEST_COMMANDS="[
    }
 ]"

+#
+# RocksDB test under undefined behavior sanitizer
+#
+UBSAN_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under UBSAN',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Test RocksDB debug under UBSAN',
+                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make J=1 ubsan_check',
+                'user':'root',
+                $PARSER
+            }
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB crash testing under udnefined behavior sanitizer
+#
+UBSAN_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb crash test under UBSAN',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug ubsan_crash_test',
+                'timeout': 86400,
+                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
 #
 # RocksDB unit test under valgrind
 #
@@ -462,9 +505,9 @@ run_format_compatible()
  if [ -e "build_detect_platform" ]
  then
    sed "s/tcmalloc/nothingnothingnothing/g" build_detect_platform > $TEST_TMPDIR/temp_build_file
-    rm -rf build_detect_platform 
+    rm -rf build_detect_platform
    cp $TEST_TMPDIR/temp_build_file build_detect_platform
-    chmod +x build_detect_platform 
+    chmod +x build_detect_platform
  fi

  make ldb -j32
@@ -632,6 +675,12 @@ case $1 in
  asan_crash)
    echo $ASAN_CRASH_TEST_COMMANDS
    ;;
+  ubsan)
+    echo $UBSAN_TEST_COMMANDS
+    ;;
+  ubsan_crash)
+    echo $UBSAN_CRASH_TEST_COMMANDS
+    ;;
  valgrind)
    echo $VALGRIND_TEST_COMMANDS
    ;;
@@ -9,7 +9,7 @@ Param(
  [string]$WorkFolder = "",  # Direct tests to use that folder
  [int]$Limit = -1, # -1 means run all otherwise limit for testing purposes
  [string]$Exclude = "", # Expect a comma separated list, no spaces
-  [string]$Run = "db_test",  # Run db_test|tests|testname1,testname2...
+  [string]$Run = "db_test",  # Run db_test|db_test2|tests|testname1,testname2...
   # Number of async tasks that would run concurrently. Recommend a number below 64.
   # However, CPU utlization really depends on the storage media. Recommend ram based disk.
   # a value of 1 will run everything serially
@@ -46,15 +46,25 @@ if($WorkFolder -eq "") {
  $Env:TEST_TMPDIR = $WorkFolder
 }

+Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
+
 # Use JEMALLOC executables
-if($EnableJE) {
-    $db_test = -Join ($BinariesFolder, "db_test_je.exe")
-} else {
-    $db_test = -Join ($BinariesFolder, "db_test.exe")
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2" ) {
+
+   $file_name = $Run
+
+   if($EnableJE) {
+     $file_name += "_je"
+   }
+
+   $file_name += ".exe"
+
+   $db_test = -Join ($BinariesFolder, $file_name)
+
+   Write-Output "Binaries: $BinariesFolder db_test: $db_test"
 }

-Write-Output "Root: $RootFolder, WorkFolder: $WorkFolder"
-Write-Output "Binaries: $BinariesFolder db_test: $db_test"

 #Exclusions that we do not want to run
 $ExcludeTests = New-Object System.Collections.Generic.HashSet[string]
@@ -153,7 +163,8 @@ function Discover-TestBinaries([string]$Pattern, $HashTable) {

 $TestsToRun = [ordered]@{}

-if($Run -ceq "db_test") {
+if($Run -ceq "db_test" -or
+   $Run -ceq "db_test2") {
    Normalize-DbTests -HashTable $TestsToRun
 } elseif($Run -ceq "tests") {
    if($EnableJE) {
@@ -225,7 +236,8 @@ function RunJobs($TestToLog, [int]$ConcurrencyVal, [bool]$AddForRerun)
            Write-Host "Starting $k"
            $log_path = ($TestToLog.$k)

-            if($Run -ceq "db_test") {
+            if($Run -ceq "db_test" -or
+               $Run -ceq "db_test2") {
              $job = Start-Job -Name $k -ScriptBlock $InvokeTestCase -ArgumentList @($db_test,$k,$log_path)
            } else {
              [string]$Exe =  -Join ($BinariesFolder, $k)
@@ -0,0 +1,127 @@
+#!/bin/sh
+#
+# Update dependencies.sh file with the latest avaliable versions
+
+BASEDIR=$(dirname $0)
+OUTPUT=""
+
+function log_variable()
+{
+  echo "$1=${!1}" >> "$OUTPUT"
+}
+
+
+TP2_LATEST="/mnt/vol/engshare/fbcode/third-party2"
+## $1 => lib name
+## $2 => lib version (if not provided, will try to pick latest)
+## $3 => platform (if not provided, will try to pick latest gcc)
+##
+## get_lib_base will set a variable named ${LIB_NAME}_BASE to the lib location
+function get_lib_base()
+{
+  local lib_name=$1
+  local lib_version=$2
+  local lib_platform=$3
+
+  local result="$TP2_LATEST/$lib_name/"
+  
+  # Lib Version
+  if [ -z "$lib_version" ] || [ "$lib_version" = "LATEST" ]; then
+    # version is not provided, use latest
+    result=`ls -dr1v $result/*/ | head -n1`
+  else
+    result="$result/$lib_version/"
+  fi
+  
+  # Lib Platform
+  if [ -z "$lib_platform" ]; then
+    # platform is not provided, use latest gcc
+    result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
+  else
+    result="$result/$lib_platform/"
+  fi
+  
+  result=`ls -1d $result/*/ | head -n1`
+  
+  # lib_name => LIB_NAME_BASE
+  local __res_var=${lib_name^^}"_BASE"
+  __res_var=`echo $__res_var | tr - _`
+  # LIB_NAME_BASE=$result
+  eval $__res_var=`readlink -f $result`
+  
+  log_variable $__res_var
+}
+
+###########################################################
+#                   4.9.x dependencies                    #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
+CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc     4.9.x
+get_lib_base glibc      2.20 
+get_lib_base snappy     LATEST
+get_lib_base zlib       LATEST
+get_lib_base bzip2      LATEST
+get_lib_base lz4        LATEST
+get_lib_base zstd       LATEST
+get_lib_base gflags     LATEST
+get_lib_base jemalloc   LATEST
+get_lib_base numa       LATEST
+get_lib_base libunwind  LATEST
+
+get_lib_base kernel-headers LATEST 
+get_lib_base binutils   LATEST centos6-native 
+get_lib_base valgrind   LATEST
+
+git diff $OUTPUT
+
+###########################################################
+#                   4.8.1 dependencies                    #
+###########################################################
+
+OUTPUT="$BASEDIR/dependencies_4.8.1.sh"
+
+rm -f "$OUTPUT"
+touch "$OUTPUT"
+
+echo "Writing 4.8.1 dependencies to $OUTPUT"
+
+# Compilers locations
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
+CLANG_BASE=`ls -d1 /mnt/gvfs/third-party2/clang/fc904e50a9266b9d7b98cae1993afa0c5aae1440/3.7.1/centos6-native/*/ | head -n1`
+
+log_variable GCC_BASE
+log_variable CLANG_BASE
+
+# Libraries locations
+get_lib_base libgcc     4.8.1  gcc-4.8.1-glibc-2.17
+get_lib_base glibc      2.17   gcc-4.8.1-glibc-2.17  
+get_lib_base snappy     LATEST gcc-4.8.1-glibc-2.17
+get_lib_base zlib       LATEST gcc-4.8.1-glibc-2.17
+get_lib_base bzip2      LATEST gcc-4.8.1-glibc-2.17
+get_lib_base lz4        LATEST gcc-4.8.1-glibc-2.17
+get_lib_base zstd       LATEST gcc-4.8.1-glibc-2.17
+get_lib_base gflags     LATEST gcc-4.8.1-glibc-2.17
+get_lib_base jemalloc   LATEST gcc-4.8.1-glibc-2.17
+get_lib_base numa       LATEST gcc-4.8.1-glibc-2.17
+get_lib_base libunwind  LATEST gcc-4.8.1-glibc-2.17
+
+get_lib_base kernel-headers LATEST gcc-4.8.1-glibc-2.17 
+get_lib_base binutils   LATEST centos6-native 
+get_lib_base valgrind   3.8.1  gcc-4.8.1-glibc-2.17
+
+git diff $OUTPUT
@@ -1,9 +1,9 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
-#include "util/auto_roll_logger.h"
+#include "db/auto_roll_logger.h"
 #include "util/mutexlock.h"

 using namespace std;
@@ -12,7 +12,9 @@ namespace rocksdb {

 // -- AutoRollLogger
 Status AutoRollLogger::ResetLogger() {
+  TEST_SYNC_POINT("AutoRollLogger::ResetLogger:BeforeNewLogger");
  status_ = env_->NewLogger(log_fname_, &logger_);
+  TEST_SYNC_POINT("AutoRollLogger::ResetLogger:AfterNewLogger");

  if (!status_.ok()) {
    return status_;
@@ -32,8 +34,16 @@ Status AutoRollLogger::ResetLogger() {
 }

 void AutoRollLogger::RollLogFile() {
-  std::string old_fname = OldInfoLogFileName(
-      dbname_, env_->NowMicros(), db_absolute_path_, db_log_dir_);
+  // This function is called when log is rotating. Two rotations
+  // can happen quickly (NowMicro returns same value). To not overwrite
+  // previous log file we increment by one micro second and try again.
+  uint64_t now = env_->NowMicros();
+  std::string old_fname;
+  do {
+    old_fname = OldInfoLogFileName(
+      dbname_, now, db_absolute_path_, db_log_dir_);
+    now++;
+  } while (env_->FileExists(old_fname).ok());
  env_->RenameFile(log_fname_, old_fname);
 }

@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -13,6 +13,8 @@
 #include "db/filename.h"
 #include "port/port.h"
 #include "port/util_logger.h"
+#include "util/sync_point.h"
+#include "util/mutexlock.h"

 namespace rocksdb {

@@ -53,11 +55,26 @@ class AutoRollLogger : public Logger {
    return status_;
  }

-  size_t GetLogFileSize() const override { return logger_->GetLogFileSize(); }
+  size_t GetLogFileSize() const override {
+    std::shared_ptr<Logger> logger;
+    {
+      MutexLock l(&mutex_);
+      // pin down the current logger_ instance before releasing the mutex.
+      logger = logger_;
+    }
+    return logger->GetLogFileSize();
+  }

  void Flush() override {
-    if (logger_) {
-      logger_->Flush();
+    std::shared_ptr<Logger> logger;
+    {
+      MutexLock l(&mutex_);
+      // pin down the current logger_ instance before releasing the mutex.
+      logger = logger_;
+    }
+    TEST_SYNC_POINT("AutoRollLogger::Flush:PinnedLogger");
+    if (logger) {
+      logger->Flush();
    }
  }

@@ -101,7 +118,7 @@ class AutoRollLogger : public Logger {
  uint64_t ctime_;
  uint64_t cached_now_access_count;
  uint64_t call_NowMicros_every_N_records_;
-  port::Mutex mutex_;
+  mutable port::Mutex mutex_;
 };

 // Facade to craete logger automatically
@@ -1,17 +1,20 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
 #include <string>
+#include <thread>
 #include <vector>
 #include <cmath>
 #include <iostream>
 #include <fstream>
 #include <iterator>
 #include <algorithm>
+#include "db/auto_roll_logger.h"
+#include "port/port.h"
+#include "util/sync_point.h"
 #include "util/testharness.h"
-#include "util/auto_roll_logger.h"
 #include "rocksdb/db.h"
 #include <sys/stat.h>
 #include <errno.h>
@@ -260,7 +263,60 @@ TEST_F(AutoRollLoggerTest, CreateLoggerFromOptions) {
      auto_roll_logger, options.log_file_time_to_roll,
      kSampleMessage + ":CreateLoggerFromOptions - both");
 }
-#endif
+
+TEST_F(AutoRollLoggerTest, LogFlushWhileRolling) {
+  DBOptions options;
+  shared_ptr<Logger> logger;
+
+  InitTestDb();
+  options.max_log_file_size = 1024 * 5;
+  ASSERT_OK(CreateLoggerFromOptions(kTestDir, options, &logger));
+  AutoRollLogger* auto_roll_logger =
+      dynamic_cast<AutoRollLogger*>(logger.get());
+  ASSERT_TRUE(auto_roll_logger);
+  std::thread flush_thread;
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency({
+      // Need to pin the old logger before beginning the roll, as rolling grabs
+      // the mutex, which would prevent us from accessing the old logger.
+      {"AutoRollLogger::Flush:PinnedLogger",
+       "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit"},
+      // Need to finish the flush thread init before this callback because the
+      // callback accesses flush_thread.get_id() in order to apply certain sync
+      // points only to the flush thread.
+      {"AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit",
+       "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallbackBegin"},
+      // Need to reset logger at this point in Flush() to exercise a race
+      // condition case, which is executing the flush with the pinned (old)
+      // logger after the roll has cut over to a new logger.
+      {"AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback1",
+       "AutoRollLogger::ResetLogger:BeforeNewLogger"},
+      {"AutoRollLogger::ResetLogger:AfterNewLogger",
+       "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback2"},
+  });
+  rocksdb::SyncPoint::GetInstance()->SetCallBack(
+      "PosixLogger::Flush:BeginCallback", [&](void* arg) {
+        TEST_SYNC_POINT(
+            "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallbackBegin");
+        if (std::this_thread::get_id() == flush_thread.get_id()) {
+          TEST_SYNC_POINT(
+              "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback1");
+          TEST_SYNC_POINT(
+              "AutoRollLoggerTest::LogFlushWhileRolling:FlushCallback2");
+        }
+      });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  flush_thread = std::thread([&]() { auto_roll_logger->Flush(); });
+  TEST_SYNC_POINT(
+      "AutoRollLoggerTest::LogFlushWhileRolling:PreRollAndPostThreadInit");
+  RollLogFileBySizeTest(auto_roll_logger, options.max_log_file_size,
+                        kSampleMessage + ":LogFlushWhileRolling");
+  flush_thread.join();
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+}
+
+#endif  // OS_WIN

 TEST_F(AutoRollLoggerTest, InfoLogLevel) {
  InitTestDb();
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -41,13 +41,17 @@ TableBuilder* NewTableBuilder(
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, WritableFileWriter* file,
-    const CompressionType compression_type,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
    const CompressionOptions& compression_opts, const bool skip_filters) {
+  assert((column_family_id ==
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+         column_family_name.empty());
  return ioptions.table_factory->NewTableBuilder(
      TableBuilderOptions(ioptions, internal_comparator,
                          int_tbl_prop_collector_factories, compression_type,
-                          compression_opts, skip_filters),
+                          compression_opts, skip_filters,
+                          column_family_name),
      column_family_id, file);
 }

@@ -58,12 +62,16 @@ Status BuildTable(
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, const std::string& column_family_name,
+    std::vector<SequenceNumber> snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
    const CompressionType compression,
    const CompressionOptions& compression_opts, bool paranoid_file_checks,
    InternalStats* internal_stats, const Env::IOPriority io_priority,
-    TableProperties* table_properties) {
+    TableProperties* table_properties, int level) {
+  assert((column_family_id ==
+          TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
+         column_family_name.empty());
  // Reports the IOStats for flush for every following bytes.
  const size_t kReportFlushIOStatsEvery = 1048576;
  Status s;
@@ -87,7 +95,8 @@ Status BuildTable(

      builder = NewTableBuilder(
          ioptions, internal_comparator, int_tbl_prop_collector_factories,
-          column_family_id, file_writer.get(), compression, compression_opts);
+          column_family_id, column_family_name, file_writer.get(), compression,
+          compression_opts);
    }

    MergeHelper merge(env, internal_comparator.user_comparator(),
@@ -149,7 +158,8 @@ Status BuildTable(
          ReadOptions(), env_options, internal_comparator, meta->fd, nullptr,
          (internal_stats == nullptr) ? nullptr
                                      : internal_stats->GetFileReadHist(0),
-          false));
+          false /* for_compaction */, nullptr /* arena */,
+          false /* skip_filter */, level));
      s = it->status();
      if (s.ok() && paranoid_file_checks) {
        for (it->SeekToFirst(); it->Valid(); it->Next()) {
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -33,13 +33,16 @@ class WritableFileWriter;
 class InternalStats;
 class InternalIterator;

+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown. It must outlive the
+//    TableBuilder returned by this function.
 TableBuilder* NewTableBuilder(
    const ImmutableCFOptions& options,
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, WritableFileWriter* file,
-    const CompressionType compression_type,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
    const CompressionOptions& compression_opts,
    const bool skip_filters = false);

@@ -48,6 +51,9 @@ TableBuilder* NewTableBuilder(
 // *meta will be filled with metadata about the generated table.
 // If no data is present in *iter, meta->file_size will be set to
 // zero, and no Table file will be produced.
+//
+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
    const std::string& dbname, Env* env, const ImmutableCFOptions& options,
    const EnvOptions& env_options, TableCache* table_cache,
@@ -55,12 +61,13 @@ extern Status BuildTable(
    const InternalKeyComparator& internal_comparator,
    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
        int_tbl_prop_collector_factories,
-    uint32_t column_family_id, std::vector<SequenceNumber> snapshots,
+    uint32_t column_family_id, const std::string& column_family_name,
+    std::vector<SequenceNumber> snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
    const CompressionType compression,
    const CompressionOptions& compression_opts, bool paranoid_file_checks,
    InternalStats* internal_stats,
    const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr);
+    TableProperties* table_properties = nullptr, int level = -1);

 }  // namespace rocksdb
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1288,6 +1288,11 @@ void rocksdb_block_based_options_set_cache_index_and_filter_blocks(
  options->rep.cache_index_and_filter_blocks = v;
 }

+void rocksdb_block_based_options_set_pin_l0_filter_and_index_blocks_in_cache(
+    rocksdb_block_based_table_options_t* options, unsigned char v) {
+  options->rep.pin_l0_filter_and_index_blocks_in_cache = v;
+}
+
 void rocksdb_block_based_options_set_skip_table_builder_flush(
    rocksdb_block_based_table_options_t* options, unsigned char v) {
  options->rep.skip_table_builder_flush = v;
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -135,6 +135,10 @@ Status CheckConcurrentWritesSupported(const ColumnFamilyOptions& cf_options) {
        "Delete filtering (filter_deletes) is not compatible with concurrent "
        "memtable writes (allow_concurrent_memtable_writes)");
  }
+  if (!cf_options.memtable_factory->IsInsertConcurrentlySupported()) {
+    return Status::InvalidArgument(
+        "Memtable doesn't concurrent writes (allow_concurrent_memtable_write)");
+  }
  return Status::OK();
 }

@@ -143,13 +147,10 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
                                    const ColumnFamilyOptions& src) {
  ColumnFamilyOptions result = src;
  result.comparator = icmp;
-#ifdef OS_MACOSX
-  // TODO(icanadi) make write_buffer_size uint64_t instead of size_t
-  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, ((size_t)1) << 30);
-#else
-  ClipToRange(&result.write_buffer_size,
-              ((size_t)64) << 10, ((size_t)64) << 30);
-#endif
+  size_t clamp_max = std::conditional<
+      sizeof(size_t) == 4, std::integral_constant<size_t, 0xffffffff>,
+      std::integral_constant<size_t, 64ull << 30>>::type::value;
+  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, clamp_max);
  // if user sets arena_block_size, we trust user to use this value. Otherwise,
  // calculate a proper value from writer_buffer_size;
  if (result.arena_block_size <= 0) {
@@ -239,6 +240,17 @@ ColumnFamilyOptions SanitizeOptions(const DBOptions& db_options,
         result.level0_slowdown_writes_trigger,
         result.level0_file_num_compaction_trigger);
  }
+
+  if (result.soft_pending_compaction_bytes_limit == 0) {
+    result.soft_pending_compaction_bytes_limit =
+        result.hard_pending_compaction_bytes_limit;
+  } else if (result.hard_pending_compaction_bytes_limit > 0 &&
+             result.soft_pending_compaction_bytes_limit >
+                 result.hard_pending_compaction_bytes_limit) {
+    result.soft_pending_compaction_bytes_limit =
+        result.hard_pending_compaction_bytes_limit;
+  }
+
  if (result.level_compaction_dynamic_level_bytes) {
    if (result.compaction_style != kCompactionStyleLevel ||
        db_options.db_paths.size() > 1U) {
@@ -513,6 +525,21 @@ std::unique_ptr<WriteControllerToken> SetupDelay(
  }
  return write_controller->GetDelayToken(write_rate);
 }
+
+int GetL0ThresholdSpeedupCompaction(int level0_file_num_compaction_trigger,
+                                    int level0_slowdown_writes_trigger) {
+  // SanitizeOptions() ensures it.
+  assert(level0_file_num_compaction_trigger <= level0_slowdown_writes_trigger);
+
+  // 1/4 of the way between L0 compaction trigger threshold and slowdown
+  // condition.
+  // Or twice as compaction trigger, if it is smaller.
+  return std::min(level0_file_num_compaction_trigger * 2,
+                  level0_file_num_compaction_trigger +
+                      (level0_slowdown_writes_trigger -
+                       level0_file_num_compaction_trigger) /
+                          4);
+}
 }  // namespace

 void ColumnFamilyData::RecalculateWriteStallConditions(
@@ -531,21 +558,6 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
          "(waiting for flush), max_write_buffer_number is set to %d",
          name_.c_str(), imm()->NumNotFlushed(),
          mutable_cf_options.max_write_buffer_number);
-    } else if (mutable_cf_options.max_write_buffer_number > 3 &&
-               imm()->NumNotFlushed() >=
-                   mutable_cf_options.max_write_buffer_number - 1) {
-      write_controller_token_ =
-          SetupDelay(ioptions_.delayed_write_rate, write_controller,
-                     compaction_needed_bytes, prev_compaction_needed_bytes_,
-                     mutable_cf_options.disable_auto_compactions);
-      internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
-      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
-          "[%s] Stalling writes because we have %d immutable memtables "
-          "(waiting for flush), max_write_buffer_number is set to %d "
-          "rate %" PRIu64,
-          name_.c_str(), imm()->NumNotFlushed(),
-          mutable_cf_options.max_write_buffer_number,
-          write_controller->delayed_write_rate());
    } else if (vstorage->l0_delay_trigger_count() >=
               mutable_cf_options.level0_stop_writes_trigger) {
      write_controller_token_ = write_controller->GetStopToken();
@@ -567,6 +579,21 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
          "[%s] Stopping writes because of estimated pending compaction "
          "bytes %" PRIu64,
          name_.c_str(), compaction_needed_bytes);
+    } else if (mutable_cf_options.max_write_buffer_number > 3 &&
+               imm()->NumNotFlushed() >=
+                   mutable_cf_options.max_write_buffer_number - 1) {
+      write_controller_token_ =
+          SetupDelay(ioptions_.delayed_write_rate, write_controller,
+                     compaction_needed_bytes, prev_compaction_needed_bytes_,
+                     mutable_cf_options.disable_auto_compactions);
+      internal_stats_->AddCFStats(InternalStats::MEMTABLE_SLOWDOWN, 1);
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Stalling writes because we have %d immutable memtables "
+          "(waiting for flush), max_write_buffer_number is set to %d "
+          "rate %" PRIu64,
+          name_.c_str(), imm()->NumNotFlushed(),
+          mutable_cf_options.max_write_buffer_number,
+          write_controller->delayed_write_rate());
    } else if (mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
               vstorage->l0_delay_trigger_count() >=
                   mutable_cf_options.level0_slowdown_writes_trigger) {
@@ -598,6 +625,29 @@ void ColumnFamilyData::RecalculateWriteStallConditions(
          "bytes %" PRIu64 " rate %" PRIu64,
          name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
          write_controller->delayed_write_rate());
+    } else if (vstorage->l0_delay_trigger_count() >=
+               GetL0ThresholdSpeedupCompaction(
+                   mutable_cf_options.level0_file_num_compaction_trigger,
+                   mutable_cf_options.level0_slowdown_writes_trigger)) {
+      write_controller_token_ = write_controller->GetCompactionPressureToken();
+      Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+          "[%s] Increasing compaction threads because we have %d level-0 "
+          "files ",
+          name_.c_str(), vstorage->l0_delay_trigger_count());
+    } else if (vstorage->estimated_compaction_needed_bytes() >=
+               mutable_cf_options.soft_pending_compaction_bytes_limit / 4) {
+      // Increase compaction threads if bytes needed for compaction exceeds
+      // 1/4 of threshold for slowing down.
+      // If soft pending compaction byte limit is not set, always speed up
+      // compaction.
+      write_controller_token_ = write_controller->GetCompactionPressureToken();
+      if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
+        Log(InfoLogLevel::WARN_LEVEL, ioptions_.info_log,
+            "[%s] Increasing compaction threads because of estimated pending "
+            "compaction "
+            "bytes %" PRIu64,
+            name_.c_str(), vstorage->estimated_compaction_needed_bytes());
+      }
    } else {
      write_controller_token_.reset();
    }
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -465,6 +465,8 @@ class ColumnFamilySet {
  // Don't call while iterating over ColumnFamilySet
  void FreeDeadColumnFamilies();

+  Cache* get_table_cache() { return table_cache_; }
+
 private:
  friend class ColumnFamilyData;
  // helper function that gets called from cfd destructor
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -65,6 +65,7 @@ class ColumnFamilyTest : public testing::Test {

  ~ColumnFamilyTest() {
    Close();
+    rocksdb::SyncPoint::GetInstance()->DisableProcessing();
    Destroy();
    delete env_;
  }
@@ -1951,13 +1952,27 @@ TEST_F(ColumnFamilyTest, ReadDroppedColumnFamily) {
    PutRandomData(1, kKeysNum, 100);
    PutRandomData(2, kKeysNum, 100);

-    if (iter == 0) {
-      // Drop CF two
-      ASSERT_OK(db_->DropColumnFamily(handles_[2]));
-    } else {
-      // delete CF two
-      delete handles_[2];
-      handles_[2] = nullptr;
+    {
+      std::unique_ptr<Iterator> iterator(
+          db_->NewIterator(ReadOptions(), handles_[2]));
+      iterator->SeekToFirst();
+
+      if (iter == 0) {
+        // Drop CF two
+        ASSERT_OK(db_->DropColumnFamily(handles_[2]));
+      } else {
+        // delete CF two
+        delete handles_[2];
+        handles_[2] = nullptr;
+      }
+      // Make sure iterator created can still be used.
+      int count = 0;
+      for (; iterator->Valid(); iterator->Next()) {
+        ASSERT_OK(iterator->status());
+        ++count;
+      }
+      ASSERT_OK(iterator->status());
+      ASSERT_EQ(count, kKeysNum);
    }

    // Add bunch more data to other CFs
@@ -1999,7 +2014,9 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {
  Reopen({options, options});

  rocksdb::SyncPoint::GetInstance()->LoadDependency(
-      {{"VersionSet::LogAndApply::ColumnFamilyDrop:1",
+      {{"VersionSet::LogAndApply::ColumnFamilyDrop:0",
+        "FlushJob::WriteLevel0Table"},
+       {"VersionSet::LogAndApply::ColumnFamilyDrop:1",
        "FlushJob::InstallResults"},
       {"FlushJob::InstallResults",
        "VersionSet::LogAndApply::ColumnFamilyDrop:2"}});
@@ -2045,7 +2062,6 @@ TEST_F(ColumnFamilyTest, FlushAndDropRaceCondition) {

  Close();
  Destroy();
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
 }

 #ifndef ROCKSDB_LITE
@@ -2123,7 +2139,6 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
  drop_cf_thread.join();
  Close();
  Destroy();
-  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  for (auto* comparator : comparators) {
    if (comparator) {
      delete comparator;
@@ -2135,6 +2150,9 @@ TEST_F(ColumnFamilyTest, CreateAndDropRace) {
 TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  const uint64_t kBaseRate = 810000u;
  db_options_.delayed_write_rate = kBaseRate;
+  db_options_.base_background_compactions = 2;
+  db_options_.max_background_compactions = 6;
+
  Open({"default"});
  ColumnFamilyData* cfd =
      static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
@@ -2160,6 +2178,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  ASSERT_TRUE(!dbfull()->TEST_write_controler().IsStopped());
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate, dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());

  vstorage->TEST_set_estimated_compaction_needed_bytes(400);
  cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2167,6 +2186,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.2,
            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());

  vstorage->TEST_set_estimated_compaction_needed_bytes(500);
  cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2222,6 +2242,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  cfd->RecalculateWriteStallConditions(mutable_cf_options);
  ASSERT_TRUE(dbfull()->TEST_write_controler().IsStopped());
  ASSERT_TRUE(!dbfull()->TEST_write_controler().NeedsDelay());
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());

  vstorage->TEST_set_estimated_compaction_needed_bytes(3001);
  cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2246,6 +2267,7 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
  ASSERT_TRUE(dbfull()->TEST_write_controler().NeedsDelay());
  ASSERT_EQ(kBaseRate / 1.2,
            dbfull()->TEST_write_controler().delayed_write_rate());
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());

  vstorage->set_l0_delay_trigger_count(101);
  cfd->RecalculateWriteStallConditions(mutable_cf_options);
@@ -2318,6 +2340,73 @@ TEST_F(ColumnFamilyTest, WriteStallSingleColumnFamily) {
            dbfull()->TEST_write_controler().delayed_write_rate());
 }

+TEST_F(ColumnFamilyTest, CompactionSpeedupSingleColumnFamily) {
+  db_options_.base_background_compactions = 2;
+  db_options_.max_background_compactions = 6;
+  Open({"default"});
+  ColumnFamilyData* cfd =
+      static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
+
+  VersionStorageInfo* vstorage = cfd->current()->storage_info();
+
+  MutableCFOptions mutable_cf_options(
+      Options(db_options_, column_family_options_),
+      ImmutableCFOptions(Options(db_options_, column_family_options_)));
+
+  // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
+  mutable_cf_options.level0_file_num_compaction_trigger = 4;
+  mutable_cf_options.level0_slowdown_writes_trigger = 36;
+  mutable_cf_options.level0_stop_writes_trigger = 50;
+  // Speedup threshold = 200 / 4 = 50
+  mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
+  mutable_cf_options.hard_pending_compaction_bytes_limit = 2000;
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(40);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(50);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(300);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(45);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(7);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(9);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(6);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  // Speed up threshold = min(4 * 2, 4 + (12 - 4)/4) = 6
+  mutable_cf_options.level0_file_num_compaction_trigger = 4;
+  mutable_cf_options.level0_slowdown_writes_trigger = 16;
+  mutable_cf_options.level0_stop_writes_trigger = 30;
+
+  vstorage->set_l0_delay_trigger_count(5);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(7);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(3);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+}
+
 TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
  const uint64_t kBaseRate = 810000u;
  db_options_.delayed_write_rate = kBaseRate;
@@ -2399,6 +2488,104 @@ TEST_F(ColumnFamilyTest, WriteStallTwoColumnFamilies) {
  ASSERT_EQ(kBaseRate / 1.2,
            dbfull()->TEST_write_controler().delayed_write_rate());
 }
+
+TEST_F(ColumnFamilyTest, CompactionSpeedupTwoColumnFamilies) {
+  db_options_.base_background_compactions = 2;
+  db_options_.max_background_compactions = 6;
+  column_family_options_.soft_pending_compaction_bytes_limit = 200;
+  column_family_options_.hard_pending_compaction_bytes_limit = 2000;
+  Open();
+  CreateColumnFamilies({"one"});
+  ColumnFamilyData* cfd =
+      static_cast<ColumnFamilyHandleImpl*>(db_->DefaultColumnFamily())->cfd();
+  VersionStorageInfo* vstorage = cfd->current()->storage_info();
+
+  ColumnFamilyData* cfd1 =
+      static_cast<ColumnFamilyHandleImpl*>(handles_[1])->cfd();
+  VersionStorageInfo* vstorage1 = cfd1->current()->storage_info();
+
+  MutableCFOptions mutable_cf_options(
+      Options(db_options_, column_family_options_),
+      ImmutableCFOptions(Options(db_options_, column_family_options_)));
+  // Speed up threshold = min(4 * 2, 4 + (36 - 4)/4) = 8
+  mutable_cf_options.level0_file_num_compaction_trigger = 4;
+  mutable_cf_options.level0_slowdown_writes_trigger = 36;
+  mutable_cf_options.level0_stop_writes_trigger = 30;
+  // Speedup threshold = 200 / 4 = 50
+  mutable_cf_options.soft_pending_compaction_bytes_limit = 200;
+  mutable_cf_options.hard_pending_compaction_bytes_limit = 2000;
+
+  MutableCFOptions mutable_cf_options1 = mutable_cf_options;
+  mutable_cf_options1.level0_slowdown_writes_trigger = 16;
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(40);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(60);
+  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(30);
+  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(70);
+  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage->TEST_set_estimated_compaction_needed_bytes(20);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage1->TEST_set_estimated_compaction_needed_bytes(3);
+  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(9);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage1->set_l0_delay_trigger_count(2);
+  cfd1->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(6, dbfull()->BGCompactionsAllowed());
+
+  vstorage->set_l0_delay_trigger_count(0);
+  cfd->RecalculateWriteStallConditions(mutable_cf_options);
+  ASSERT_EQ(2, dbfull()->BGCompactionsAllowed());
+}
+
+TEST_F(ColumnFamilyTest, LogSyncConflictFlush) {
+  Open();
+  CreateColumnFamiliesAndReopen({"one", "two"});
+
+  Put(0, "", "");
+  Put(1, "foo", "bar");
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency(
+      {{"DBImpl::SyncWAL:BeforeMarkLogsSynced:1",
+        "ColumnFamilyTest::LogSyncConflictFlush:1"},
+       {"ColumnFamilyTest::LogSyncConflictFlush:2",
+        "DBImpl::SyncWAL:BeforeMarkLogsSynced:2"}});
+
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  std::thread thread([&] { db_->SyncWAL(); });
+
+  TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:1");
+  Flush(1);
+  Put(1, "foo", "bar");
+  Flush(1);
+
+  TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:2");
+
+  thread.join();
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+  Close();
+}
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -1,4 +1,4 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -7,6 +7,7 @@

 #include <mutex>
 #include <string>
+#include <thread>
 #include <vector>

 #include "rocksdb/db.h"
@@ -107,6 +108,7 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
      break;
    }
  }
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
  delete db;
 }

@@ -141,9 +143,6 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
  }

  auto l0_files = collector->GetFlushedFiles();
-  CompactionOptions compact_opt;
-  compact_opt.compression = kNoCompression;
-  compact_opt.output_file_size_limit = kWriteBufferSize * 5;
  ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1));

  // verify all compaction input files are deleted
@@ -153,6 +152,62 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
  delete db;
 }

+TEST_F(CompactFilesTest, CapturingPendingFiles) {
+  Options options;
+  options.create_if_missing = true;
+  // Disable RocksDB background compaction.
+  options.compaction_style = kCompactionStyleNone;
+  // Always do full scans for obsolete files (needed to reproduce the issue).
+  options.delete_obsolete_files_period_micros = 0;
+
+  // Add listener.
+  FlushedFileCollector* collector = new FlushedFileCollector();
+  options.listeners.emplace_back(collector);
+
+  DB* db = nullptr;
+  DestroyDB(db_name_, options);
+  Status s = DB::Open(options, db_name_, &db);
+  assert(s.ok());
+  assert(db);
+
+  // Create 5 files.
+  for (int i = 0; i < 5; ++i) {
+    db->Put(WriteOptions(), "key" + ToString(i), "value");
+    db->Flush(FlushOptions());
+  }
+
+  auto l0_files = collector->GetFlushedFiles();
+  EXPECT_EQ(5, l0_files.size());
+
+  rocksdb::SyncPoint::GetInstance()->LoadDependency({
+      {"CompactFilesImpl:2", "CompactFilesTest.CapturingPendingFiles:0"},
+      {"CompactFilesTest.CapturingPendingFiles:1", "CompactFilesImpl:3"},
+  });
+  rocksdb::SyncPoint::GetInstance()->EnableProcessing();
+
+  // Start compacting files.
+  std::thread compaction_thread(
+      [&] { EXPECT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1)); });
+
+  // In the meantime flush another file.
+  TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:0");
+  db->Put(WriteOptions(), "key5", "value");
+  db->Flush(FlushOptions());
+  TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:1");
+
+  compaction_thread.join();
+
+  rocksdb::SyncPoint::GetInstance()->DisableProcessing();
+
+  delete db;
+
+  // Make sure we can reopen the DB.
+  s = DB::Open(options, db_name_, &db);
+  ASSERT_TRUE(s.ok());
+  assert(db);
+  delete db;
+}
+
 }  // namespace rocksdb

 int main(int argc, char** argv) {
@@ -1,4 +1,4 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -162,9 +162,6 @@ Compaction::Compaction(VersionStorageInfo* vstorage,
      deletion_compaction_(_deletion_compaction),
      inputs_(std::move(_inputs)),
      grandparents_(std::move(_grandparents)),
-      grandparent_index_(0),
-      seen_key_(false),
-      overlapped_bytes_(0),
      score_(_score),
      bottommost_level_(IsBottommostLevel(output_level_, vstorage, inputs_)),
      is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
@@ -290,32 +287,6 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
  return true;
 }

-bool Compaction::ShouldStopBefore(const Slice& internal_key) {
-  // Scan to find earliest grandparent file that contains key.
-  const InternalKeyComparator* icmp = &cfd_->internal_comparator();
-  while (grandparent_index_ < grandparents_.size() &&
-      icmp->Compare(internal_key,
-                    grandparents_[grandparent_index_]->largest.Encode()) > 0) {
-    if (seen_key_) {
-      overlapped_bytes_ += grandparents_[grandparent_index_]->fd.GetFileSize();
-    }
-    assert(grandparent_index_ + 1 >= grandparents_.size() ||
-           icmp->Compare(grandparents_[grandparent_index_]->largest.Encode(),
-                         grandparents_[grandparent_index_+1]->smallest.Encode())
-                         < 0);
-    grandparent_index_++;
-  }
-  seen_key_ = true;
-
-  if (overlapped_bytes_ > max_grandparent_overlap_bytes_) {
-    // Too much overlap for current output; start new output
-    overlapped_bytes_ = 0;
-    return true;
-  } else {
-    return false;
-  }
-}
-
 // Mark (or clear) each file that is being compacted
 void Compaction::MarkFilesBeingCompacted(bool mark_as_compacted) {
  for (size_t i = 0; i < num_input_levels(); i++) {
@@ -421,7 +392,7 @@ void Compaction::Summary(char* output, int len) {
  snprintf(output + write, len - write, "]");
 }

-uint64_t Compaction::OutputFilePreallocationSize() {
+uint64_t Compaction::OutputFilePreallocationSize() const {
  uint64_t preallocation_size = 0;

  if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel ||
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -104,7 +104,7 @@ class Compaction {
  }

  // Returns the LevelFilesBrief of the specified compaction input level.
-  LevelFilesBrief* input_levels(size_t compaction_input_level) {
+  const LevelFilesBrief* input_levels(size_t compaction_input_level) const {
    return &input_levels_[compaction_input_level];
  }

@@ -132,12 +132,10 @@ class Compaction {
  bool KeyNotExistsBeyondOutputLevel(const Slice& user_key,
                                     std::vector<size_t>* level_ptrs) const;

-  // Returns true iff we should stop building the current output
-  // before processing "internal_key".
-  bool ShouldStopBefore(const Slice& internal_key);
-
  // Clear all files to indicate that they are not being compacted
  // Delete this compaction from the list of running compactions.
+  //
+  // Requirement: DB mutex held
  void ReleaseCompactionFiles(Status status);

  // Returns the summary of the compaction in "output" with maximum "len"
@@ -149,13 +147,13 @@ class Compaction {
  double score() const { return score_; }

  // Is this compaction creating a file in the bottom most level?
-  bool bottommost_level() { return bottommost_level_; }
+  bool bottommost_level() const { return bottommost_level_; }

  // Does this compaction include all sst files?
-  bool is_full_compaction() { return is_full_compaction_; }
+  bool is_full_compaction() const { return is_full_compaction_; }

  // Was this compaction triggered manually by the client?
-  bool is_manual_compaction() { return is_manual_compaction_; }
+  bool is_manual_compaction() const { return is_manual_compaction_; }

  // Used when allow_trivial_move option is set in
  // Universal compaction. If all the input files are
@@ -168,19 +166,21 @@ class Compaction {
  // Used when allow_trivial_move option is set in
  // Universal compaction. Returns true, if the input files
  // are non-overlapping and can be trivially moved.
-  bool is_trivial_move() { return is_trivial_move_; }
+  bool is_trivial_move() const { return is_trivial_move_; }

  // How many total levels are there?
  int number_levels() const { return number_levels_; }

  // Return the MutableCFOptions that should be used throughout the compaction
  // procedure
-  const MutableCFOptions* mutable_cf_options() { return &mutable_cf_options_; }
+  const MutableCFOptions* mutable_cf_options() const {
+    return &mutable_cf_options_;
+  }

  // Returns the size in bytes that the output file should be preallocated to.
  // In level compaction, that is max_file_size_. In universal compaction, that
  // is the sum of all input file sizes.
-  uint64_t OutputFilePreallocationSize();
+  uint64_t OutputFilePreallocationSize() const;

  void SetInputVersion(Version* input_version);

@@ -223,6 +223,14 @@ class Compaction {

  CompactionReason compaction_reason() { return compaction_reason_; }

+  const std::vector<FileMetaData*>& grandparents() const {
+    return grandparents_;
+  }
+
+  uint64_t max_grandparent_overlap_bytes() const {
+    return max_grandparent_overlap_bytes_;
+  }
+
 private:
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool mark_as_compacted);
@@ -266,10 +274,6 @@ class Compaction {
  // State used to check for number of of overlapping grandparent files
  // (grandparent == "output_level_ + 1")
  std::vector<FileMetaData*> grandparents_;
-  size_t grandparent_index_;   // Index in grandparent_starts_
-  bool seen_key_;              // Some output key has been seen
-  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
-                               // and grandparent files
  const double score_;         // score that was used to pick this compaction.

  // Is this compaction creating a file in the bottom most level?
@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -14,7 +14,7 @@ CompactionIterator::CompactionIterator(
    InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
    SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
    SequenceNumber earliest_write_conflict_snapshot, Env* env,
-    bool expect_valid_internal_key, Compaction* compaction,
+    bool expect_valid_internal_key, const Compaction* compaction,
    const CompactionFilter* compaction_filter, LogBuffer* log_buffer)
    : input_(input),
      cmp_(cmp),
@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -42,7 +42,7 @@ class CompactionIterator {
                     std::vector<SequenceNumber>* snapshots,
                     SequenceNumber earliest_write_conflict_snapshot, Env* env,
                     bool expect_valid_internal_key,
-                     Compaction* compaction = nullptr,
+                     const Compaction* compaction = nullptr,
                     const CompactionFilter* compaction_filter = nullptr,
                     LogBuffer* log_buffer = nullptr);

@@ -92,7 +92,7 @@ class CompactionIterator {
  const SequenceNumber earliest_write_conflict_snapshot_;
  Env* env_;
  bool expect_valid_internal_key_;
-  Compaction* compaction_;
+  const Compaction* compaction_;
  const CompactionFilter* compaction_filter_;
  LogBuffer* log_buffer_;
  bool bottommost_level_;
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -51,6 +51,7 @@
 #include "util/iostats_context_imp.h"
 #include "util/log_buffer.h"
 #include "util/logging.h"
+#include "util/sst_file_manager_impl.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
@@ -62,7 +63,7 @@ namespace rocksdb {

 // Maintains state for each sub-compaction
 struct CompactionJob::SubcompactionState {
-  Compaction* compaction;
+  const Compaction* compaction;
  std::unique_ptr<CompactionIterator> c_iter;

  // The boundaries of the key-range this compaction is interested in. No two
@@ -103,6 +104,14 @@ struct CompactionJob::SubcompactionState {
  uint64_t num_output_records;
  CompactionJobStats compaction_job_stats;
  uint64_t approx_size;
+  // An index that used to speed up Compaction::ShouldStopBefore().
+  size_t grandparent_index = 0;
+  // The number of bytes overlapping between the current output and
+  // grandparent files used in Compaction::ShouldStopBefore().
+  uint64_t overlapped_bytes = 0;
+  // A flag determine whether the key has been seen in
+  // Compaction::ShouldStopBefore()
+  bool seen_key = false;

  SubcompactionState(Compaction* c, Slice* _start, Slice* _end,
                     uint64_t size = 0)
@@ -114,7 +123,10 @@ struct CompactionJob::SubcompactionState {
        total_bytes(0),
        num_input_records(0),
        num_output_records(0),
-        approx_size(size) {
+        approx_size(size),
+        grandparent_index(0),
+        overlapped_bytes(0),
+        seen_key(false) {
    assert(compaction != nullptr);
  }

@@ -133,6 +145,9 @@ struct CompactionJob::SubcompactionState {
    num_output_records = std::move(o.num_output_records);
    compaction_job_stats = std::move(o.compaction_job_stats);
    approx_size = std::move(o.approx_size);
+    grandparent_index = std::move(o.grandparent_index);
+    overlapped_bytes = std::move(o.overlapped_bytes);
+    seen_key = std::move(o.seen_key);
    return *this;
  }

@@ -140,6 +155,38 @@ struct CompactionJob::SubcompactionState {
  SubcompactionState(const SubcompactionState&) = delete;

  SubcompactionState& operator=(const SubcompactionState&) = delete;
+
+  // Returns true iff we should stop building the current output
+  // before processing "internal_key".
+  bool ShouldStopBefore(const Slice& internal_key) {
+    const InternalKeyComparator* icmp =
+        &compaction->column_family_data()->internal_comparator();
+    const std::vector<FileMetaData*>& grandparents = compaction->grandparents();
+
+    // Scan to find earliest grandparent file that contains key.
+    while (grandparent_index < grandparents.size() &&
+           icmp->Compare(internal_key,
+                         grandparents[grandparent_index]->largest.Encode()) >
+               0) {
+      if (seen_key) {
+        overlapped_bytes += grandparents[grandparent_index]->fd.GetFileSize();
+      }
+      assert(grandparent_index + 1 >= grandparents.size() ||
+             icmp->Compare(
+                 grandparents[grandparent_index]->largest.Encode(),
+                 grandparents[grandparent_index + 1]->smallest.Encode()) < 0);
+      grandparent_index++;
+    }
+    seen_key = true;
+
+    if (overlapped_bytes > compaction->max_grandparent_overlap_bytes()) {
+      // Too much overlap for current output; start new output
+      overlapped_bytes = 0;
+      return true;
+    }
+
+    return false;
+  }
 };

 // Maintains state for the entire compaction
@@ -211,6 +258,7 @@ CompactionJob::CompactionJob(
    const EnvOptions& env_options, VersionSet* versions,
    std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
    Directory* db_directory, Directory* output_directory, Statistics* stats,
+    InstrumentedMutex* db_mutex, Status* db_bg_error,
    std::vector<SequenceNumber> existing_snapshots,
    SequenceNumber earliest_write_conflict_snapshot,
    std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
@@ -230,6 +278,8 @@ CompactionJob::CompactionJob(
      db_directory_(db_directory),
      output_directory_(output_directory),
      stats_(stats),
+      db_mutex_(db_mutex),
+      db_bg_error_(db_bg_error),
      existing_snapshots_(std::move(existing_snapshots)),
      earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
      table_cache_(std::move(table_cache)),
@@ -237,7 +287,9 @@ CompactionJob::CompactionJob(
      paranoid_file_checks_(paranoid_file_checks),
      measure_io_stats_(measure_io_stats) {
  assert(log_buffer_ != nullptr);
-  ThreadStatusUtil::SetColumnFamily(compact_->compaction->column_family_data());
+  const auto* cfd = compact_->compaction->column_family_data();
+  ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
+                                    cfd->options()->enable_thread_tracking);
  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_COMPACTION);
  ReportStartedCompaction(compaction);
 }
@@ -249,8 +301,9 @@ CompactionJob::~CompactionJob() {

 void CompactionJob::ReportStartedCompaction(
    Compaction* compaction) {
-  ThreadStatusUtil::SetColumnFamily(
-      compact_->compaction->column_family_data());
+  const auto* cfd = compact_->compaction->column_family_data();
+  ThreadStatusUtil::SetColumnFamily(cfd, cfd->ioptions()->env,
+                                    cfd->options()->enable_thread_tracking);

  ThreadStatusUtil::SetThreadOperationProperty(
      ThreadStatus::COMPACTION_JOB_ID,
@@ -356,7 +409,7 @@ void CompactionJob::GenSubcompactionBoundaries() {
      size_t num_files = flevel->num_files;

      if (num_files == 0) {
-        break;
+        continue;
      }

      if (lvl == 0) {
@@ -415,12 +468,9 @@ void CompactionJob::GenSubcompactionBoundaries() {

  // Group the ranges into subcompactions
  const double min_file_fill_percent = 4.0 / 5;
-  uint64_t max_output_files = 
-    static_cast<uint64_t>(
-        std::ceil(
-          sum / min_file_fill_percent /
-          cfd->GetCurrentMutableCFOptions()->MaxFileSizeForLevel(out_lvl))
-          );
+  uint64_t max_output_files = static_cast<uint64_t>(std::ceil(
+      sum / min_file_fill_percent /
+      cfd->GetCurrentMutableCFOptions()->MaxFileSizeForLevel(out_lvl)));
  uint64_t subcompactions =
      std::min({static_cast<uint64_t>(ranges.size()),
                static_cast<uint64_t>(db_options_.max_subcompactions),
@@ -518,18 +568,17 @@ Status CompactionJob::Run() {
  return status;
 }

-Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options,
-                              InstrumentedMutex* db_mutex) {
+Status CompactionJob::Install(const MutableCFOptions& mutable_cf_options) {
  AutoThreadOperationStageUpdater stage_updater(
      ThreadStatus::STAGE_COMPACTION_INSTALL);
-  db_mutex->AssertHeld();
+  db_mutex_->AssertHeld();
  Status status = compact_->status;
  ColumnFamilyData* cfd = compact_->compaction->column_family_data();
  cfd->internal_stats()->AddCompactionStats(
      compact_->compaction->output_level(), compaction_stats_);

  if (status.ok()) {
-    status = InstallCompactionResults(mutable_cf_options, db_mutex);
+    status = InstallCompactionResults(mutable_cf_options);
  }
  VersionStorageInfo::LevelSummaryStorage tmp;
  auto vstorage = cfd->current()->storage_info();
@@ -666,7 +715,7 @@ void CompactionJob::ProcessKeyValueCompaction(SubcompactionState* sub_compact) {
    if (end != nullptr &&
        cfd->user_comparator()->Compare(c_iter->user_key(), *end) >= 0) {
      break;
-    } else if (sub_compact->compaction->ShouldStopBefore(key) &&
+    } else if (sub_compact->ShouldStopBefore(key) &&
               sub_compact->builder != nullptr) {
      status = FinishCompactionOutputFile(input->status(), sub_compact);
      if (!status.ok()) {
@@ -855,13 +904,33 @@ Status CompactionJob::FinishCompactionOutputFile(
          event_logger_, cfd->ioptions()->listeners, meta->fd, info);
    }
  }
+
+  // Report new file to SstFileManagerImpl
+  auto sfm =
+      static_cast<SstFileManagerImpl*>(db_options_.sst_file_manager.get());
+  if (sfm && meta->fd.GetPathId() == 0) {
+    ColumnFamilyData* cfd = sub_compact->compaction->column_family_data();
+    auto fn = TableFileName(cfd->ioptions()->db_paths, meta->fd.GetNumber(),
+                            meta->fd.GetPathId());
+    sfm->OnAddFile(fn);
+    if (sfm->IsMaxAllowedSpaceReached()) {
+      InstrumentedMutexLock l(db_mutex_);
+      if (db_bg_error_->ok()) {
+        s = Status::IOError("Max allowed space was reached");
+        *db_bg_error_ = s;
+        TEST_SYNC_POINT(
+            "CompactionJob::FinishCompactionOutputFile:MaxAllowedSpaceReached");
+      }
+    }
+  }
+
  sub_compact->builder.reset();
  return s;
 }

 Status CompactionJob::InstallCompactionResults(
-    const MutableCFOptions& mutable_cf_options, InstrumentedMutex* db_mutex) {
-  db_mutex->AssertHeld();
+    const MutableCFOptions& mutable_cf_options) {
+  db_mutex_->AssertHeld();

  auto* compaction = compact_->compaction;
  // paranoia: verify that the files that we started with
@@ -896,7 +965,7 @@ Status CompactionJob::InstallCompactionResults(
  }
  return versions_->LogAndApply(compaction->column_family_data(),
                                mutable_cf_options, compaction->edit(),
-                                db_mutex, db_directory_);
+                                db_mutex_, db_directory_);
 }

 void CompactionJob::RecordCompactionIOStats() {
@@ -950,7 +1019,7 @@ Status CompactionJob::OpenCompactionOutputFile(
      cfd->ioptions()->optimize_filters_for_hits && bottommost_level_;
  sub_compact->builder.reset(NewTableBuilder(
      *cfd->ioptions(), cfd->internal_comparator(),
-      cfd->int_tbl_prop_collector_factories(), cfd->GetID(),
+      cfd->int_tbl_prop_collector_factories(), cfd->GetID(), cfd->GetName(),
      sub_compact->outfile.get(), sub_compact->compaction->output_compression(),
      cfd->ioptions()->compression_opts, skip_filters));
  LogFlush(db_options_.info_log);
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -56,7 +56,8 @@ class CompactionJob {
                const EnvOptions& env_options, VersionSet* versions,
                std::atomic<bool>* shutting_down, LogBuffer* log_buffer,
                Directory* db_directory, Directory* output_directory,
-                Statistics* stats,
+                Statistics* stats, InstrumentedMutex* db_mutex,
+                Status* db_bg_error,
                std::vector<SequenceNumber> existing_snapshots,
                SequenceNumber earliest_write_conflict_snapshot,
                std::shared_ptr<Cache> table_cache, EventLogger* event_logger,
@@ -77,8 +78,7 @@ class CompactionJob {
  Status Run();

  // REQUIRED: mutex held
-  Status Install(const MutableCFOptions& mutable_cf_options,
-                 InstrumentedMutex* db_mutex);
+  Status Install(const MutableCFOptions& mutable_cf_options);

 private:
  struct SubcompactionState;
@@ -95,8 +95,7 @@ class CompactionJob {

  Status FinishCompactionOutputFile(const Status& input_status,
                                    SubcompactionState* sub_compact);
-  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options,
-                                  InstrumentedMutex* db_mutex);
+  Status InstallCompactionResults(const MutableCFOptions& mutable_cf_options);
  void RecordCompactionIOStats();
  Status OpenCompactionOutputFile(SubcompactionState* sub_compact);
  void CleanupCompaction();
@@ -130,6 +129,8 @@ class CompactionJob {
  Directory* db_directory_;
  Directory* output_directory_;
  Statistics* stats_;
+  InstrumentedMutex* db_mutex_;
+  Status* db_bg_error_;
  // If there were two snapshots with seq numbers s1 and
  // s2 and s1 < s2, and if we find two instances of a key k1 then lies
  // entirely within s1 and s2, then the earlier version of k1 can be safely
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -660,7 +660,7 @@ TEST_P(CompactionJobStatsTest, CompactionJobStatsTest) {
  options.max_subcompactions = max_subcompactions_;
  options.bytes_per_sync = 512 * 1024;

-  options.compaction_measure_io_stats = true;
+  options.report_bg_io_stats = true;
  for (int test = 0; test < 2; ++test) {
    DestroyAndReopen(options);
    CreateAndReopenWithCF({"pikachu"}, options);
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -250,9 +250,9 @@ class CompactionJobTest : public testing::Test {
    EventLogger event_logger(db_options_.info_log.get());
    CompactionJob compaction_job(
        0, &compaction, db_options_, env_options_, versions_.get(),
-        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, snapshots,
-        earliest_write_conflict_snapshot, table_cache_, &event_logger, false,
-        false, dbname_, &compaction_job_stats_);
+        &shutting_down_, &log_buffer, nullptr, nullptr, nullptr, &mutex_,
+        &bg_error_, snapshots, earliest_write_conflict_snapshot, table_cache_,
+        &event_logger, false, false, dbname_, &compaction_job_stats_);

    VerifyInitializationOfCompactionJobStats(compaction_job_stats_);

@@ -262,8 +262,7 @@ class CompactionJobTest : public testing::Test {
    s = compaction_job.Run();
    ASSERT_OK(s);
    mutex_.Lock();
-    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions(),
-                                     &mutex_));
+    ASSERT_OK(compaction_job.Install(*cfd->GetLatestMutableCFOptions()));
    mutex_.Unlock();

    if (expected_results.size() == 0) {
@@ -295,6 +294,7 @@ class CompactionJobTest : public testing::Test {
  ColumnFamilyData* cfd_;
  std::unique_ptr<CompactionFilter> compaction_filter_;
  std::shared_ptr<MergeOperator> merge_op_;
+  Status bg_error_;
 };

 TEST_F(CompactionJobTest, Simple) {
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -141,7 +141,8 @@ CompactionPicker::~CompactionPicker() {}

 // Delete this compaction from the list of running compactions.
 void CompactionPicker::ReleaseCompactionFiles(Compaction* c, Status status) {
-  if (c->start_level() == 0) {
+  if (c->start_level() == 0 ||
+      ioptions_.compaction_style == kCompactionStyleUniversal) {
    level0_compactions_in_progress_.erase(c);
  }
  if (!status.ok()) {
@@ -612,6 +613,17 @@ Compaction* CompactionPicker::CompactRange(
  if (input_level == 0) {
    level0_compactions_in_progress_.insert(compaction);
  }
+
+  // Creating a compaction influences the compaction score because the score
+  // takes running compactions into account (by skipping files that are already
+  // being compacted). Since we just changed compaction score, we recalculate it
+  // here
+  {  // this piece of code recomputes compaction score
+    CompactionOptionsFIFO dummy_compaction_options_fifo;
+    vstorage->ComputeCompactionScore(mutable_cf_options,
+                                     dummy_compaction_options_fifo);
+  }
+
  return compaction;
 }

@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -83,6 +83,8 @@ class CompactionPicker {
 #endif  // ROCKSDB_LITE

  // Free up the files that participated in a compaction
+  //
+  // Requirement: DB mutex held
  void ReleaseCompactionFiles(Compaction* c, Status status);

  // Returns true if any one of the specified files are being compacted
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -195,6 +195,7 @@ TEST_F(CompactionPickerTest, LevelMaxScore) {
  NewVersionStorage(6, kCompactionStyleLevel);
  mutable_cf_options_.target_file_size_base = 10000000;
  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
  Add(0, 1U, "150", "200", 1000000000U);
  // Level 1 score 1.2
  Add(1, 66U, "150", "200", 6000000U);
@@ -487,6 +488,90 @@ TEST_F(CompactionPickerTest, NeedsCompactionFIFO) {
 }
 #endif  // ROCKSDB_LITE

+TEST_F(CompactionPickerTest, CompactionPriMinOverlapping1) {
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.target_file_size_base = 10000000;
+  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
+
+  Add(2, 6U, "150", "179", 50000000U);
+  Add(2, 7U, "180", "220", 50000000U);
+  Add(2, 8U, "321", "400", 50000000U);  // File not overlapping
+  Add(2, 9U, "721", "800", 50000000U);
+
+  Add(3, 26U, "150", "170", 260000000U);
+  Add(3, 27U, "171", "179", 260000000U);
+  Add(3, 28U, "191", "220", 260000000U);
+  Add(3, 29U, "221", "300", 260000000U);
+  Add(3, 30U, "750", "900", 260000000U);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(1U, compaction->num_input_files(0));
+  // Pick file 8 because it overlaps with 0 files on level 3.
+  ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
+}
+
+TEST_F(CompactionPickerTest, CompactionPriMinOverlapping2) {
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.target_file_size_base = 10000000;
+  mutable_cf_options_.target_file_size_multiplier = 10;
+  mutable_cf_options_.max_bytes_for_level_base = 10 * 1024 * 1024;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
+
+  Add(2, 6U, "150", "175",
+      60000000U);  // Overlaps with file 26, 27, total size 521M
+  Add(2, 7U, "176", "200", 60000000U);  // Overlaps with file 27, 28, total size
+                                        // 520M, the smalelst overlapping
+  Add(2, 8U, "201", "300",
+      60000000U);  // Overlaps with file 28, 29, total size 521M
+
+  Add(3, 26U, "100", "110", 261000000U);
+  Add(3, 26U, "150", "170", 261000000U);
+  Add(3, 27U, "171", "179", 260000000U);
+  Add(3, 28U, "191", "220", 260000000U);
+  Add(3, 29U, "221", "300", 261000000U);
+  Add(3, 30U, "321", "400", 261000000U);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(1U, compaction->num_input_files(0));
+  // Picking file 7 because overlapping ratio is the biggest.
+  ASSERT_EQ(7U, compaction->input(0, 0)->fd.GetNumber());
+}
+
+TEST_F(CompactionPickerTest, CompactionPriMinOverlapping3) {
+  NewVersionStorage(6, kCompactionStyleLevel);
+  mutable_cf_options_.max_bytes_for_level_base = 10000000;
+  mutable_cf_options_.max_bytes_for_level_multiplier = 10;
+  mutable_cf_options_.compaction_pri = kMinOverlappingRatio;
+
+  // file 7 and 8 over lap with the same file, but file 8 is smaller so
+  // it will be picked.
+  Add(2, 6U, "150", "167", 60000000U);  // Overlaps with file 26, 27
+  Add(2, 7U, "168", "169", 60000000U);  // Overlaps with file 27
+  Add(2, 8U, "201", "300", 61000000U);  // Overlaps with file 28, but the file
+                                        // itself is larger. Should be picked.
+
+  Add(3, 26U, "160", "165", 260000000U);
+  Add(3, 27U, "166", "170", 260000000U);
+  Add(3, 28U, "180", "400", 260000000U);
+  Add(3, 29U, "401", "500", 260000000U);
+  UpdateVersionStorageInfo();
+
+  std::unique_ptr<Compaction> compaction(level_compaction_picker.PickCompaction(
+      cf_name_, mutable_cf_options_, vstorage_.get(), &log_buffer_));
+  ASSERT_TRUE(compaction.get() != nullptr);
+  ASSERT_EQ(1U, compaction->num_input_files(0));
+  // Picking file 8 because overlapping ratio is the biggest.
+  ASSERT_EQ(8U, compaction->input(0, 0)->fd.GetNumber());
+}
+
 // This test exhibits the bug where we don't properly reset parent_index in
 // PickCompaction()
 TEST_F(CompactionPickerTest, ParentIndexResetBug) {
@@ -1,6 +1,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -0,0 +1,240 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include <cstdlib>
+#include "db/db_test_util.h"
+#include "port/stack_trace.h"
+
+namespace rocksdb {
+
+static uint64_t TestGetTickerCount(const Options& options,
+                                   Tickers ticker_type) {
+  return options.statistics->getTickerCount(ticker_type);
+}
+
+class DBBlockCacheTest : public DBTestBase {
+ private:
+  size_t miss_count_ = 0;
+  size_t hit_count_ = 0;
+  size_t insert_count_ = 0;
+  size_t failure_count_ = 0;
+  size_t compressed_miss_count_ = 0;
+  size_t compressed_hit_count_ = 0;
+  size_t compressed_insert_count_ = 0;
+  size_t compressed_failure_count_ = 0;
+
+ public:
+  const size_t kNumBlocks = 10;
+  const size_t kValueSize = 100;
+
+  DBBlockCacheTest() : DBTestBase("/db_block_cache_test") {}
+
+  BlockBasedTableOptions GetTableOptions() {
+    BlockBasedTableOptions table_options;
+    // Set a small enough block size so that each key-value get its own block.
+    table_options.block_size = 1;
+    return table_options;
+  }
+
+  Options GetOptions(const BlockBasedTableOptions& table_options) {
+    Options options = CurrentOptions();
+    options.create_if_missing = true;
+    // options.compression = kNoCompression;
+    options.statistics = rocksdb::CreateDBStatistics();
+    options.table_factory.reset(new BlockBasedTableFactory(table_options));
+    return options;
+  }
+
+  void InitTable(const Options& options) {
+    std::string value(kValueSize, 'a');
+    for (size_t i = 0; i < kNumBlocks; i++) {
+      ASSERT_OK(Put(ToString(i), value.c_str()));
+    }
+  }
+
+  void RecordCacheCounters(const Options& options) {
+    miss_count_ = TestGetTickerCount(options, BLOCK_CACHE_MISS);
+    hit_count_ = TestGetTickerCount(options, BLOCK_CACHE_HIT);
+    insert_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD);
+    failure_count_ = TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
+    compressed_miss_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
+    compressed_hit_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
+    compressed_insert_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
+    compressed_failure_count_ =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
+  }
+
+  void CheckCacheCounters(const Options& options, size_t expected_misses,
+                          size_t expected_hits, size_t expected_inserts,
+                          size_t expected_failures) {
+    size_t new_miss_count = TestGetTickerCount(options, BLOCK_CACHE_MISS);
+    size_t new_hit_count = TestGetTickerCount(options, BLOCK_CACHE_HIT);
+    size_t new_insert_count = TestGetTickerCount(options, BLOCK_CACHE_ADD);
+    size_t new_failure_count =
+        TestGetTickerCount(options, BLOCK_CACHE_ADD_FAILURES);
+    ASSERT_EQ(miss_count_ + expected_misses, new_miss_count);
+    ASSERT_EQ(hit_count_ + expected_hits, new_hit_count);
+    ASSERT_EQ(insert_count_ + expected_inserts, new_insert_count);
+    ASSERT_EQ(failure_count_ + expected_failures, new_failure_count);
+    miss_count_ = new_miss_count;
+    hit_count_ = new_hit_count;
+    insert_count_ = new_insert_count;
+    failure_count_ = new_failure_count;
+  }
+
+  void CheckCompressedCacheCounters(const Options& options,
+                                    size_t expected_misses,
+                                    size_t expected_hits,
+                                    size_t expected_inserts,
+                                    size_t expected_failures) {
+    size_t new_miss_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_MISS);
+    size_t new_hit_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_HIT);
+    size_t new_insert_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD);
+    size_t new_failure_count =
+        TestGetTickerCount(options, BLOCK_CACHE_COMPRESSED_ADD_FAILURES);
+    ASSERT_EQ(compressed_miss_count_ + expected_misses, new_miss_count);
+    ASSERT_EQ(compressed_hit_count_ + expected_hits, new_hit_count);
+    ASSERT_EQ(compressed_insert_count_ + expected_inserts, new_insert_count);
+    ASSERT_EQ(compressed_failure_count_ + expected_failures, new_failure_count);
+    compressed_miss_count_ = new_miss_count;
+    compressed_hit_count_ = new_hit_count;
+    compressed_insert_count_ = new_insert_count;
+    compressed_failure_count_ = new_failure_count;
+  }
+};
+
+TEST_F(DBBlockCacheTest, TestWithoutCompressedBlockCache) {
+  ReadOptions read_options;
+  auto table_options = GetTableOptions();
+  auto options = GetOptions(table_options);
+  InitTable(options);
+
+  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  table_options.block_cache = cache;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  Reopen(options);
+  RecordCacheCounters(options);
+
+  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
+  Iterator* iter = nullptr;
+
+  // Load blocks into cache.
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iter = db_->NewIterator(read_options);
+    iter->Seek(ToString(i));
+    ASSERT_OK(iter->status());
+    CheckCacheCounters(options, 1, 0, 1, 0);
+    iterators[i].reset(iter);
+  }
+  size_t usage = cache->GetUsage();
+  ASSERT_LT(0, usage);
+  cache->SetCapacity(usage);
+  ASSERT_EQ(usage, cache->GetPinnedUsage());
+
+  // Test with strict capacity limit.
+  cache->SetStrictCapacityLimit(true);
+  iter = db_->NewIterator(read_options);
+  iter->Seek(ToString(kNumBlocks - 1));
+  ASSERT_TRUE(iter->status().IsIncomplete());
+  CheckCacheCounters(options, 1, 0, 0, 1);
+  delete iter;
+  iter = nullptr;
+
+  // Release interators and access cache again.
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iterators[i].reset();
+    CheckCacheCounters(options, 0, 0, 0, 0);
+  }
+  ASSERT_EQ(0, cache->GetPinnedUsage());
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iter = db_->NewIterator(read_options);
+    iter->Seek(ToString(i));
+    ASSERT_OK(iter->status());
+    CheckCacheCounters(options, 0, 1, 0, 0);
+    iterators[i].reset(iter);
+  }
+}
+
+#ifdef SNAPPY
+TEST_F(DBBlockCacheTest, TestWithCompressedBlockCache) {
+  ReadOptions read_options;
+  auto table_options = GetTableOptions();
+  auto options = GetOptions(table_options);
+  options.compression = CompressionType::kSnappyCompression;
+  InitTable(options);
+
+  std::shared_ptr<Cache> cache = NewLRUCache(0, 0, false);
+  std::shared_ptr<Cache> compressed_cache = NewLRUCache(0, 0, false);
+  table_options.block_cache = cache;
+  table_options.block_cache_compressed = compressed_cache;
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  Reopen(options);
+  RecordCacheCounters(options);
+
+  std::vector<std::unique_ptr<Iterator>> iterators(kNumBlocks - 1);
+  Iterator* iter = nullptr;
+
+  // Load blocks into cache.
+  for (size_t i = 0; i < kNumBlocks - 1; i++) {
+    iter = db_->NewIterator(read_options);
+    iter->Seek(ToString(i));
+    ASSERT_OK(iter->status());
+    CheckCacheCounters(options, 1, 0, 1, 0);
+    CheckCompressedCacheCounters(options, 1, 0, 1, 0);
+    iterators[i].reset(iter);
+  }
+  size_t usage = cache->GetUsage();
+  ASSERT_LT(0, usage);
+  ASSERT_EQ(usage, cache->GetPinnedUsage());
+  size_t compressed_usage = compressed_cache->GetUsage();
+  ASSERT_LT(0, compressed_usage);
+  // Compressed block cache cannot be pinned.
+  ASSERT_EQ(0, compressed_cache->GetPinnedUsage());
+
+  // Set strict capacity limit flag. Now block will only load into compressed
+  // block cache.
+  cache->SetCapacity(usage);
+  cache->SetStrictCapacityLimit(true);
+  ASSERT_EQ(usage, cache->GetPinnedUsage());
+  // compressed_cache->SetCapacity(compressed_usage);
+  compressed_cache->SetCapacity(0);
+  // compressed_cache->SetStrictCapacityLimit(true);
+  iter = db_->NewIterator(read_options);
+  iter->Seek(ToString(kNumBlocks - 1));
+  ASSERT_TRUE(iter->status().IsIncomplete());
+  CheckCacheCounters(options, 1, 0, 0, 1);
+  CheckCompressedCacheCounters(options, 1, 0, 1, 0);
+  delete iter;
+  iter = nullptr;
+
+  // Clear strict capacity limit flag. This time we shall hit compressed block
+  // cache.
+  cache->SetStrictCapacityLimit(false);
+  iter = db_->NewIterator(read_options);
+  iter->Seek(ToString(kNumBlocks - 1));
+  ASSERT_OK(iter->status());
+  CheckCacheCounters(options, 1, 0, 1, 0);
+  CheckCompressedCacheCounters(options, 0, 1, 0, 0);
+  delete iter;
+  iter = nullptr;
+}
+#endif
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -368,11 +368,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilter) {
 // entries are deleted. The compaction should create bunch of 'DeleteFile'
 // entries in VersionEdit, but none of the 'AddFile's.
 TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // put some data
@@ -400,11 +399,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterDeletesAll) {

 TEST_F(DBTestCompactionFilter, CompactionFilterWithValueChange) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.num_levels = 3;
    options.compaction_filter_factory =
      std::make_shared<ChangeFilterFactory>();
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Write 100K+1 keys, these are written to a few files
@@ -467,8 +465,7 @@ TEST_F(DBTestCompactionFilter, CompactionFilterWithMergeOperator) {
  PutFixed64(&three, 3);
  PutFixed64(&four, 4);

-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.create_if_missing = true;
  options.merge_operator = MergeOperators::CreateUInt64AddOperator();
  options.num_levels = 3;
@@ -621,11 +618,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterContextCfId) {
 // Compaction filters should only be applied to records that are newer than the
 // latest snapshot. This test inserts records and applies a delete filter.
 TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<DeleteFilterFactory>();
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // Put some data.
@@ -659,11 +655,10 @@ TEST_F(DBTestCompactionFilter, CompactionFilterSnapshot) {
 // records newer than the snapshot will also be processed
 TEST_F(DBTestCompactionFilter, CompactionFilterIgnoreSnapshot) {
  std::string five = ToString(5);
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_filter_factory = std::make_shared<DeleteISFilterFactory>();
  options.disable_auto_compactions = true;
  options.create_if_missing = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // Put some data.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -288,8 +288,7 @@ TEST_F(DBCompactionTest, SkipStatsUpdateTest) {
 }

 TEST_F(DBCompactionTest, TestTableReaderForCompaction) {
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();
  options.env = env_;
  options.new_table_reader_for_compaction_inputs = true;
  options.max_open_files = 100;
@@ -493,14 +492,13 @@ TEST_F(DBCompactionTest, DisableStatsUpdateReopen) {
 TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
  const int kNumKeysPerFile = 100;

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 << 10;
  options.num_levels = 3;
  options.level0_file_num_compaction_trigger = 3;
  options.max_subcompactions = max_subcompactions_;
  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  Random rnd(301);
@@ -533,11 +531,107 @@ TEST_P(DBCompactionTestWithParam, CompactionTrigger) {
  ASSERT_EQ(NumTableFilesAtLevel(1, 1), 1);
 }

+TEST_F(DBCompactionTest, BGCompactionsAllowed) {
+  // Create several column families. Make compaction triggers in all of them
+  // and see number of compactions scheduled to be less than allowed.
+  const int kNumKeysPerFile = 100;
+
+  Options options = CurrentOptions();
+  options.write_buffer_size = 110 << 10;  // 110KB
+  options.arena_block_size = 4 << 10;
+  options.num_levels = 3;
+  // Should speed up compaction when there are 4 files.
+  options.level0_file_num_compaction_trigger = 2;
+  options.level0_slowdown_writes_trigger = 20;
+  options.soft_pending_compaction_bytes_limit = 1 << 30;  // Infinitely large
+  options.base_background_compactions = 1;
+  options.max_background_compactions = 3;
+  options.memtable_factory.reset(new SpecialSkipListFactory(kNumKeysPerFile));
+
+  // Block all threads in thread pool.
+  const size_t kTotalTasks = 4;
+  env_->SetBackgroundThreads(4, Env::LOW);
+  test::SleepingBackgroundTask sleeping_tasks[kTotalTasks];
+  for (size_t i = 0; i < kTotalTasks; i++) {
+    env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                   &sleeping_tasks[i], Env::Priority::LOW);
+    sleeping_tasks[i].WaitUntilSleeping();
+  }
+
+  CreateAndReopenWithCF({"one", "two", "three"}, options);
+
+  Random rnd(301);
+  for (int cf = 0; cf < 4; cf++) {
+    for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
+      for (int i = 0; i < kNumKeysPerFile; i++) {
+        ASSERT_OK(Put(cf, Key(i), ""));
+      }
+      // put extra key to trigger flush
+      ASSERT_OK(Put(cf, "", ""));
+      dbfull()->TEST_WaitForFlushMemTable(handles_[cf]);
+      ASSERT_EQ(NumTableFilesAtLevel(0, cf), num + 1);
+    }
+  }
+
+  // Now all column families qualify compaction but only one should be
+  // scheduled, because no column family hits speed up condition.
+  ASSERT_EQ(1, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
+
+  // Create two more files for one column family, which triggers speed up
+  // condition, three compactions will be scheduled.
+  for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
+    for (int i = 0; i < kNumKeysPerFile; i++) {
+      ASSERT_OK(Put(2, Key(i), ""));
+    }
+    // put extra key to trigger flush
+    ASSERT_OK(Put(2, "", ""));
+    dbfull()->TEST_WaitForFlushMemTable(handles_[2]);
+    ASSERT_EQ(options.level0_file_num_compaction_trigger + num + 1,
+              NumTableFilesAtLevel(0, 2));
+  }
+  ASSERT_EQ(3, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
+
+  // Unblock all threads to unblock all compactions.
+  for (size_t i = 0; i < kTotalTasks; i++) {
+    sleeping_tasks[i].WakeUp();
+    sleeping_tasks[i].WaitUntilDone();
+  }
+  dbfull()->TEST_WaitForCompact();
+
+  // Verify number of compactions allowed will come back to 1.
+
+  for (size_t i = 0; i < kTotalTasks; i++) {
+    sleeping_tasks[i].Reset();
+    env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask,
+                   &sleeping_tasks[i], Env::Priority::LOW);
+    sleeping_tasks[i].WaitUntilSleeping();
+  }
+  for (int cf = 0; cf < 4; cf++) {
+    for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
+      for (int i = 0; i < kNumKeysPerFile; i++) {
+        ASSERT_OK(Put(cf, Key(i), ""));
+      }
+      // put extra key to trigger flush
+      ASSERT_OK(Put(cf, "", ""));
+      dbfull()->TEST_WaitForFlushMemTable(handles_[cf]);
+      ASSERT_EQ(NumTableFilesAtLevel(0, cf), num + 1);
+    }
+  }
+
+  // Now all column families qualify compaction but only one should be
+  // scheduled, because no column family hits speed up condition.
+  ASSERT_EQ(1, env_->GetThreadPoolQueueLen(Env::Priority::LOW));
+
+  for (size_t i = 0; i < kTotalTasks; i++) {
+    sleeping_tasks[i].WakeUp();
+    sleeping_tasks[i].WaitUntilDone();
+  }
+}
+
 TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;        // Large write buffer
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  Random rnd(301);
@@ -564,9 +658,8 @@ TEST_P(DBCompactionTestWithParam, CompactionsGenerateMultipleFiles) {

 TEST_F(DBCompactionTest, MinorCompactionsHappen) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.write_buffer_size = 10000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    const int N = 500;
@@ -591,7 +684,7 @@ TEST_F(DBCompactionTest, MinorCompactionsHappen) {
 }

 TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleLevel;
  options.level0_file_num_compaction_trigger = 3;

@@ -605,7 +698,6 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
  const size_t key_len =
    static_cast<size_t>(compact_opt.output_file_size_limit) / 5;

-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  std::vector<const Snapshot*> snaps;
@@ -649,10 +741,8 @@ TEST_F(DBCompactionTest, ZeroSeqIdCompaction) {
 // if the database is shutdown during the memtable compaction.
 TEST_F(DBCompactionTest, RecoverDuringMemtableCompaction) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.env = env_;
-    options.write_buffer_size = 1000000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Trigger a long memtable compaction and reopen the database during it
@@ -676,10 +766,9 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveOneFile) {
      [&](void* arg) { trivial_move++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  int32_t num_keys = 80;
@@ -1271,10 +1360,9 @@ TEST_P(DBCompactionTestWithParam, TrivialMoveToLastLevelWithFiles) {
      [&](void* arg) { non_trivial_move++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  int32_t value_size = 10 * 1024;  // 10 KB
@@ -1563,7 +1651,7 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
  int max_key_universal_insert = 600;

  // Stage 1: generate a db with level compaction
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 110 << 10;  // 110KB
  options.arena_block_size = 4 << 10;
  options.num_levels = 4;
@@ -1573,7 +1661,6 @@ TEST_P(DBCompactionTestWithParam, ConvertCompactionStyle) {
  options.target_file_size_base = 200 << 10;  // 200KB
  options.target_file_size_multiplier = 1;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  for (int i = 0; i <= max_key_level_insert; i++) {
@@ -1898,7 +1985,7 @@ TEST_P(DBCompactionTestWithParam, DISABLED_CompactFilesOnLevelCompaction) {
    std::set<std::string> overlapping_file_names;
    std::vector<std::string> compaction_input_file_names;
    for (int f = 0; f < file_picked; ++f) {
-      int level;
+      int level = 0;
      auto file_meta = PickFileRandomly(cf_meta, &rnd, &level);
      compaction_input_file_names.push_back(file_meta->name);
      GetOverlappingFileNumbersForLevelCompaction(
@@ -2198,6 +2285,25 @@ TEST_P(DBCompactionTestWithParam, CompressLevelCompaction) {
  Destroy(options);
 }

+TEST_F(DBCompactionTest, SanitizeCompactionOptionsTest) {
+  Options options = CurrentOptions();
+  options.max_background_compactions = 5;
+  options.soft_pending_compaction_bytes_limit = 0;
+  options.hard_pending_compaction_bytes_limit = 100;
+  options.create_if_missing = true;
+  DestroyAndReopen(options);
+  ASSERT_EQ(5, db_->GetOptions().base_background_compactions);
+  ASSERT_EQ(100, db_->GetOptions().soft_pending_compaction_bytes_limit);
+
+  options.base_background_compactions = 4;
+  options.max_background_compactions = 3;
+  options.soft_pending_compaction_bytes_limit = 200;
+  options.hard_pending_compaction_bytes_limit = 150;
+  DestroyAndReopen(options);
+  ASSERT_EQ(3, db_->GetOptions().base_background_compactions);
+  ASSERT_EQ(150, db_->GetOptions().soft_pending_compaction_bytes_limit);
+}
+
 // This tests for a bug that could cause two level0 compactions running
 // concurrently
 // TODO(aekmekji): Make sure that the reason this fails when run with
@@ -2269,10 +2375,9 @@ TEST_P(DBCompactionTestWithParam, ForceBottommostLevelCompaction) {
      [&](void* arg) { non_trivial_move++; });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 100000000;
  options.max_subcompactions = max_subcompactions_;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  int32_t value_size = 10 * 1024;  // 10 KB
@@ -2362,14 +2467,14 @@ class CompactionPriTest : public DBTestBase,
 };

 TEST_P(CompactionPriTest, Test) {
-  Options options;
+  Options options = CurrentOptions();
  options.write_buffer_size = 16 * 1024;
  options.compaction_pri = static_cast<CompactionPri>(compaction_pri_);
  options.hard_pending_compaction_bytes_limit = 256 * 1024;
  options.max_bytes_for_level_base = 64 * 1024;
  options.max_bytes_for_level_multiplier = 4;
  options.compression = kNoCompression;
-  options = CurrentOptions(options);
+
  DestroyAndReopen(options);

  Random rnd(301);
@@ -2390,8 +2495,12 @@ TEST_P(CompactionPriTest, Test) {
  }
 }

-INSTANTIATE_TEST_CASE_P(CompactionPriTest, CompactionPriTest,
-                        ::testing::Values(0, 1, 2));
+INSTANTIATE_TEST_CASE_P(
+    CompactionPriTest, CompactionPriTest,
+    ::testing::Values(CompactionPri::kByCompensatedSize,
+                      CompactionPri::kOldestLargestSeqFirst,
+                      CompactionPri::kOldestSmallestSeqFirst,
+                      CompactionPri::kMinOverlappingRatio));

 #endif // !defined(ROCKSDB_LITE)
 }  // namespace rocksdb
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -347,6 +347,10 @@ class DBImpl : public DB {

 #endif  // NDEBUG

+  // Return maximum background compaction alowed to be scheduled based on
+  // compaction status.
+  int BGCompactionsAllowed() const;
+
  // Returns the list of live files in 'live' and the list
  // of all files in the filesystem in 'candidate_files'.
  // If force == false and the last call was less than
@@ -490,10 +494,11 @@ class DBImpl : public DB {

  // Background process needs to call
  //     auto x = CaptureCurrentFileNumberInPendingOutputs()
+  //     auto file_num = versions_->NewFileNumber();
  //     <do something>
  //     ReleaseFileNumberFromPendingOutputs(x)
-  // This will protect any temporary files created while <do something> is
-  // executing from being deleted.
+  // This will protect any file with number `file_num` or greater from being
+  // deleted while <do something> is running.
  // -----------
  // This function will capture current file number and append it to
  // pending_outputs_. This will prevent any background process to delete any
@@ -518,7 +523,7 @@ class DBImpl : public DB {
                         SequenceNumber* max_sequence, bool read_only);

  // The following two methods are used to flush a memtable to
-  // storage. The first one is used atdatabase RecoveryTime (when the
+  // storage. The first one is used at database RecoveryTime (when the
  // database is opened) and is heavyweight because it holds the mutex
  // for the entire period. The second method WriteLevel0Table supports
  // concurrent flush memtables to storage.
@@ -818,7 +823,10 @@ class DBImpl : public DB {
  // they're unique
  std::atomic<int> next_job_id_;

-  bool flush_on_destroy_; // Used when disableWAL is true.
+  // A flag indicating whether the current rocksdb database has any
+  // data that is not yet persisted into either WAL or SST file.
+  // Used when disableWAL is true.
+  bool has_unpersisted_data_;

  static const int KEEP_LOG_FILE_NUM = 1000;
  // MSVC version 1800 still does not have constexpr for ::max()
@@ -836,9 +844,12 @@ class DBImpl : public DB {
  // Unified interface for logging events
  EventLogger event_logger_;

-  // A value of >0 temporarily disables scheduling of background work
+  // A value of > 0 temporarily disables scheduling of background work
  int bg_work_paused_;

+  // A value of > 0 temporarily disables scheduling of background compaction
+  int bg_compaction_paused_;
+
  // Guard against multiple concurrent refitting
  bool refitting_level_;

@@ -889,9 +900,8 @@ class DBImpl : public DB {
                 bool* value_found = nullptr);

  bool GetIntPropertyInternal(ColumnFamilyData* cfd,
-                              DBPropertyType property_type,
-                              bool need_out_of_mutex, bool is_locked,
-                              uint64_t* value);
+                              const DBPropertyInfo& property_info,
+                              bool is_locked, uint64_t* value);

  bool HasPendingManualCompaction();
  bool HasExclusiveManualCompaction();
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -54,10 +54,11 @@ Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
  auto db_iter = NewArenaWrappedDbIterator(
      env_, *cfd->ioptions(), cfd->user_comparator(),
      (read_options.snapshot != nullptr
-           ? reinterpret_cast<const SnapshotImpl*>(
-                read_options.snapshot)->number_
+           ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
+                 ->number_
           : latest_snapshot),
-      super_version->mutable_cf_options.max_sequential_skip_in_iterations);
+      super_version->mutable_cf_options.max_sequential_skip_in_iterations,
+      super_version->version_number);
  auto internal_iter = NewInternalIterator(
      read_options, cfd, super_version, db_iter->GetArena());
  db_iter->SetIterUnderDBIter(internal_iter);
@@ -81,10 +82,11 @@ Status DBImplReadOnly::NewIterators(
    auto* db_iter = NewArenaWrappedDbIterator(
        env_, *cfd->ioptions(), cfd->user_comparator(),
        (read_options.snapshot != nullptr
-            ? reinterpret_cast<const SnapshotImpl*>(
-                  read_options.snapshot)->number_
-            : latest_snapshot),
-        sv->mutable_cf_options.max_sequential_skip_in_iterations);
+             ? reinterpret_cast<const SnapshotImpl*>(read_options.snapshot)
+                   ->number_
+             : latest_snapshot),
+        sv->mutable_cf_options.max_sequential_skip_in_iterations,
+        sv->version_number);
    auto* internal_iter = NewInternalIterator(
        read_options, cfd, sv, db_iter->GetArena());
    db_iter->SetIterUnderDBIter(internal_iter);
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -7,6 +7,8 @@
 #define __STDC_FORMAT_MACROS
 #endif

+#include "db/db_info_dumper.h"
+
 #include <inttypes.h>
 #include <stdio.h>
 #include <string>
@@ -16,7 +18,6 @@
 #include "db/filename.h"
 #include "rocksdb/options.h"
 #include "rocksdb/env.h"
-#include "util/db_info_dumper.h"

 namespace rocksdb {

@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -18,12 +18,11 @@ class DBTestInPlaceUpdate : public DBTestBase {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;
    options.env = env_;
    options.write_buffer_size = 100000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of smaller size
@@ -41,12 +40,11 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdate) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;
    options.env = env_;
    options.write_buffer_size = 100000;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of larger size
@@ -64,7 +62,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateLargeNewValue) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -72,7 +70,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerSize;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of smaller size
@@ -92,7 +89,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerSize) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -100,7 +97,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceSmallerVarintSize;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of smaller varint size
@@ -120,7 +116,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackSmallerVarintSize) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -128,7 +124,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceLargerSize;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Update key with values of larger size
@@ -146,7 +141,7 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackLargeNewValue) {

 TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
  do {
-    Options options;
+    Options options = CurrentOptions();
    options.create_if_missing = true;
    options.inplace_update_support = true;

@@ -154,7 +149,6 @@ TEST_F(DBTestInPlaceUpdate, InPlaceUpdateCallbackNoAction) {
    options.write_buffer_size = 100000;
    options.inplace_callback =
      rocksdb::DBTestInPlaceUpdate::updateInPlaceNoAction;
-    options = CurrentOptions(options);
    CreateAndReopenWithCF({"pikachu"}, options);

    // Callback function requests no actions from db
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -13,18 +13,20 @@
 #include <string>
 #include <limits>

-#include "db/filename.h"
 #include "db/dbformat.h"
+#include "db/filename.h"
+#include "db/merge_context.h"
 #include "port/port.h"
 #include "rocksdb/env.h"
-#include "rocksdb/options.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/merge_operator.h"
+#include "rocksdb/options.h"
 #include "table/internal_iterator.h"
 #include "util/arena.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
 #include "util/perf_context_imp.h"
+#include "util/string_util.h"

 namespace rocksdb {

@@ -59,9 +61,47 @@ class DBIter: public Iterator {
    kReverse
  };

+  // LocalStatistics contain Statistics counters that will be aggregated per
+  // each iterator instance and then will be sent to the global statistics when
+  // the iterator is destroyed.
+  //
+  // The purpose of this approach is to avoid perf regression happening
+  // when multiple threads bump the atomic counters from a DBIter::Next().
+  struct LocalStatistics {
+    explicit LocalStatistics() { ResetCounters(); }
+
+    void ResetCounters() {
+      next_count_ = 0;
+      next_found_count_ = 0;
+      prev_count_ = 0;
+      prev_found_count_ = 0;
+      bytes_read_ = 0;
+    }
+
+    void BumpGlobalStatistics(Statistics* global_statistics) {
+      RecordTick(global_statistics, NUMBER_DB_NEXT, next_count_);
+      RecordTick(global_statistics, NUMBER_DB_NEXT_FOUND, next_found_count_);
+      RecordTick(global_statistics, NUMBER_DB_PREV, prev_count_);
+      RecordTick(global_statistics, NUMBER_DB_PREV_FOUND, prev_found_count_);
+      RecordTick(global_statistics, ITER_BYTES_READ, bytes_read_);
+      ResetCounters();
+    }
+
+    // Map to Tickers::NUMBER_DB_NEXT
+    uint64_t next_count_;
+    // Map to Tickers::NUMBER_DB_NEXT_FOUND
+    uint64_t next_found_count_;
+    // Map to Tickers::NUMBER_DB_PREV
+    uint64_t prev_count_;
+    // Map to Tickers::NUMBER_DB_PREV_FOUND
+    uint64_t prev_found_count_;
+    // Map to Tickers::ITER_BYTES_READ
+    uint64_t bytes_read_;
+  };
+
  DBIter(Env* env, const ImmutableCFOptions& ioptions, const Comparator* cmp,
         InternalIterator* iter, SequenceNumber s, bool arena_mode,
-         uint64_t max_sequential_skip_in_iterations,
+         uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
         const Slice* iterate_upper_bound = nullptr,
         bool prefix_same_as_start = false)
      : arena_mode_(arena_mode),
@@ -75,6 +115,7 @@ class DBIter: public Iterator {
        valid_(false),
        current_entry_is_merged_(false),
        statistics_(ioptions.statistics),
+        version_number_(version_number),
        iterate_upper_bound_(iterate_upper_bound),
        prefix_same_as_start_(prefix_same_as_start),
        iter_pinned_(false) {
@@ -84,6 +125,7 @@ class DBIter: public Iterator {
  }
  virtual ~DBIter() {
    RecordTick(statistics_, NO_ITERATORS, -1);
+    local_stats_.BumpGlobalStatistics(statistics_);
    if (!arena_mode_) {
      delete iter_;
    } else {
@@ -136,9 +178,27 @@ class DBIter: public Iterator {
    }
    return s;
  }
-  virtual bool IsKeyPinned() const override {
-    assert(valid_);
-    return iter_pinned_ && saved_key_.IsKeyPinned();
+
+  virtual Status GetProperty(std::string prop_name,
+                             std::string* prop) override {
+    if (prop == nullptr) {
+      return Status::InvalidArgument("prop is nullptr");
+    }
+    if (prop_name == "rocksdb.iterator.super-version-number") {
+      // First try to pass the value returned from inner iterator.
+      if (!iter_->GetProperty(prop_name, prop).ok()) {
+        *prop = ToString(version_number_);
+      }
+      return Status::OK();
+    } else if (prop_name == "rocksdb.iterator.is-key-pinned") {
+      if (valid_) {
+        *prop = (iter_pinned_ && saved_key_.IsKeyPinned()) ? "1" : "0";
+      } else {
+        *prop = "Iterator is not valid.";
+      }
+      return Status::OK();
+    }
+    return Status::InvalidArgument("Undentified property.");
  }

  virtual void Next() override;
@@ -186,12 +246,14 @@ class DBIter: public Iterator {
  bool current_entry_is_merged_;
  Statistics* statistics_;
  uint64_t max_skip_;
+  uint64_t version_number_;
  const Slice* iterate_upper_bound_;
  IterKey prefix_start_;
  bool prefix_same_as_start_;
  bool iter_pinned_;
  // List of operands for merge operator.
-  std::deque<std::string> merge_operands_;
+  MergeContext merge_context_;
+  LocalStatistics local_stats_;

  // No copying allowed
  DBIter(const DBIter&);
@@ -229,6 +291,9 @@ void DBIter::Next() {
    PERF_COUNTER_ADD(internal_key_skipped_count, 1);
  }

+  if (statistics_ != nullptr) {
+    local_stats_.next_count_++;
+  }
  // Now we point to the next internal position, for both of merge and
  // not merge cases.
  if (!iter_->Valid()) {
@@ -236,18 +301,15 @@ void DBIter::Next() {
    return;
  }
  FindNextUserEntry(true /* skipping the current user key */);
-  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_NEXT);
-    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_NEXT_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
-    }
-  }
  if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
      prefix_extractor_->Transform(saved_key_.GetKey())
              .compare(prefix_start_.GetKey()) != 0) {
    valid_ = false;
  }
+  if (statistics_ != nullptr && valid_) {
+    local_stats_.next_found_count_++;
+    local_stats_.bytes_read_ += (key().size() + value().size());
+  }
 }

 // PRE: saved_key_ has the current user key if skipping
@@ -275,7 +337,7 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {

    if (ParseKey(&ikey)) {
      if (iterate_upper_bound_ != nullptr &&
-          ikey.user_key.compare(*iterate_upper_bound_) >= 0) {
+          user_comparator_->Compare(ikey.user_key, *iterate_upper_bound_) >= 0) {
        break;
      }

@@ -350,9 +412,9 @@ void DBIter::MergeValuesNewToOld() {
    return;
  }

+  merge_context_.Clear();
  // Start the merge process by pushing the first operand
-  std::deque<std::string> operands;
-  operands.push_front(iter_->value().ToString());
+  merge_context_.PushOperand(iter_->value());

  ParsedInternalKey ikey;
  for (iter_->Next(); iter_->Valid(); iter_->Next()) {
@@ -377,7 +439,8 @@ void DBIter::MergeValuesNewToOld() {
      {
        StopWatchNano timer(env_, statistics_ != nullptr);
        PERF_TIMER_GUARD(merge_operator_time_nanos);
-        user_merge_operator_->FullMerge(ikey.user_key, &val, operands,
+        user_merge_operator_->FullMerge(ikey.user_key, &val,
+                                        merge_context_.GetOperands(),
                                        &saved_value_, logger_);
        RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                   timer.ElapsedNanos());
@@ -389,7 +452,7 @@ void DBIter::MergeValuesNewToOld() {
      // hit a merge, add the value as an operand and run associative merge.
      // when complete, add result to operands and continue.
      const Slice& val = iter_->value();
-      operands.push_front(val.ToString());
+      merge_context_.PushOperand(val);
    } else {
      assert(false);
    }
@@ -402,8 +465,9 @@ void DBIter::MergeValuesNewToOld() {
    // a deletion marker.
    // feed null as the existing value to the merge operator, such that
    // client can differentiate this scenario and do things accordingly.
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
-                                    &saved_value_, logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
+                                    merge_context_.GetOperands(), &saved_value_,
+                                    logger_);
    RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
  }
 }
@@ -415,10 +479,10 @@ void DBIter::Prev() {
  }
  PrevInternal();
  if (statistics_ != nullptr) {
-    RecordTick(statistics_, NUMBER_DB_PREV);
+    local_stats_.prev_count_++;
    if (valid_) {
-      RecordTick(statistics_, NUMBER_DB_PREV_FOUND);
-      RecordTick(statistics_, ITER_BYTES_READ, key().size() + value().size());
+      local_stats_.prev_found_count_++;
+      local_stats_.bytes_read_ += (key().size() + value().size());
    }
  }
  if (valid_ && prefix_extractor_ && prefix_same_as_start_ &&
@@ -495,7 +559,7 @@ void DBIter::PrevInternal() {
 // saved_value_
 bool DBIter::FindValueForCurrentKey() {
  assert(iter_->Valid());
-  merge_operands_.clear();
+  merge_context_.Clear();
  // last entry before merge (could be kTypeDeletion, kTypeSingleDeletion or
  // kTypeValue)
  ValueType last_not_merge_type = kTypeDeletion;
@@ -515,19 +579,19 @@ bool DBIter::FindValueForCurrentKey() {
    last_key_entry_type = ikey.type;
    switch (last_key_entry_type) {
      case kTypeValue:
-        merge_operands_.clear();
+        merge_context_.Clear();
        saved_value_ = iter_->value().ToString();
        last_not_merge_type = kTypeValue;
        break;
      case kTypeDeletion:
      case kTypeSingleDeletion:
-        merge_operands_.clear();
+        merge_context_.Clear();
        last_not_merge_type = last_key_entry_type;
        PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
        break;
      case kTypeMerge:
        assert(user_merge_operator_ != nullptr);
-        merge_operands_.push_back(iter_->value().ToString());
+        merge_context_.PushOperandBack(iter_->value());
        break;
      default:
        assert(false);
@@ -550,8 +614,8 @@ bool DBIter::FindValueForCurrentKey() {
        StopWatchNano timer(env_, statistics_ != nullptr);
        PERF_TIMER_GUARD(merge_operator_time_nanos);
        user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
-                                        merge_operands_, &saved_value_,
-                                        logger_);
+                                        merge_context_.GetOperands(),
+                                        &saved_value_, logger_);
        RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                   timer.ElapsedNanos());
      } else {
@@ -562,8 +626,8 @@ bool DBIter::FindValueForCurrentKey() {
          StopWatchNano timer(env_, statistics_ != nullptr);
          PERF_TIMER_GUARD(merge_operator_time_nanos);
          user_merge_operator_->FullMerge(saved_key_.GetKey(), &temp_slice,
-                                          merge_operands_, &saved_value_,
-                                          logger_);
+                                          merge_context_.GetOperands(),
+                                          &saved_value_, logger_);
          RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME,
                     timer.ElapsedNanos());
        }
@@ -606,11 +670,11 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {

  // kTypeMerge. We need to collect all kTypeMerge values and save them
  // in operands
-  std::deque<std::string> operands;
+  merge_context_.Clear();
  while (iter_->Valid() &&
         user_comparator_->Equal(ikey.user_key, saved_key_.GetKey()) &&
         ikey.type == kTypeMerge) {
-    operands.push_front(iter_->value().ToString());
+    merge_context_.PushOperand(iter_->value());
    iter_->Next();
    FindParseableKey(&ikey, kForward);
  }
@@ -621,7 +685,8 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
    {
      StopWatchNano timer(env_, statistics_ != nullptr);
      PERF_TIMER_GUARD(merge_operator_time_nanos);
-      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr, operands,
+      user_merge_operator_->FullMerge(saved_key_.GetKey(), nullptr,
+                                      merge_context_.GetOperands(),
                                      &saved_value_, logger_);
      RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
    }
@@ -639,8 +704,9 @@ bool DBIter::FindValueForCurrentKeyUsingSeek() {
  {
    StopWatchNano timer(env_, statistics_ != nullptr);
    PERF_TIMER_GUARD(merge_operator_time_nanos);
-    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val, operands,
-                                    &saved_value_, logger_);
+    user_merge_operator_->FullMerge(saved_key_.GetKey(), &val,
+                                    merge_context_.GetOperands(), &saved_value_,
+                                    logger_);
    RecordTick(statistics_, MERGE_OPERATION_TOTAL_TIME, timer.ElapsedNanos());
  }
  valid_ = true;
@@ -818,12 +884,13 @@ Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& ioptions,
                        InternalIterator* internal_iter,
                        const SequenceNumber& sequence,
                        uint64_t max_sequential_skip_in_iterations,
+                        uint64_t version_number,
                        const Slice* iterate_upper_bound,
                        bool prefix_same_as_start, bool pin_data) {
  DBIter* db_iter =
      new DBIter(env, ioptions, user_key_comparator, internal_iter, sequence,
-                 false, max_sequential_skip_in_iterations, iterate_upper_bound,
-                 prefix_same_as_start);
+                 false, max_sequential_skip_in_iterations, version_number,
+                 iterate_upper_bound, prefix_same_as_start);
  if (pin_data) {
    db_iter->PinData();
  }
@@ -850,12 +917,13 @@ inline Slice ArenaWrappedDBIter::key() const { return db_iter_->key(); }
 inline Slice ArenaWrappedDBIter::value() const { return db_iter_->value(); }
 inline Status ArenaWrappedDBIter::status() const { return db_iter_->status(); }
 inline Status ArenaWrappedDBIter::PinData() { return db_iter_->PinData(); }
+inline Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
+                                              std::string* prop) {
+  return db_iter_->GetProperty(prop_name, prop);
+}
 inline Status ArenaWrappedDBIter::ReleasePinnedData() {
  return db_iter_->ReleasePinnedData();
 }
-inline bool ArenaWrappedDBIter::IsKeyPinned() const {
-  return db_iter_->IsKeyPinned();
-}
 void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
                                         void* arg2) {
  db_iter_->RegisterCleanup(function, arg1, arg2);
@@ -864,7 +932,7 @@ void ArenaWrappedDBIter::RegisterCleanup(CleanupFunction function, void* arg1,
 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ImmutableCFOptions& ioptions,
    const Comparator* user_key_comparator, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations,
+    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
    const Slice* iterate_upper_bound, bool prefix_same_as_start,
    bool pin_data) {
  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
@@ -872,7 +940,7 @@ ArenaWrappedDBIter* NewArenaWrappedDbIterator(
  auto mem = arena->AllocateAligned(sizeof(DBIter));
  DBIter* db_iter =
      new (mem) DBIter(env, ioptions, user_key_comparator, nullptr, sequence,
-                       true, max_sequential_skip_in_iterations,
+                       true, max_sequential_skip_in_iterations, version_number,
                       iterate_upper_bound, prefix_same_as_start);

  iter->SetDBIter(db_iter);
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -9,6 +9,7 @@

 #pragma once
 #include <stdint.h>
+#include <string>
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "db/dbformat.h"
@@ -24,14 +25,12 @@ class InternalIterator;
 // Return a new iterator that converts internal keys (yielded by
 // "*internal_iter") that were live at the specified "sequence" number
 // into appropriate user keys.
-extern Iterator* NewDBIterator(Env* env, const ImmutableCFOptions& options,
-                               const Comparator* user_key_comparator,
-                               InternalIterator* internal_iter,
-                               const SequenceNumber& sequence,
-                               uint64_t max_sequential_skip_in_iterations,
-                               const Slice* iterate_upper_bound = nullptr,
-                               bool prefix_same_as_start = false,
-                               bool pin_data = false);
+extern Iterator* NewDBIterator(
+    Env* env, const ImmutableCFOptions& options,
+    const Comparator* user_key_comparator, InternalIterator* internal_iter,
+    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
+    uint64_t version_number, const Slice* iterate_upper_bound = nullptr,
+    bool prefix_same_as_start = false, bool pin_data = false);

 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed be allocated. This class is used as an entry point of
@@ -66,7 +65,7 @@ class ArenaWrappedDBIter : public Iterator {
  void RegisterCleanup(CleanupFunction function, void* arg1, void* arg2);
  virtual Status PinData();
  virtual Status ReleasePinnedData();
-  virtual bool IsKeyPinned() const override;
+  virtual Status GetProperty(std::string prop_name, std::string* prop) override;

 private:
  DBIter* db_iter_;
@@ -77,7 +76,7 @@ class ArenaWrappedDBIter : public Iterator {
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
    Env* env, const ImmutableCFOptions& options,
    const Comparator* user_key_comparator, const SequenceNumber& sequence,
-    uint64_t max_sequential_skip_in_iterations,
+    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
    const Slice* iterate_upper_bound = nullptr,
    bool prefix_same_as_start = false, bool pin_data = false);

@@ -1,4 +1,4 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -181,10 +181,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    internal_iter->AddPut("b", "val_b");
    internal_iter->Finish();

-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 10,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -215,7 +214,7 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations));
+        10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -241,7 +240,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -273,7 +273,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -308,7 +309,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -337,7 +339,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        7, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -374,7 +377,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        4, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -399,7 +403,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
@@ -421,7 +426,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        10, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -456,7 +462,8 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations, ro.iterate_upper_bound));
+        7, options.max_sequential_skip_in_iterations, 0,
+        ro.iterate_upper_bound));

    SetPerfLevel(kEnableCount);
    ASSERT_TRUE(GetPerfLevel() == kEnableCount);
@@ -482,10 +489,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    internal_iter->AddPut("b", "val_b");
    internal_iter->Finish();

-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 10,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToFirst();
    ASSERT_TRUE(db_iter->Valid());
@@ -524,10 +530,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    internal_iter->AddPut("b", "val_b");
    internal_iter->Finish();

-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 2,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -555,10 +560,9 @@ TEST_F(DBIteratorTest, DBIteratorPrevNext) {
    internal_iter->AddPut("c", "val_c");
    internal_iter->Finish();

-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 10,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        10, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -583,10 +587,9 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
    TestIterator* internal_iter = new TestIterator(BytewiseComparator());
    internal_iter->Finish();

-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 0,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -595,10 +598,9 @@ TEST_F(DBIteratorTest, DBIteratorEmpty) {
    TestIterator* internal_iter = new TestIterator(BytewiseComparator());
    internal_iter->Finish();

-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 0,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToFirst();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -617,10 +619,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkipCountSkips) {
  }
  internal_iter->Finish();

-  std::unique_ptr<Iterator> db_iter(
-      NewDBIterator(env_, ImmutableCFOptions(options),
-                    BytewiseComparator(), internal_iter, 2,
-                    options.max_sequential_skip_in_iterations));
+  std::unique_ptr<Iterator> db_iter(NewDBIterator(
+      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2,
+      options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -659,9 +660,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {

      options.statistics = rocksdb::CreateDBStatistics();
      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options),
-          BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          env_, ImmutableCFOptions(options), BytewiseComparator(),
+          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -695,9 +695,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options),
-          BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          env_, ImmutableCFOptions(options), BytewiseComparator(),
+          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -724,9 +723,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options),
-          BytewiseComparator(), internal_iter, 202,
-          options.max_sequential_skip_in_iterations));
+          env_, ImmutableCFOptions(options), BytewiseComparator(),
+          internal_iter, 202, options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -756,10 +754,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      }
      internal_iter->AddPut("c", "200");
      internal_iter->Finish();
-      std::unique_ptr<Iterator> db_iter(
-          NewDBIterator(env_, ImmutableCFOptions(options),
-                        BytewiseComparator(), internal_iter, i,
-                        options.max_sequential_skip_in_iterations));
+      std::unique_ptr<Iterator> db_iter(NewDBIterator(
+          env_, ImmutableCFOptions(options), BytewiseComparator(),
+          internal_iter, i, options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(!db_iter->Valid());

@@ -773,10 +770,9 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
    }
    internal_iter->AddPut("c", "200");
    internal_iter->Finish();
-    std::unique_ptr<Iterator> db_iter(
-        NewDBIterator(env_, ImmutableCFOptions(options),
-                      BytewiseComparator(), internal_iter, 200,
-                      options.max_sequential_skip_in_iterations));
+    std::unique_ptr<Iterator> db_iter(NewDBIterator(
+        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
+        200, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -809,9 +805,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options),
-          BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          env_, ImmutableCFOptions(options), BytewiseComparator(),
+          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -844,9 +839,8 @@ TEST_F(DBIteratorTest, DBIteratorUseSkip) {
      internal_iter->Finish();

      std::unique_ptr<Iterator> db_iter(NewDBIterator(
-          env_, ImmutableCFOptions(options),
-          BytewiseComparator(), internal_iter, i + 2,
-          options.max_sequential_skip_in_iterations));
+          env_, ImmutableCFOptions(options), BytewiseComparator(),
+          internal_iter, i + 2, options.max_sequential_skip_in_iterations, 0));
      db_iter->SeekToLast();
      ASSERT_TRUE(db_iter->Valid());

@@ -887,7 +881,7 @@ TEST_F(DBIteratorTest, DBIterator1) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations));
+      options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -913,7 +907,7 @@ TEST_F(DBIteratorTest, DBIterator2) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 0,
-      options.max_sequential_skip_in_iterations));
+      options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -936,7 +930,7 @@ TEST_F(DBIteratorTest, DBIterator3) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 2,
-      options.max_sequential_skip_in_iterations));
+      options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -958,7 +952,7 @@ TEST_F(DBIteratorTest, DBIterator4) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 4,
-      options.max_sequential_skip_in_iterations));
+      options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -987,7 +981,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations));
+        0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1009,7 +1003,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        1, options.max_sequential_skip_in_iterations));
+        1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1031,7 +1025,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations));
+        2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1053,7 +1047,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        3, options.max_sequential_skip_in_iterations));
+        3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1075,7 +1069,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations));
+        4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1097,7 +1091,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations));
+        5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1119,7 +1113,7 @@ TEST_F(DBIteratorTest, DBIterator5) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations));
+        6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1145,7 +1139,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations));
+        0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1167,7 +1161,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        1, options.max_sequential_skip_in_iterations));
+        1, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1189,7 +1183,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations));
+        2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1211,7 +1205,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        3, options.max_sequential_skip_in_iterations));
+        3, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(!db_iter->Valid());
  }
@@ -1229,7 +1223,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations));
+        4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1251,7 +1245,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations));
+        5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1273,7 +1267,7 @@ TEST_F(DBIteratorTest, DBIterator6) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations));
+        6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1311,7 +1305,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        0, options.max_sequential_skip_in_iterations));
+        0, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
    ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1345,7 +1339,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        2, options.max_sequential_skip_in_iterations));
+        2, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1385,7 +1379,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        4, options.max_sequential_skip_in_iterations));
+        4, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1425,7 +1419,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        5, options.max_sequential_skip_in_iterations));
+        5, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1470,7 +1464,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        6, options.max_sequential_skip_in_iterations));
+        6, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1516,7 +1510,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        7, options.max_sequential_skip_in_iterations));
+        7, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1556,7 +1550,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        9, options.max_sequential_skip_in_iterations));
+        9, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1602,7 +1596,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        13, options.max_sequential_skip_in_iterations));
+        13, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1649,7 +1643,7 @@ TEST_F(DBIteratorTest, DBIterator7) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        14, options.max_sequential_skip_in_iterations));
+        14, options.max_sequential_skip_in_iterations, 0));
    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());

@@ -1678,7 +1672,7 @@ TEST_F(DBIteratorTest, DBIterator8) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, options.max_sequential_skip_in_iterations));
+      10, options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "b");
@@ -1707,7 +1701,7 @@ TEST_F(DBIteratorTest, DBIterator9) {

    std::unique_ptr<Iterator> db_iter(NewDBIterator(
        env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-        10, options.max_sequential_skip_in_iterations));
+        10, options.max_sequential_skip_in_iterations, 0));

    db_iter->SeekToLast();
    ASSERT_TRUE(db_iter->Valid());
@@ -1754,7 +1748,7 @@ TEST_F(DBIteratorTest, DBIterator10) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, options.max_sequential_skip_in_iterations));
+      10, options.max_sequential_skip_in_iterations, 0));

  db_iter->Seek("c");
  ASSERT_TRUE(db_iter->Valid());
@@ -1778,9 +1772,9 @@ TEST_F(DBIteratorTest, SeekToLastOccurrenceSeq0) {
  internal_iter->AddPut("b", "2");
  internal_iter->Finish();

-  std::unique_ptr<Iterator> db_iter(NewDBIterator(
-      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter,
-      10, 0 /* force seek */));
+  std::unique_ptr<Iterator> db_iter(
+      NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
+                    internal_iter, 10, 0 /* force seek */, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1807,7 +1801,7 @@ TEST_F(DBIteratorTest, DBIterator11) {

  std::unique_ptr<Iterator> db_iter(NewDBIterator(
      env_, ImmutableCFOptions(options), BytewiseComparator(), internal_iter, 1,
-      options.max_sequential_skip_in_iterations));
+      options.max_sequential_skip_in_iterations, 0));
  db_iter->SeekToFirst();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "a");
@@ -1832,7 +1826,7 @@ TEST_F(DBIteratorTest, DBIterator12) {

  std::unique_ptr<Iterator> db_iter(
      NewDBIterator(env_, ImmutableCFOptions(options), BytewiseComparator(),
-                    internal_iter, 10, 0));
+                    internal_iter, 10, 0, 0));
  db_iter->SeekToLast();
  ASSERT_TRUE(db_iter->Valid());
  ASSERT_EQ(db_iter->key().ToString(), "c");
@@ -1874,7 +1868,7 @@ class DBIterWithMergeIterTest : public testing::Test {
    db_iter_.reset(NewDBIterator(env_, ImmutableCFOptions(options_),
                                 BytewiseComparator(), merge_iter,
                                 8 /* read data earlier than seqId 8 */,
-                                 3 /* max iterators before reseek */));
+                                 3 /* max iterators before reseek */, 0));
  }

  Env* env_;
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -215,6 +215,34 @@ TEST_F(DBTablePropertiesTest, GetPropertiesOfTablesInRange) {
    TestGetPropertiesOfTablesInRange(std::move(ranges));
  }
 }
+
+TEST_F(DBTablePropertiesTest, GetColumnFamilyNameProperty) {
+  std::string kExtraCfName = "pikachu";
+  CreateAndReopenWithCF({kExtraCfName}, CurrentOptions());
+
+  // Create one table per CF, then verify it was created with the column family
+  // name property.
+  for (int cf = 0; cf < 2; ++cf) {
+    Put(cf, "key", "val");
+    Flush(cf);
+
+    TablePropertiesCollection fname_to_props;
+    ASSERT_OK(db_->GetPropertiesOfAllTables(handles_[cf], &fname_to_props));
+    ASSERT_EQ(1U, fname_to_props.size());
+
+    std::string expected_cf_name;
+    if (cf > 0) {
+      expected_cf_name = kExtraCfName;
+    } else {
+      expected_cf_name = kDefaultColumnFamilyName;
+    }
+    ASSERT_EQ(expected_cf_name,
+              fname_to_props.begin()->second->column_family_name);
+    ASSERT_EQ(cf, static_cast<uint32_t>(
+                      fname_to_props.begin()->second->column_family_id));
+  }
+}
+
 }  // namespace rocksdb

 #endif  // ROCKSDB_LITE
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -654,6 +654,51 @@ TEST_F(DBTestTailingIterator, ManagedTailingIteratorSeekToSame) {
  ASSERT_EQ(found, iter->key().ToString());
 }

+TEST_F(DBTestTailingIterator, ForwardIteratorVersionProperty) {
+  Options options = CurrentOptions();
+  options.write_buffer_size = 1000;
+
+  ReadOptions read_options;
+  read_options.tailing = true;
+
+  Put("foo", "bar");
+
+  uint64_t v1, v2, v3, v4;
+  {
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
+    iter->Seek("foo");
+    std::string prop_value;
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
+                                &prop_value));
+    v1 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+
+    Put("foo1", "bar1");
+    Flush();
+
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
+                                &prop_value));
+    v2 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+
+    iter->Seek("f");
+
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
+                                &prop_value));
+    v3 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+
+    ASSERT_EQ(v1, v2);
+    ASSERT_GT(v3, v2);
+  }
+
+  {
+    std::unique_ptr<Iterator> iter(db_->NewIterator(read_options));
+    iter->Seek("foo");
+    std::string prop_value;
+    ASSERT_OK(iter->GetProperty("rocksdb.iterator.super-version-number",
+                                &prop_value));
+    v4 = static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+  }
+  ASSERT_EQ(v3, v4);
+}
 }  // namespace rocksdb

 #endif  // !defined(ROCKSDB_LITE)
@@ -0,0 +1,864 @@
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+#include <cstdlib>
+#include "db/db_test_util.h"
+#include "port/stack_trace.h"
+#include "rocksdb/wal_filter.h"
+
+namespace rocksdb {
+
+static uint64_t TestGetTickerCount(const Options& options,
+                                   Tickers ticker_type) {
+  return options.statistics->getTickerCount(ticker_type);
+}
+
+class DBTest2 : public DBTestBase {
+ public:
+  DBTest2() : DBTestBase("/db_test2") {}
+};
+
+class PrefixFullBloomWithReverseComparator
+    : public DBTestBase,
+      public ::testing::WithParamInterface<bool> {
+ public:
+  PrefixFullBloomWithReverseComparator()
+      : DBTestBase("/prefix_bloom_reverse") {}
+  virtual void SetUp() override { if_cache_filter_ = GetParam(); }
+  bool if_cache_filter_;
+};
+
+TEST_P(PrefixFullBloomWithReverseComparator,
+       PrefixFullBloomWithReverseComparator) {
+  Options options = last_options_;
+  options.comparator = ReverseBytewiseComparator();
+  options.prefix_extractor.reset(NewCappedPrefixTransform(3));
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions bbto;
+  if (if_cache_filter_) {
+    bbto.no_block_cache = false;
+    bbto.cache_index_and_filter_blocks = true;
+    bbto.block_cache = NewLRUCache(1);
+  }
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = false;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  DestroyAndReopen(options);
+
+  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar123", "foo"));
+  ASSERT_OK(dbfull()->Put(WriteOptions(), "bar234", "foo2"));
+  ASSERT_OK(dbfull()->Put(WriteOptions(), "foo123", "foo3"));
+
+  dbfull()->Flush(FlushOptions());
+
+  if (bbto.block_cache) {
+    bbto.block_cache->EraseUnRefEntries();
+  }
+
+  unique_ptr<Iterator> iter(db_->NewIterator(ReadOptions()));
+  iter->Seek("bar345");
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("bar234", iter->key().ToString());
+  ASSERT_EQ("foo2", iter->value().ToString());
+  iter->Next();
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("bar123", iter->key().ToString());
+  ASSERT_EQ("foo", iter->value().ToString());
+
+  iter->Seek("foo234");
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(iter->Valid());
+  ASSERT_EQ("foo123", iter->key().ToString());
+  ASSERT_EQ("foo3", iter->value().ToString());
+
+  iter->Seek("bar");
+  ASSERT_OK(iter->status());
+  ASSERT_TRUE(!iter->Valid());
+}
+
+INSTANTIATE_TEST_CASE_P(PrefixFullBloomWithReverseComparator,
+                        PrefixFullBloomWithReverseComparator, testing::Bool());
+
+TEST_F(DBTest2, IteratorPropertyVersionNumber) {
+  Put("", "");
+  Iterator* iter1 = db_->NewIterator(ReadOptions());
+  std::string prop_value;
+  ASSERT_OK(
+      iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
+  uint64_t version_number1 =
+      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+
+  Put("", "");
+  Flush();
+
+  Iterator* iter2 = db_->NewIterator(ReadOptions());
+  ASSERT_OK(
+      iter2->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
+  uint64_t version_number2 =
+      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+
+  ASSERT_GT(version_number2, version_number1);
+
+  Put("", "");
+
+  Iterator* iter3 = db_->NewIterator(ReadOptions());
+  ASSERT_OK(
+      iter3->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
+  uint64_t version_number3 =
+      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+
+  ASSERT_EQ(version_number2, version_number3);
+
+  iter1->SeekToFirst();
+  ASSERT_OK(
+      iter1->GetProperty("rocksdb.iterator.super-version-number", &prop_value));
+  uint64_t version_number1_new =
+      static_cast<uint64_t>(std::atoi(prop_value.c_str()));
+  ASSERT_EQ(version_number1, version_number1_new);
+
+  delete iter1;
+  delete iter2;
+  delete iter3;
+}
+
+TEST_F(DBTest2, CacheIndexAndFilterWithDBRestart) {
+  Options options = CurrentOptions();
+  options.create_if_missing = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  Put(1, "a", "begin");
+  Put(1, "z", "end");
+  ASSERT_OK(Flush(1));
+  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+
+  std::string value;
+  value = Get(1, "a");
+}
+
+#ifndef ROCKSDB_LITE
+namespace {
+  void ValidateKeyExistence(DB* db, const std::vector<Slice>& keys_must_exist,
+    const std::vector<Slice>& keys_must_not_exist) {
+    // Ensure that expected keys exist
+    std::vector<std::string> values;
+    if (keys_must_exist.size() > 0) {
+      std::vector<Status> status_list =
+        db->MultiGet(ReadOptions(), keys_must_exist, &values);
+      for (size_t i = 0; i < keys_must_exist.size(); i++) {
+        ASSERT_OK(status_list[i]);
+      }
+    }
+
+    // Ensure that given keys don't exist
+    if (keys_must_not_exist.size() > 0) {
+      std::vector<Status> status_list =
+        db->MultiGet(ReadOptions(), keys_must_not_exist, &values);
+      for (size_t i = 0; i < keys_must_not_exist.size(); i++) {
+        ASSERT_TRUE(status_list[i].IsNotFound());
+      }
+    }
+  }
+
+}  // namespace
+
+TEST_F(DBTest2, WalFilterTest) {
+  class TestWalFilter : public WalFilter {
+  private:
+    // Processing option that is requested to be applied at the given index
+    WalFilter::WalProcessingOption wal_processing_option_;
+    // Index at which to apply wal_processing_option_
+    // At other indexes default wal_processing_option::kContinueProcessing is
+    // returned.
+    size_t apply_option_at_record_index_;
+    // Current record index, incremented with each record encountered.
+    size_t current_record_index_;
+
+  public:
+    TestWalFilter(WalFilter::WalProcessingOption wal_processing_option,
+      size_t apply_option_for_record_index)
+      : wal_processing_option_(wal_processing_option),
+      apply_option_at_record_index_(apply_option_for_record_index),
+      current_record_index_(0) {}
+
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) const override {
+      WalFilter::WalProcessingOption option_to_return;
+
+      if (current_record_index_ == apply_option_at_record_index_) {
+        option_to_return = wal_processing_option_;
+      }
+      else {
+        option_to_return = WalProcessingOption::kContinueProcessing;
+      }
+
+      // Filter is passed as a const object for RocksDB to not modify the
+      // object, however we modify it for our own purpose here and hence
+      // cast the constness away.
+      (const_cast<TestWalFilter*>(this)->current_record_index_)++;
+
+      return option_to_return;
+    }
+
+    virtual const char* Name() const override { return "TestWalFilter"; }
+  };
+
+  // Create 3 batches with two keys each
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  // Test with all WAL processing options
+  for (int option = 0;
+    option < static_cast<int>(
+    WalFilter::WalProcessingOption::kWalProcessingOptionMax);
+  option++) {
+    Options options = OptionsForLogIterTest();
+    DestroyAndReopen(options);
+    CreateAndReopenWithCF({ "pikachu" }, options);
+
+    // Write given keys in given batches
+    for (size_t i = 0; i < batch_keys.size(); i++) {
+      WriteBatch batch;
+      for (size_t j = 0; j < batch_keys[i].size(); j++) {
+        batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+      }
+      dbfull()->Write(WriteOptions(), &batch);
+    }
+
+    WalFilter::WalProcessingOption wal_processing_option =
+      static_cast<WalFilter::WalProcessingOption>(option);
+
+    // Create a test filter that would apply wal_processing_option at the first
+    // record
+    size_t apply_option_for_record_index = 1;
+    TestWalFilter test_wal_filter(wal_processing_option,
+      apply_option_for_record_index);
+
+    // Reopen database with option to use WAL filter
+    options = OptionsForLogIterTest();
+    options.wal_filter = &test_wal_filter;
+    Status status =
+      TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
+    if (wal_processing_option ==
+      WalFilter::WalProcessingOption::kCorruptedRecord) {
+      assert(!status.ok());
+      // In case of corruption we can turn off paranoid_checks to reopen
+      // databse
+      options.paranoid_checks = false;
+      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+    }
+    else {
+      assert(status.ok());
+    }
+
+    // Compute which keys we expect to be found
+    // and which we expect not to be found after recovery.
+    std::vector<Slice> keys_must_exist;
+    std::vector<Slice> keys_must_not_exist;
+    switch (wal_processing_option) {
+    case WalFilter::WalProcessingOption::kCorruptedRecord:
+    case WalFilter::WalProcessingOption::kContinueProcessing: {
+      fprintf(stderr, "Testing with complete WAL processing\n");
+      // we expect all records to be processed
+      for (size_t i = 0; i < batch_keys.size(); i++) {
+        for (size_t j = 0; j < batch_keys[i].size(); j++) {
+          keys_must_exist.push_back(Slice(batch_keys[i][j]));
+        }
+      }
+      break;
+    }
+    case WalFilter::WalProcessingOption::kIgnoreCurrentRecord: {
+      fprintf(stderr,
+        "Testing with ignoring record %" ROCKSDB_PRIszt " only\n",
+        apply_option_for_record_index);
+      // We expect the record with apply_option_for_record_index to be not
+      // found.
+      for (size_t i = 0; i < batch_keys.size(); i++) {
+        for (size_t j = 0; j < batch_keys[i].size(); j++) {
+          if (i == apply_option_for_record_index) {
+            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+          }
+          else {
+            keys_must_exist.push_back(Slice(batch_keys[i][j]));
+          }
+        }
+      }
+      break;
+    }
+    case WalFilter::WalProcessingOption::kStopReplay: {
+      fprintf(stderr,
+        "Testing with stopping replay from record %" ROCKSDB_PRIszt
+        "\n",
+        apply_option_for_record_index);
+      // We expect records beyond apply_option_for_record_index to be not
+      // found.
+      for (size_t i = 0; i < batch_keys.size(); i++) {
+        for (size_t j = 0; j < batch_keys[i].size(); j++) {
+          if (i >= apply_option_for_record_index) {
+            keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+          }
+          else {
+            keys_must_exist.push_back(Slice(batch_keys[i][j]));
+          }
+        }
+      }
+      break;
+    }
+    default:
+      assert(false);  // unhandled case
+    }
+
+    bool checked_after_reopen = false;
+
+    while (true) {
+      // Ensure that expected keys exists
+      // and not expected keys don't exist after recovery
+      ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+
+      if (checked_after_reopen) {
+        break;
+      }
+
+      // reopen database again to make sure previous log(s) are not used
+      //(even if they were skipped)
+      // reopn database with option to use WAL filter
+      options = OptionsForLogIterTest();
+      ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+      checked_after_reopen = true;
+    }
+  }
+}
+
+TEST_F(DBTest2, WalFilterTestWithChangeBatch) {
+  class ChangeBatchHandler : public WriteBatch::Handler {
+  private:
+    // Batch to insert keys in
+    WriteBatch* new_write_batch_;
+    // Number of keys to add in the new batch
+    size_t num_keys_to_add_in_new_batch_;
+    // Number of keys added to new batch
+    size_t num_keys_added_;
+
+  public:
+    ChangeBatchHandler(WriteBatch* new_write_batch,
+      size_t num_keys_to_add_in_new_batch)
+      : new_write_batch_(new_write_batch),
+      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
+      num_keys_added_(0) {}
+    virtual void Put(const Slice& key, const Slice& value) override {
+      if (num_keys_added_ < num_keys_to_add_in_new_batch_) {
+        new_write_batch_->Put(key, value);
+        ++num_keys_added_;
+      }
+    }
+  };
+
+  class TestWalFilterWithChangeBatch : public WalFilter {
+  private:
+    // Index at which to start changing records
+    size_t change_records_from_index_;
+    // Number of keys to add in the new batch
+    size_t num_keys_to_add_in_new_batch_;
+    // Current record index, incremented with each record encountered.
+    size_t current_record_index_;
+
+  public:
+    TestWalFilterWithChangeBatch(size_t change_records_from_index,
+      size_t num_keys_to_add_in_new_batch)
+      : change_records_from_index_(change_records_from_index),
+      num_keys_to_add_in_new_batch_(num_keys_to_add_in_new_batch),
+      current_record_index_(0) {}
+
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) const override {
+      if (current_record_index_ >= change_records_from_index_) {
+        ChangeBatchHandler handler(new_batch, num_keys_to_add_in_new_batch_);
+        batch.Iterate(&handler);
+        *batch_changed = true;
+      }
+
+      // Filter is passed as a const object for RocksDB to not modify the
+      // object, however we modify it for our own purpose here and hence
+      // cast the constness away.
+      (const_cast<TestWalFilterWithChangeBatch*>(this)
+        ->current_record_index_)++;
+
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "TestWalFilterWithChangeBatch";
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({ "pikachu" }, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // Create a test filter that would apply wal_processing_option at the first
+  // record
+  size_t change_records_from_index = 1;
+  size_t num_keys_to_add_in_new_batch = 1;
+  TestWalFilterWithChangeBatch test_wal_filter_with_change_batch(
+    change_records_from_index, num_keys_to_add_in_new_batch);
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_with_change_batch;
+  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+  // Ensure that all keys exist before change_records_from_index_
+  // And after that index only single key exists
+  // as our filter adds only single key for each batch
+  std::vector<Slice> keys_must_exist;
+  std::vector<Slice> keys_must_not_exist;
+
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      if (i >= change_records_from_index && j >= 
+        num_keys_to_add_in_new_batch) {
+        keys_must_not_exist.push_back(Slice(batch_keys[i][j]));
+      }
+      else {
+        keys_must_exist.push_back(Slice(batch_keys[i][j]));
+      }
+    }
+  }
+
+  bool checked_after_reopen = false;
+
+  while (true) {
+    // Ensure that expected keys exists
+    // and not expected keys don't exist after recovery
+    ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+
+    if (checked_after_reopen) {
+      break;
+    }
+
+    // reopen database again to make sure previous log(s) are not used
+    //(even if they were skipped)
+    // reopn database with option to use WAL filter
+    options = OptionsForLogIterTest();
+    ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+    checked_after_reopen = true;
+  }
+}
+
+TEST_F(DBTest2, WalFilterTestWithChangeBatchExtraKeys) {
+  class TestWalFilterWithChangeBatchAddExtraKeys : public WalFilter {
+  public:
+    virtual WalProcessingOption LogRecord(const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) const override {
+      *new_batch = batch;
+      new_batch->Put("key_extra", "value_extra");
+      *batch_changed = true;
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "WalFilterTestWithChangeBatchExtraKeys";
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys(3);
+
+  batch_keys[0].push_back("key1");
+  batch_keys[0].push_back("key2");
+  batch_keys[1].push_back("key3");
+  batch_keys[1].push_back("key4");
+  batch_keys[2].push_back("key5");
+  batch_keys[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({ "pikachu" }, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // Create a test filter that would add extra keys
+  TestWalFilterWithChangeBatchAddExtraKeys test_wal_filter_extra_keys;
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_extra_keys;
+  Status status = TryReopenWithColumnFamilies({ "default", "pikachu" }, 
+    options);
+  ASSERT_TRUE(status.IsNotSupported());
+
+  // Reopen without filter, now reopen should succeed - previous
+  // attempt to open must not have altered the db.
+  options = OptionsForLogIterTest();
+  ReopenWithColumnFamilies({ "default", "pikachu" }, options);
+
+  std::vector<Slice> keys_must_exist;
+  std::vector<Slice> keys_must_not_exist;  // empty vector
+
+  for (size_t i = 0; i < batch_keys.size(); i++) {
+    for (size_t j = 0; j < batch_keys[i].size(); j++) {
+      keys_must_exist.push_back(Slice(batch_keys[i][j]));
+    }
+  }
+
+  ValidateKeyExistence(db_, keys_must_exist, keys_must_not_exist);
+}
+
+TEST_F(DBTest2, WalFilterTestWithColumnFamilies) {
+  class TestWalFilterWithColumnFamilies : public WalFilter {
+  private:
+    // column_family_id -> log_number map (provided to WALFilter)
+    std::map<uint32_t, uint64_t> cf_log_number_map_;
+    // column_family_name -> column_family_id map (provided to WALFilter)
+    std::map<std::string, uint32_t> cf_name_id_map_;
+    // column_family_name -> keys_found_in_wal map
+    // We store keys that are applicable to the column_family
+    // during recovery (i.e. aren't already flushed to SST file(s))
+    // for verification against the keys we expect.
+    std::map<uint32_t, std::vector<std::string>> cf_wal_keys_;
+  public:
+    virtual void ColumnFamilyLogNumberMap(
+      const std::map<uint32_t, uint64_t>& cf_lognumber_map,
+      const std::map<std::string, uint32_t>& cf_name_id_map) override {
+      cf_log_number_map_ = cf_lognumber_map;
+      cf_name_id_map_ = cf_name_id_map;
+    }
+
+    virtual WalProcessingOption LogRecordFound(unsigned long long log_number,
+      const std::string& log_file_name,
+      const WriteBatch& batch,
+      WriteBatch* new_batch,
+      bool* batch_changed) override {
+      class LogRecordBatchHandler : public WriteBatch::Handler {
+      private:
+        const std::map<uint32_t, uint64_t> & cf_log_number_map_;
+        std::map<uint32_t, std::vector<std::string>> & cf_wal_keys_;
+        unsigned long long log_number_;
+      public:
+        LogRecordBatchHandler(unsigned long long current_log_number,
+          const std::map<uint32_t, uint64_t> & cf_log_number_map,
+          std::map<uint32_t, std::vector<std::string>> & cf_wal_keys) :
+          cf_log_number_map_(cf_log_number_map),
+          cf_wal_keys_(cf_wal_keys),
+          log_number_(current_log_number){}
+
+        virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+          const Slice& /*value*/) override {
+          auto it = cf_log_number_map_.find(column_family_id);
+          assert(it != cf_log_number_map_.end());
+          unsigned long long log_number_for_cf = it->second;
+          // If the current record is applicable for column_family_id
+          // (i.e. isn't flushed to SST file(s) for column_family_id)
+          // add it to the cf_wal_keys_ map for verification.
+          if (log_number_ >= log_number_for_cf) {
+            cf_wal_keys_[column_family_id].push_back(std::string(key.data(),
+              key.size()));
+          }
+          return Status::OK();
+        }
+      } handler(log_number, cf_log_number_map_, cf_wal_keys_);
+
+      batch.Iterate(&handler);
+
+      return WalProcessingOption::kContinueProcessing;
+    }
+
+    virtual const char* Name() const override {
+      return "WalFilterTestWithColumnFamilies";
+    }
+
+    const std::map<uint32_t, std::vector<std::string>> & 
+    GetColumnFamilyKeys() {
+      return cf_wal_keys_;
+    }
+
+    const std::map<std::string, uint32_t> & GetColumnFamilyNameIdMap() {
+      return cf_name_id_map_;
+    }
+  };
+
+  std::vector<std::vector<std::string>> batch_keys_pre_flush(3);
+
+  batch_keys_pre_flush[0].push_back("key1");
+  batch_keys_pre_flush[0].push_back("key2");
+  batch_keys_pre_flush[1].push_back("key3");
+  batch_keys_pre_flush[1].push_back("key4");
+  batch_keys_pre_flush[2].push_back("key5");
+  batch_keys_pre_flush[2].push_back("key6");
+
+  Options options = OptionsForLogIterTest();
+  DestroyAndReopen(options);
+  CreateAndReopenWithCF({ "pikachu" }, options);
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys_pre_flush[i][j], DummyString(1024));
+      batch.Put(handles_[1], batch_keys_pre_flush[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  //Flush default column-family
+  db_->Flush(FlushOptions(), handles_[0]);
+
+  // Do some more writes
+  std::vector<std::vector<std::string>> batch_keys_post_flush(3);
+
+  batch_keys_post_flush[0].push_back("key7");
+  batch_keys_post_flush[0].push_back("key8");
+  batch_keys_post_flush[1].push_back("key9");
+  batch_keys_post_flush[1].push_back("key10");
+  batch_keys_post_flush[2].push_back("key11");
+  batch_keys_post_flush[2].push_back("key12");
+
+  // Write given keys in given batches
+  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
+    WriteBatch batch;
+    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
+      batch.Put(handles_[0], batch_keys_post_flush[i][j], DummyString(1024));
+      batch.Put(handles_[1], batch_keys_post_flush[i][j], DummyString(1024));
+    }
+    dbfull()->Write(WriteOptions(), &batch);
+  }
+
+  // On Recovery we should only find the second batch applicable to default CF
+  // But both batches applicable to pikachu CF
+
+  // Create a test filter that would add extra keys
+  TestWalFilterWithColumnFamilies test_wal_filter_column_families;
+
+  // Reopen database with option to use WAL filter
+  options = OptionsForLogIterTest();
+  options.wal_filter = &test_wal_filter_column_families;
+  Status status =
+    TryReopenWithColumnFamilies({ "default", "pikachu" }, options);
+  ASSERT_TRUE(status.ok());
+
+  // verify that handles_[0] only has post_flush keys
+  // while handles_[1] has pre and post flush keys
+  auto cf_wal_keys = test_wal_filter_column_families.GetColumnFamilyKeys();
+  auto name_id_map = 
+    test_wal_filter_column_families.GetColumnFamilyNameIdMap();
+  size_t index = 0;
+  auto keys_cf = cf_wal_keys[name_id_map[kDefaultColumnFamilyName]];
+  //default column-family, only post_flush keys are expected
+  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
+    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
+      Slice key_from_the_log(keys_cf[index++]);
+      Slice batch_key(batch_keys_post_flush[i][j]);
+      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
+    }
+  }
+  ASSERT_TRUE(index == keys_cf.size());
+
+  index = 0;
+  keys_cf = cf_wal_keys[name_id_map["pikachu"]];
+  //pikachu column-family, all keys are expected
+  for (size_t i = 0; i < batch_keys_pre_flush.size(); i++) {
+    for (size_t j = 0; j < batch_keys_pre_flush[i].size(); j++) {
+      Slice key_from_the_log(keys_cf[index++]);
+      Slice batch_key(batch_keys_pre_flush[i][j]);
+      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
+    }
+  }
+
+  for (size_t i = 0; i < batch_keys_post_flush.size(); i++) {
+    for (size_t j = 0; j < batch_keys_post_flush[i].size(); j++) {
+      Slice key_from_the_log(keys_cf[index++]);
+      Slice batch_key(batch_keys_post_flush[i][j]);
+      ASSERT_TRUE(key_from_the_log.compare(batch_key) == 0);
+    }
+  }
+  ASSERT_TRUE(index == keys_cf.size());
+}
+#endif  // ROCKSDB_LITE
+
+TEST_F(DBTest2, DISABLED_FirstSnapshotTest) {
+  Options options;
+  options.write_buffer_size = 100000;  // Small write buffer
+  options = CurrentOptions(options);
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  // This snapshot will have sequence number 0.  When compaction encounters
+  // this snapshot, CompactionIterator::findEarliestVisibleSnapshot() will
+  // assert as it expects non-zero snapshots.
+  //
+  // One fix would be to simply remove this assert.  However, a better fix
+  // might
+  // be to always have db sequence numbers start from 1 so that no code is
+  // ever
+  // confused by 0.
+  const Snapshot* s1 = db_->GetSnapshot();
+
+  Put(1, "k1", std::string(100000, 'x'));  // Fill memtable
+  Put(1, "k2", std::string(100000, 'y'));  // Trigger flush
+
+  db_->ReleaseSnapshot(s1);
+}
+
+class PinL0IndexAndFilterBlocksTest : public DBTestBase,
+                                      public testing::WithParamInterface<bool> {
+ public:
+  PinL0IndexAndFilterBlocksTest() : DBTestBase("/db_pin_l0_index_bloom_test") {}
+  virtual void SetUp() override { infinite_max_files_ = GetParam(); }
+
+  bool infinite_max_files_;
+};
+
+TEST_P(PinL0IndexAndFilterBlocksTest,
+       IndexAndFilterBlocksOfNewTableAddedToCacheWithPinning) {
+  Options options = CurrentOptions();
+  if (infinite_max_files_) {
+    options.max_open_files = -1;
+  }
+  options.create_if_missing = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  ASSERT_OK(Put(1, "key", "val"));
+  // Create a new table.
+  ASSERT_OK(Flush(1));
+
+  // index/filter blocks added to block cache right after table creation.
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+
+  // only index/filter were added
+  ASSERT_EQ(2, TestGetTickerCount(options, BLOCK_CACHE_ADD));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_DATA_MISS));
+
+  std::string value;
+  // Miss and hit count should remain the same, they're all pinned.
+  db_->KeyMayExist(ReadOptions(), handles_[1], "key", &value);
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+
+  // Miss and hit count should remain the same, they're all pinned.
+  value = Get(1, "key");
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(0, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+}
+
+TEST_P(PinL0IndexAndFilterBlocksTest,
+       MultiLevelIndexAndFilterBlocksCachedWithPinning) {
+  Options options = CurrentOptions();
+  if (infinite_max_files_) {
+    options.max_open_files = -1;
+  }
+  options.create_if_missing = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  BlockBasedTableOptions table_options;
+  table_options.cache_index_and_filter_blocks = true;
+  table_options.pin_l0_filter_and_index_blocks_in_cache = true;
+  table_options.filter_policy.reset(NewBloomFilterPolicy(20));
+  options.table_factory.reset(new BlockBasedTableFactory(table_options));
+  CreateAndReopenWithCF({"pikachu"}, options);
+
+  Put(1, "a", "begin");
+  Put(1, "z", "end");
+  ASSERT_OK(Flush(1));
+  // move this table to L1
+  dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[1]);
+
+  // reset block cache
+  table_options.block_cache = NewLRUCache(64 * 1024);
+  options.table_factory.reset(NewBlockBasedTableFactory(table_options));
+  TryReopenWithColumnFamilies({"default", "pikachu"}, options);
+  // create new table at L0
+  Put(1, "a2", "begin2");
+  Put(1, "z2", "end2");
+  ASSERT_OK(Flush(1));
+
+  table_options.block_cache->EraseUnRefEntries();
+
+  // get base cache values
+  uint64_t fm = TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS);
+  uint64_t fh = TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT);
+  uint64_t im = TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS);
+  uint64_t ih = TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT);
+
+  std::string value;
+  // this should be read from L0
+  // so cache values don't change
+  value = Get(1, "a2");
+  ASSERT_EQ(fm, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(fh, TestGetTickerCount(options, BLOCK_CACHE_FILTER_HIT));
+  ASSERT_EQ(im, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+  ASSERT_EQ(ih, TestGetTickerCount(options, BLOCK_CACHE_INDEX_HIT));
+
+  // this should be read from L1
+  // the file is opened, prefetching results in a cache filter miss
+  // the block is loaded and added to the cache,
+  // then the get results in a cache hit for L1
+  value = Get(1, "a");
+  ASSERT_EQ(fm + 1, TestGetTickerCount(options, BLOCK_CACHE_FILTER_MISS));
+  ASSERT_EQ(im + 1, TestGetTickerCount(options, BLOCK_CACHE_INDEX_MISS));
+}
+
+INSTANTIATE_TEST_CASE_P(PinL0IndexAndFilterBlocksTest,
+                        PinL0IndexAndFilterBlocksTest, ::testing::Bool());
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::port::InstallStackTraceHandler();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
@@ -1,4 +1,4 @@
-// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -73,63 +73,69 @@ DBTestBase::~DBTestBase() {
  delete env_;
 }

-// Switch to a fresh database with the next option configuration to
-// test.  Return false if there are no more configurations to test.
-bool DBTestBase::ChangeOptions(int skip_mask) {
-  for (option_config_++; option_config_ < kEnd; option_config_++) {
+bool DBTestBase::ShouldSkipOptions(int option_config, int skip_mask) {
 #ifdef ROCKSDB_LITE
    // These options are not supported in ROCKSDB_LITE
-    if (option_config_ == kHashSkipList ||
-        option_config_ == kPlainTableFirstBytePrefix ||
-        option_config_ == kPlainTableCappedPrefix ||
-        option_config_ == kPlainTableCappedPrefixNonMmap ||
-        option_config_ == kPlainTableAllBytesPrefix ||
-        option_config_ == kVectorRep || option_config_ == kHashLinkList ||
-        option_config_ == kHashCuckoo ||
-        option_config_ == kUniversalCompaction ||
-        option_config_ == kUniversalCompactionMultiLevel ||
-        option_config_ == kUniversalSubcompactions ||
-        option_config_ == kFIFOCompaction) {
-      continue;
+  if (option_config == kHashSkipList ||
+      option_config == kPlainTableFirstBytePrefix ||
+      option_config == kPlainTableCappedPrefix ||
+      option_config == kPlainTableCappedPrefixNonMmap ||
+      option_config == kPlainTableAllBytesPrefix ||
+      option_config == kVectorRep || option_config == kHashLinkList ||
+      option_config == kHashCuckoo || option_config == kUniversalCompaction ||
+      option_config == kUniversalCompactionMultiLevel ||
+      option_config == kUniversalSubcompactions ||
+      option_config == kFIFOCompaction ||
+      option_config == kConcurrentSkipList) {
+    return true;
    }
 #endif

    if ((skip_mask & kSkipDeletesFilterFirst) &&
-        option_config_ == kDeletesFilterFirst) {
-      continue;
+        option_config == kDeletesFilterFirst) {
+      return true;
    }
    if ((skip_mask & kSkipUniversalCompaction) &&
-        (option_config_ == kUniversalCompaction ||
-         option_config_ == kUniversalCompactionMultiLevel)) {
-      continue;
+        (option_config == kUniversalCompaction ||
+         option_config == kUniversalCompactionMultiLevel)) {
+      return true;
    }
-    if ((skip_mask & kSkipMergePut) && option_config_ == kMergePut) {
-      continue;
+    if ((skip_mask & kSkipMergePut) && option_config == kMergePut) {
+      return true;
    }
    if ((skip_mask & kSkipNoSeekToLast) &&
-        (option_config_ == kHashLinkList || option_config_ == kHashSkipList)) {
-      continue;
+        (option_config == kHashLinkList || option_config == kHashSkipList)) {
+      return true;
    }
    if ((skip_mask & kSkipPlainTable) &&
-        (option_config_ == kPlainTableAllBytesPrefix ||
-         option_config_ == kPlainTableFirstBytePrefix ||
-         option_config_ == kPlainTableCappedPrefix ||
-         option_config_ == kPlainTableCappedPrefixNonMmap)) {
-      continue;
+        (option_config == kPlainTableAllBytesPrefix ||
+         option_config == kPlainTableFirstBytePrefix ||
+         option_config == kPlainTableCappedPrefix ||
+         option_config == kPlainTableCappedPrefixNonMmap)) {
+      return true;
    }
    if ((skip_mask & kSkipHashIndex) &&
-        (option_config_ == kBlockBasedTableWithPrefixHashIndex ||
-         option_config_ == kBlockBasedTableWithWholeKeyHashIndex)) {
-      continue;
+        (option_config == kBlockBasedTableWithPrefixHashIndex ||
+         option_config == kBlockBasedTableWithWholeKeyHashIndex)) {
+      return true;
    }
-    if ((skip_mask & kSkipHashCuckoo) && (option_config_ == kHashCuckoo)) {
-      continue;
+    if ((skip_mask & kSkipHashCuckoo) && (option_config == kHashCuckoo)) {
+      return true;
    }
-    if ((skip_mask & kSkipFIFOCompaction) &&
-        option_config_ == kFIFOCompaction) {
-      continue;
+    if ((skip_mask & kSkipFIFOCompaction) && option_config == kFIFOCompaction) {
+      return true;
    }
-    if ((skip_mask & kSkipMmapReads) && option_config_ == kWalDirAndMmapReads) {
+    if ((skip_mask & kSkipMmapReads) && option_config == kWalDirAndMmapReads) {
+      return true;
+    }
+    return false;
+}
+
+// Switch to a fresh database with the next option configuration to
+// test.  Return false if there are no more configurations to test.
+bool DBTestBase::ChangeOptions(int skip_mask) {
+  for (option_config_++; option_config_ < kEnd; option_config_++) {
+    if (ShouldSkipOptions(option_config_, skip_mask)) {
      continue;
    }
    break;
@@ -204,6 +210,8 @@ Options DBTestBase::CurrentOptions(
    const anon::OptionsOverride& options_override) {
  Options options;
  options.write_buffer_size = 4090 * 4096;
+  options.target_file_size_base = 2 * 1024 * 1024;
+  options.max_bytes_for_level_base = 10 * 1024 * 1024;
  return CurrentOptions(options, options_override);
 }

@@ -295,6 +303,7 @@ Options DBTestBase::CurrentOptions(
    case kPerfOptions:
      options.soft_rate_limit = 2.0;
      options.delayed_write_rate = 8 * 1024 * 1024;
+      options.report_bg_io_stats = true;
      // TODO(3.13) -- test more options
      break;
    case kDeletesFilterFirst:
@@ -333,6 +342,10 @@ Options DBTestBase::CurrentOptions(
      options.prefix_extractor.reset(NewNoopTransform());
      break;
    }
+    case kBlockBasedTableWithIndexRestartInterval: {
+      table_options.index_block_restart_interval = 8;
+      break;
+    }
    case kOptimizeFiltersForHits: {
      options.optimize_filters_for_hits = true;
      set_block_based_table_factory = true;
@@ -356,6 +369,11 @@ Options DBTestBase::CurrentOptions(
      options.max_subcompactions = 4;
      break;
    }
+    case kConcurrentSkipList: {
+      options.allow_concurrent_memtable_write = true;
+      options.enable_write_thread_adaptive_yield = true;
+      break;
+    }

    default:
      break;
@@ -1000,4 +1018,29 @@ void DBTestBase::CopyFile(const std::string& source,
  ASSERT_OK(destfile->Close());
 }

+std::unordered_map<std::string, uint64_t> DBTestBase::GetAllSSTFiles(
+    uint64_t* total_size) {
+  std::unordered_map<std::string, uint64_t> res;
+
+  if (total_size) {
+    *total_size = 0;
+  }
+  std::vector<std::string> files;
+  env_->GetChildren(dbname_, &files);
+  for (auto& file_name : files) {
+    uint64_t number;
+    FileType type;
+    std::string file_path = dbname_ + "/" + file_name;
+    if (ParseFileName(file_name, &number, &type) && type == kTableFile) {
+      uint64_t file_size = 0;
+      env_->GetFileSize(file_path, &file_size);
+      res[file_path] = file_size;
+      if (total_size) {
+        *total_size += file_size;
+      }
+    }
+  }
+  return res;
+}
+
 }  // namespace rocksdb
@@ -1,4 +1,4 @@
-// Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+// Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
@@ -19,6 +19,7 @@
 #endif

 #include <algorithm>
+#include <map>
 #include <set>
 #include <string>
 #include <thread>
@@ -27,7 +28,6 @@
 #include <vector>

 #include "db/db_impl.h"
-#include "db/db_test_util.h"
 #include "db/dbformat.h"
 #include "db/filename.h"
 #include "memtable/hash_linklist_rep.h"
@@ -525,9 +525,11 @@ class DBTestBase : public testing::Test {
    kOptimizeFiltersForHits = 27,
    kRowCache = 28,
    kRecycleLogFiles = 29,
-    kLevelSubcompactions = 30,
-    kUniversalSubcompactions = 31,
-    kEnd = 30
+    kConcurrentSkipList = 30,
+    kEnd = 31,
+    kLevelSubcompactions = 31,
+    kUniversalSubcompactions = 32,
+    kBlockBasedTableWithIndexRestartInterval = 33,
  };
  int option_config_;

@@ -573,6 +575,8 @@ class DBTestBase : public testing::Test {
    return std::string(buf);
  }

+  static bool ShouldSkipOptions(int option_config, int skip_mask = kNoSkip);
+
  // Switch to a fresh database with the next option configuration to
  // test.  Return false if there are no more configurations to test.
  bool ChangeOptions(int skip_mask = kNoSkip);
@@ -749,6 +753,9 @@ class DBTestBase : public testing::Test {

  void CopyFile(const std::string& source, const std::string& destination,
                uint64_t size = 0);
+
+  std::unordered_map<std::string, uint64_t> GetAllSSTFiles(
+      uint64_t* total_size = nullptr);
 };

 }  // namespace rocksdb
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -14,6 +14,11 @@

 namespace rocksdb {

+static uint64_t TestGetTickerCount(const Options& options,
+                                   Tickers ticker_type) {
+  return options.statistics->getTickerCount(ticker_type);
+}
+
 static std::string CompressibleString(Random* rnd, int len) {
  std::string r;
  test::CompressibleString(rnd, 0.8, len, &r);
@@ -119,8 +124,7 @@ class DelayFilterFactory : public CompactionFilterFactory {
 // Make sure we don't trigger a problem if the trigger conditon is given
 // to be 0, which is invalid.
 TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
-  Options options;
-  options = CurrentOptions(options);
+  Options options = CurrentOptions();

  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
@@ -154,6 +158,71 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSingleSortedRun) {
  }
 }

+TEST_P(DBTestUniversalCompaction, OptimizeFiltersForHits) {
+  Options options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.compaction_options_universal.size_ratio = 5;
+  options.num_levels = num_levels_;
+  options.write_buffer_size = 105 << 10;  // 105KB
+  options.arena_block_size = 4 << 10;
+  options.target_file_size_base = 32 << 10;  // 32KB
+  // trigger compaction if there are >= 4 files
+  options.level0_file_num_compaction_trigger = 4;
+  BlockBasedTableOptions bbto;
+  bbto.cache_index_and_filter_blocks = true;
+  bbto.filter_policy.reset(NewBloomFilterPolicy(10, false));
+  bbto.whole_key_filtering = true;
+  options.table_factory.reset(NewBlockBasedTableFactory(bbto));
+  options.optimize_filters_for_hits = true;
+  options.statistics = rocksdb::CreateDBStatistics();
+  options.memtable_factory.reset(new SpecialSkipListFactory(3));
+
+  DestroyAndReopen(options);
+
+  // block compaction from happening
+  env_->SetBackgroundThreads(1, Env::LOW);
+  test::SleepingBackgroundTask sleeping_task_low;
+  env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task_low,
+                 Env::Priority::LOW);
+
+  for (int num = 0; num < options.level0_file_num_compaction_trigger; num++) {
+    Put(Key(num * 10), "val");
+    if (num) {
+      dbfull()->TEST_WaitForFlushMemTable();
+    }
+    Put(Key(30 + num * 10), "val");
+    Put(Key(60 + num * 10), "val");
+  }
+  Put("", "");
+  dbfull()->TEST_WaitForFlushMemTable();
+
+  // Query set of non existing keys
+  for (int i = 5; i < 90; i += 10) {
+    ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
+  }
+
+  // Make sure bloom filter is used at least once.
+  ASSERT_GT(TestGetTickerCount(options, BLOOM_FILTER_USEFUL), 0);
+  auto prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL);
+
+  // Make sure bloom filter is used for all but the last L0 file when looking
+  // up a non-existent key that's in the range of all L0 files.
+  ASSERT_EQ(Get(Key(35)), "NOT_FOUND");
+  ASSERT_EQ(prev_counter + NumTableFilesAtLevel(0) - 1,
+            TestGetTickerCount(options, BLOOM_FILTER_USEFUL));
+  prev_counter = TestGetTickerCount(options, BLOOM_FILTER_USEFUL);
+
+  // Unblock compaction and wait it for happening.
+  sleeping_task_low.WakeUp();
+  dbfull()->TEST_WaitForCompact();
+
+  // The same queries will not trigger bloom filter
+  for (int i = 5; i < 90; i += 10) {
+    ASSERT_EQ(Get(Key(i)), "NOT_FOUND");
+  }
+  ASSERT_EQ(prev_counter, TestGetTickerCount(options, BLOOM_FILTER_USEFUL));
+}
+
 // TODO(kailiu) The tests on UniversalCompaction has some issues:
 //  1. A lot of magic numbers ("11" or "12").
 //  2. Made assumption on the memtable flush conditions, which may change from
@@ -266,13 +335,12 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrigger) {
 }

 TEST_P(DBTestUniversalCompaction, UniversalCompactionSizeAmplification) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
  options.level0_file_num_compaction_trigger = 3;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -317,12 +385,12 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
  ChangeCompactOptions();
  Options options;
  options.create_if_missing = true;
-  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  options.compaction_style = kCompactionStyleLevel;
  options.num_levels = 1;
  options.target_file_size_base = options.write_buffer_size;
  options.compression = kNoCompression;
  options = CurrentOptions(options);
+  options.write_buffer_size = kEntrySize * kEntriesPerBuffer;
  CreateAndReopenWithCF({"pikachu"}, options);
  ASSERT_EQ(options.compaction_style, kCompactionStyleUniversal);
  Random rnd(301);
@@ -380,12 +448,11 @@ TEST_P(DBTestUniversalCompaction, CompactFilesOnUniversalCompaction) {
 }

 TEST_P(DBTestUniversalCompaction, UniversalCompactionTargetLevel) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.num_levels = 7;
  options.disable_auto_compactions = true;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  // Generate 3 overlapping files
@@ -425,14 +492,13 @@ class DBTestUniversalCompactionMultiLevels
 };

 TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionMultiLevels) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
  options.write_buffer_size = 100 << 10;  // 100KB
  options.level0_file_num_compaction_trigger = 8;
  options.max_background_compactions = 3;
  options.target_file_size_base = 32 * 1024;
-  options = CurrentOptions(options);
  CreateAndReopenWithCF({"pikachu"}, options);

  // Trigger compaction if size amplification exceeds 110%
@@ -468,7 +534,7 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
      });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = 3;
@@ -476,7 +542,6 @@ TEST_P(DBTestUniversalCompactionMultiLevels, UniversalCompactionTrivialMove) {
  options.level0_file_num_compaction_trigger = 3;
  options.max_background_compactions = 2;
  options.target_file_size_base = 32 * 1024;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -515,7 +580,7 @@ class DBTestUniversalCompactionParallel :
 };

 TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = num_levels_;
  options.write_buffer_size = 1 << 10;  // 1KB
@@ -524,7 +589,6 @@ TEST_P(DBTestUniversalCompactionParallel, UniversalCompactionParallel) {
  options.max_background_flushes = 3;
  options.target_file_size_base = 1 * 1024;
  options.compaction_options_universal.max_size_amplification_percent = 110;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -581,7 +645,7 @@ INSTANTIATE_TEST_CASE_P(DBTestUniversalCompactionParallel,
                                           ::testing::Bool()));

 TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 105 << 10;    // 105KB
  options.arena_block_size = 4 << 10;       // 4KB
@@ -589,7 +653,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionOptions) {
  options.level0_file_num_compaction_trigger = 4;
  options.num_levels = num_levels_;
  options.compaction_options_universal.compression_size_percent = -1;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -702,14 +765,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio1) {
    return;
  }

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = num_levels_;
  options.compaction_options_universal.compression_size_percent = 70;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  Random rnd(301);
@@ -770,14 +832,13 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionCompressRatio2) {
  if (!Snappy_Supported()) {
    return;
  }
-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.write_buffer_size = 100 << 10;     // 100KB
  options.target_file_size_base = 32 << 10;  // 32KB
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = num_levels_;
  options.compaction_options_universal.compression_size_percent = 95;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);

  Random rnd(301);
@@ -813,7 +874,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
      });
  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = 2;
@@ -821,7 +882,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest1) {
  options.level0_file_num_compaction_trigger = 3;
  options.max_background_compactions = 1;
  options.target_file_size_base = 32 * 1024;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -858,7 +918,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {

  rocksdb::SyncPoint::GetInstance()->EnableProcessing();

-  Options options;
+  Options options = CurrentOptions();
  options.compaction_style = kCompactionStyleUniversal;
  options.compaction_options_universal.allow_trivial_move = true;
  options.num_levels = 15;
@@ -866,7 +926,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
  options.level0_file_num_compaction_trigger = 8;
  options.max_background_compactions = 4;
  options.target_file_size_base = 64 * 1024;
-  options = CurrentOptions(options);
  DestroyAndReopen(options);
  CreateAndReopenWithCF({"pikachu"}, options);

@@ -892,7 +951,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionTrivialMoveTest2) {
 }

 TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
-  Options options;
+  Options options = CurrentOptions();
  options.db_paths.emplace_back(dbname_, 300 * 1024);
  options.db_paths.emplace_back(dbname_ + "_2", 300 * 1024);
  options.db_paths.emplace_back(dbname_ + "_3", 500 * 1024);
@@ -905,7 +964,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionFourPaths) {
  options.arena_block_size = 4 << 10;
  options.level0_file_num_compaction_trigger = 2;
  options.num_levels = 1;
-  options = CurrentOptions(options);

  std::vector<std::string> filenames;
  env_->GetChildren(options.db_paths[1].path, &filenames);
@@ -1032,16 +1090,11 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
  for (int i = 0; i <= max_key1; i++) {
    // each value is 10K
    ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
+    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  }
  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();

-  int non_level0_num_files = 0;
-  for (int i = 1; i < options.num_levels; i++) {
-    non_level0_num_files += NumTableFilesAtLevel(i, 1);
-  }
-  ASSERT_EQ(non_level0_num_files, 0);
-
  // Stage 2: reopen with universal compaction, num_levels=4
  options.compaction_style = kCompactionStyleUniversal;
  options.num_levels = 4;
@@ -1054,6 +1107,7 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
  for (int i = max_key1 + 1; i <= max_key2; i++) {
    // each value is 10K
    ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
+    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  }
  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();
@@ -1084,6 +1138,7 @@ TEST_P(DBTestUniversalCompaction, IncreaseUniversalCompactionNumLevels) {
  for (int i = max_key2 + 1; i <= max_key3; i++) {
    // each value is 10K
    ASSERT_OK(Put(1, Key(i), RandomString(&rnd, 10000)));
+    dbfull()->TEST_WaitForFlushMemTable(handles_[1]);
  }
  ASSERT_OK(Flush(1));
  dbfull()->TEST_WaitForCompact();
@@ -1095,7 +1150,7 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  if (!Snappy_Supported()) {
    return;
  }
-  Options options;
+  Options options = CurrentOptions();
  options.db_paths.emplace_back(dbname_, 500 * 1024);
  options.db_paths.emplace_back(dbname_ + "_2", 1024 * 1024 * 1024);
  options.compaction_style = kCompactionStyleUniversal;
@@ -1107,7 +1162,6 @@ TEST_P(DBTestUniversalCompaction, UniversalCompactionSecondPathRatio) {
  options.num_levels = 1;
  options.memtable_factory.reset(
      new SpecialSkipListFactory(KNumKeysByGenerateNewFile - 1));
-  options = CurrentOptions(options);

  std::vector<std::string> filenames;
  env_->GetChildren(options.db_paths[1].path, &filenames);
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -465,6 +465,12 @@ class InternalKeySliceTransform : public SliceTransform {
  const SliceTransform* const transform_;
 };

+// Read the key of a record from a write batch.
+// if this record represent the default column family then cf_record
+// must be passed as false, otherwise it must be passed as true.
+extern bool ReadKeyFromWriteBatchEntry(Slice* input, Slice* key,
+                                       bool cf_record);
+
 // Read record from a write batch piece from input.
 // tag, column_family, key, value and blob are return values. Callers own the
 // Slice they point to.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -37,6 +37,7 @@ class DeleteFileTest : public testing::Test {
  DeleteFileTest() {
    db_ = nullptr;
    env_ = Env::Default();
+    options_.delete_obsolete_files_period_micros = 0;  // always do full purge
    options_.enable_thread_tracking = true;
    options_.write_buffer_size = 1024*1024*1000;
    options_.target_file_size_base = 1024*1024*1000;
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2015, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -66,7 +66,7 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
                   JobContext* job_context, LogBuffer* log_buffer,
                   Directory* db_directory, Directory* output_file_directory,
                   CompressionType output_compression, Statistics* stats,
-                   EventLogger* event_logger)
+                   EventLogger* event_logger, bool measure_io_stats)
    : dbname_(dbname),
      cfd_(cfd),
      db_options_(db_options),
@@ -83,7 +83,8 @@ FlushJob::FlushJob(const std::string& dbname, ColumnFamilyData* cfd,
      output_file_directory_(output_file_directory),
      output_compression_(output_compression),
      stats_(stats),
-      event_logger_(event_logger) {
+      event_logger_(event_logger),
+      measure_io_stats_(measure_io_stats) {
  // Update the thread status to indicate flush.
  ReportStartedFlush();
  TEST_SYNC_POINT("FlushJob::FlushJob()");
@@ -94,7 +95,8 @@ FlushJob::~FlushJob() {
 }

 void FlushJob::ReportStartedFlush() {
-  ThreadStatusUtil::SetColumnFamily(cfd_);
+  ThreadStatusUtil::SetColumnFamily(cfd_, cfd_->ioptions()->env,
+                                    cfd_->options()->enable_thread_tracking);
  ThreadStatusUtil::SetThreadOperation(ThreadStatus::OP_FLUSH);
  ThreadStatusUtil::SetThreadOperationProperty(
      ThreadStatus::COMPACTION_JOB_ID,
@@ -120,6 +122,21 @@ void FlushJob::RecordFlushIOStats() {
 Status FlushJob::Run(FileMetaData* file_meta) {
  AutoThreadOperationStageUpdater stage_run(
      ThreadStatus::STAGE_FLUSH_RUN);
+  // I/O measurement variables
+  PerfLevel prev_perf_level = PerfLevel::kEnableTime;
+  uint64_t prev_write_nanos = 0;
+  uint64_t prev_fsync_nanos = 0;
+  uint64_t prev_range_sync_nanos = 0;
+  uint64_t prev_prepare_write_nanos = 0;
+  if (measure_io_stats_) {
+    prev_perf_level = GetPerfLevel();
+    SetPerfLevel(PerfLevel::kEnableTime);
+    prev_write_nanos = IOSTATS(write_nanos);
+    prev_fsync_nanos = IOSTATS(fsync_nanos);
+    prev_range_sync_nanos = IOSTATS(range_sync_nanos);
+    prev_prepare_write_nanos = IOSTATS(prepare_write_nanos);
+  }
+
  // Save the contents of the earliest memtable as a new Table
  FileMetaData meta;
  autovector<MemTable*> mems;
@@ -179,6 +196,18 @@ Status FlushJob::Run(FileMetaData* file_meta) {
  }
  stream.EndArray();

+  if (measure_io_stats_) {
+    if (prev_perf_level != PerfLevel::kEnableTime) {
+      SetPerfLevel(prev_perf_level);
+    }
+    stream << "file_write_nanos" << (IOSTATS(write_nanos) - prev_write_nanos);
+    stream << "file_range_sync_nanos"
+           << (IOSTATS(range_sync_nanos) - prev_range_sync_nanos);
+    stream << "file_fsync_nanos" << (IOSTATS(fsync_nanos) - prev_fsync_nanos);
+    stream << "file_prepare_write_nanos"
+           << (IOSTATS(prepare_write_nanos) - prev_prepare_write_nanos);
+  }
+
  return s;
 }

@@ -233,14 +262,15 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,

      TEST_SYNC_POINT_CALLBACK("FlushJob::WriteLevel0Table:output_compression",
                               &output_compression_);
-      s = BuildTable(dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
-                     cfd_->table_cache(), iter.get(), meta,
-                     cfd_->internal_comparator(),
-                     cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
-                     existing_snapshots_, earliest_write_conflict_snapshot_,
-                     output_compression_, cfd_->ioptions()->compression_opts,
-                     mutable_cf_options_.paranoid_file_checks,
-                     cfd_->internal_stats(), Env::IO_HIGH, &table_properties_);
+      s = BuildTable(
+          dbname_, db_options_.env, *cfd_->ioptions(), env_options_,
+          cfd_->table_cache(), iter.get(), meta, cfd_->internal_comparator(),
+          cfd_->int_tbl_prop_collector_factories(), cfd_->GetID(),
+          cfd_->GetName(), existing_snapshots_,
+          earliest_write_conflict_snapshot_, output_compression_,
+          cfd_->ioptions()->compression_opts,
+          mutable_cf_options_.paranoid_file_checks, cfd_->internal_stats(),
+          Env::IO_HIGH, &table_properties_, 0 /* level */);
      info.table_properties = table_properties_;
      LogFlush(db_options_.info_log);
    }
@@ -270,6 +300,7 @@ Status FlushJob::WriteLevel0Table(const autovector<MemTable*>& mems,
    if (!db_options_.disableDataSync && output_file_directory_ != nullptr) {
      output_file_directory_->Fsync();
    }
+    TEST_SYNC_POINT("FlushJob::WriteLevel0Table");
    db_mutex_->Lock();
  }
  base->Unref();
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -62,7 +62,7 @@ class FlushJob {
           JobContext* job_context, LogBuffer* log_buffer,
           Directory* db_directory, Directory* output_file_directory,
           CompressionType output_compression, Statistics* stats,
-           EventLogger* event_logger);
+           EventLogger* event_logger, bool measure_io_stats);

  ~FlushJob();

@@ -93,6 +93,7 @@ class FlushJob {
  Statistics* stats_;
  EventLogger* event_logger_;
  TableProperties table_properties_;
+  bool measure_io_stats_;
 };

 }  // namespace rocksdb
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -93,7 +93,7 @@ TEST_F(FlushJobTest, Empty) {
                     db_options_, *cfd->GetLatestMutableCFOptions(),
                     env_options_, versions_.get(), &mutex_, &shutting_down_,
                     {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger);
+                     nullptr, kNoCompression, nullptr, &event_logger, false);
  ASSERT_OK(flush_job.Run());
  job_context.Clean();
 }
@@ -132,7 +132,7 @@ TEST_F(FlushJobTest, NonEmpty) {
                     db_options_, *cfd->GetLatestMutableCFOptions(),
                     env_options_, versions_.get(), &mutex_, &shutting_down_,
                     {}, kMaxSequenceNumber, &job_context, nullptr, nullptr,
-                     nullptr, kNoCompression, nullptr, &event_logger);
+                     nullptr, kNoCompression, nullptr, &event_logger, true);
  FileMetaData fd;
  mutex_.Lock();
  ASSERT_OK(flush_job.Run(&fd));
@@ -192,11 +192,11 @@ TEST_F(FlushJobTest, Snapshots) {
  }

  EventLogger event_logger(db_options_.info_log.get());
-  FlushJob flush_job(dbname_, versions_->GetColumnFamilySet()->GetDefault(),
-                     db_options_, *cfd->GetLatestMutableCFOptions(),
-                     env_options_, versions_.get(), &mutex_, &shutting_down_,
-                     snapshots, kMaxSequenceNumber, &job_context, nullptr,
-                     nullptr, nullptr, kNoCompression, nullptr, &event_logger);
+  FlushJob flush_job(
+      dbname_, versions_->GetColumnFamilySet()->GetDefault(), db_options_,
+      *cfd->GetLatestMutableCFOptions(), env_options_, versions_.get(), &mutex_,
+      &shutting_down_, snapshots, kMaxSequenceNumber, &job_context, nullptr,
+      nullptr, nullptr, kNoCompression, nullptr, &event_logger, true);
  mutex_.Lock();
  ASSERT_OK(flush_job.Run());
  mutex_.Unlock();
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -10,15 +10,16 @@
 #include <string>
 #include <utility>

-#include "db/job_context.h"
+#include "db/column_family.h"
 #include "db/db_impl.h"
 #include "db/db_iter.h"
-#include "db/column_family.h"
+#include "db/dbformat.h"
+#include "db/job_context.h"
 #include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
 #include "table/merger.h"
-#include "db/dbformat.h"
+#include "util/string_util.h"
 #include "util/sync_point.h"

 namespace rocksdb {
@@ -471,6 +472,15 @@ Status ForwardIterator::status() const {
  return immutable_status_;
 }

+Status ForwardIterator::GetProperty(std::string prop_name, std::string* prop) {
+  assert(prop != nullptr);
+  if (prop_name == "rocksdb.iterator.super-version-number") {
+    *prop = ToString(sv_->version_number);
+    return Status::OK();
+  }
+  return Status::InvalidArgument();
+}
+
 void ForwardIterator::RebuildIterators(bool refresh_sv) {
  // Clean up
  Cleanup(refresh_sv);
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -71,6 +71,8 @@ class ForwardIterator : public InternalIterator {
  virtual Slice key() const override;
  virtual Slice value() const override;
  virtual Status status() const override;
+  virtual Status GetProperty(std::string prop_name, std::string* prop) override;
+
  bool TEST_CheckDeletedIters(int* deleted_iters, int* num_iters);

 private:
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
@@ -1,4 +1,4 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional
 //  grant of patent rights can be found in the PATENTS file in the same
@@ -147,8 +147,9 @@ class InlineSkipList {
  // values are ok.
  std::atomic<int> max_height_;  // Height of the entire list

-  // Used for optimizing sequential insert patterns.  Tricky.  prev_[i] for
-  // i up to max_height_ - 1 (inclusive) is the predecessor of prev_[0].
+  // Used for optimizing sequential insert patterns.  Tricky.  prev_height_
+  // of zero means prev_ is undefined.  Otherwise: prev_[i] for i up
+  // to max_height_ - 1 (inclusive) is the predecessor of prev_[0], and
  // prev_height_ is the height of prev_[0].  prev_[0] can only be equal
  // to head when max_height_ and prev_height_ are both 1.
  Node** prev_;
@@ -510,11 +511,10 @@ InlineSkipList<Comparator>::AllocateNode(size_t key_size, int height) {

 template <class Comparator>
 void InlineSkipList<Comparator>::Insert(const char* key) {
-  // InsertConcurrently can't maintain the prev_ invariants when it needs
-  // to increase max_height_.  In that case it sets prev_height_ to zero,
-  // letting us know that we should ignore it.  A relaxed load suffices
-  // here because write thread synchronization separates Insert calls
-  // from InsertConcurrently calls.
+  // InsertConcurrently often can't maintain the prev_ invariants, so
+  // it just sets prev_height_ to zero, letting us know that we should
+  // ignore it.  A relaxed load suffices here because write thread
+  // synchronization separates Insert calls from InsertConcurrently calls.
  auto prev_height = prev_height_.load(std::memory_order_relaxed);

  // fast path for sequential insertion
@@ -595,15 +595,24 @@ void InlineSkipList<Comparator>::InsertConcurrently(const char* key) {
  int height = x->UnstashHeight();
  assert(height >= 1 && height <= kMaxHeight_);

+  // We don't have a lock-free algorithm for updating prev_, but we do have
+  // the option of invalidating the entire sequential-insertion cache.
+  // prev_'s invariant is that prev_[i] (i > 0) is the predecessor of
+  // prev_[0] at that level.  We're only going to violate that if height
+  // > 1 and key lands after prev_[height - 1] but before prev_[0].
+  // Comparisons are pretty expensive, so an easier version is to just
+  // clear the cache if height > 1.  We only write to prev_height_ if the
+  // nobody else has, to avoid invalidating the root of the skip list in
+  // all of the other CPU caches.
+  if (height > 1 && prev_height_.load(std::memory_order_relaxed) != 0) {
+    prev_height_.store(0, std::memory_order_relaxed);
+  }
+
  int max_height = max_height_.load(std::memory_order_relaxed);
  while (height > max_height) {
    if (max_height_.compare_exchange_strong(max_height, height)) {
      // successfully updated it
      max_height = height;
-
-      // we dont have a lock-free algorithm for fixing up prev_, so just
-      // mark it invalid
-      prev_height_.store(0, std::memory_order_relaxed);
      break;
    }
    // else retry, possibly exiting the loop because somebody else
--- a/Show More
+++ b/Show More