Add universal compaction to db_stress nightly build

Summary: Most code change in this diff is code cleanup/rewrite. The logic changes include: (1) add universal compaction to db_crashtest2.py (2) randomly set --test_batches_snapshots to be 0 or 1 in db_crashtest2.py. Old codes always use 1. (3) use different tmp directory as db directory in different runs. I saw some intermittent errors in my local tests. Use of different tmp directory seems to be able to solve the issue. Test Plan: Have run "make crashtest" for multiple times. Also run "make all check" Reviewers: emayanke, dhruba, haobo Reviewed By: emayanke Differential Revision: https://reviews.facebook.net/D12369
Made merge_oprator a shared_ptr; and added TTL unit tests
2026-07-07 22:55:23 +08:00 · 2013-08-20 17:37:49 -07:00 · 2013-08-20 13:35:28 -07:00 · 2013-08-19 14:29:40 -07:00 · 2013-08-19 14:27:19 -07:00 · 2013-08-19 13:37:04 -07:00
83 changed files with 4515 additions and 1967 deletions
@@ -1,13 +1,16 @@
 build_config.mk
 *.a
-*.o
+*.arc
+*.d
 *.dylib*
+*.gcda
+*.gcno
+*.o
 *.so
 *.so.*
 *_test
-*.arc
-*.d
 db_bench
+db_repl_stress
 db_stress
 ldb
 leveldb_server
@@ -15,4 +18,6 @@ leveldb_shell
 manifest_dump
 sst_dump
 util/build_version.cc
-db_repl_stress
+build_tools/VALGRIND_LOGS/
+coverage/COVERAGE_REPORT
+util/build_version.cc.tmp
@@ -14,13 +14,13 @@ OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
 #-----------------------------------------------

 # detect what platform we're building on
-$(shell ./build_detect_platform build_config.mk)
+$(shell (cd build_tools/; ROCKSDB_ROOT=.. ./build_detect_platform ../build_config.mk))
 # this file is generated by the previous line to set build flags and sources
 include build_config.mk

 WARNING_FLAGS = -Wall -Werror
 CFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
-CXXFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -std=gnu++0x
+CXXFLAGS += -g $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -std=gnu++0x -Woverloaded-virtual

 LDFLAGS += $(PLATFORM_LDFLAGS)

@@ -31,40 +31,40 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
 TESTUTIL = ./util/testutil.o
 TESTHARNESS = ./util/testharness.o $(TESTUTIL)
 VALGRIND_ERROR = 2
-VALGRIND_DIR = VALGRIND_LOGS
+VALGRIND_DIR = build_tools/VALGRIND_LOGS
 VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
 VALGRIND_OPTS = --error-exitcode=$(VALGRIND_ERROR) --leak-check=full

 TESTS = \
 	arena_test \
+	auto_roll_logger_test \
+	block_test \
 	bloom_test \
 	c_test \
 	cache_test \
 	coding_test \
-	histogram_test \
 	corruption_test \
 	crc32c_test \
 	db_test \
 	dbformat_test \
 	env_test \
+	filelock_test \
 	filename_test \
 	filter_block_test \
+	histogram_test \
 	log_test \
+	manual_compaction_test \
 	memenv_test \
-	skiplist_test \
-	table_test \
-	ttl_test \
-	block_test \
-	version_edit_test \
-	version_set_test \
-	reduce_levels_test \
-	write_batch_test \
-	auto_roll_logger_test \
-	filelock_test \
 	merge_test \
 	redis_test \
-	manual_compaction_test \
-	stringappend_test
+	reduce_levels_test \
+	skiplist_test \
+	stringappend_test \
+	table_test \
+	ttl_test \
+	version_edit_test \
+	version_set_test \
+	write_batch_test

 TOOLS = \
        sst_dump \
@@ -75,7 +75,10 @@ TOOLS = \
 PROGRAMS = db_bench signal_test $(TESTS) $(TOOLS)
 BENCHMARKS = db_bench_sqlite3 db_bench_tree_db

-LIBRARY = librocksdb.a
+# The library name is configurable since we are maintaining libraries of both
+# debug/release mode.
+LIBNAME = librocksdb
+LIBRARY = ${LIBNAME}.a
 MEMENVLIBRARY = libmemenv.a

 default: all
@@ -84,7 +87,7 @@ default: all
 ifneq ($(PLATFORM_SHARED_EXT),)

 ifneq ($(PLATFORM_SHARED_VERSIONED),true)
-SHARED1 = librocksdb.$(PLATFORM_SHARED_EXT)
+SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1)
 SHARED3 = $(SHARED1)
 SHARED = $(SHARED1)
@@ -92,7 +95,7 @@ else
 # Update db.h if you change these.
 SHARED_MAJOR = 2
 SHARED_MINOR = 0
-SHARED1 = librocksdb.$(PLATFORM_SHARED_EXT)
+SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1).$(SHARED_MAJOR)
 SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
 SHARED = $(SHARED1) $(SHARED2) $(SHARED3)
@@ -103,15 +106,25 @@ $(SHARED2): $(SHARED3)
 endif

 $(SHARED3):
-	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)
+	$(CXX) $(LDFLAGS) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(COVERAGEFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) -o $(SHARED3)

 endif  # PLATFORM_SHARED_EXT

 all: $(SHARED) $(LIBRARY) $(PROGRAMS)

+.PHONY: blackbox_crash_test check clean coverage crash_test ldb_tests \
+	release tags valgrind_check whitebox_crash_test
+
 release:
-	make clean
-	OPT=-DNDEBUG make -j32
+	$(MAKE) clean
+	OPT=-DNDEBUG $(MAKE) -j32
+
+coverage:
+	$(MAKE) clean
+	COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) all check
+	(cd coverage; ./coverage_test.sh)
+	# Delete intermediate files
+	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm

 check: all $(PROGRAMS) $(TESTS) $(TOOLS) ldb_tests
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
@@ -128,6 +141,7 @@ whitebox_crash_test: db_stress
 	python -u tools/db_crashtest2.py

 valgrind_check: all $(PROGRAMS) $(TESTS)
+	mkdir -p $(VALGRIND_DIR)
 	echo TESTS THAT HAVE VALGRIND ERRORS > $(VALGRIND_DIR)/valgrind_failed_tests; \
 	echo TIMES in seconds TAKEN BY TESTS ON VALGRIND > $(VALGRIND_DIR)/valgrind_tests_times; \
 	for t in $(filter-out skiplist_test,$(TESTS)); do \
@@ -143,138 +157,146 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
 clean:
 	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) $(THRIFTSERVER) build_config.mk
 	-rm -rf ios-x86/* ios-arm/*
-	-for x in `find . -name "*.[od]"`; do rm $$x; done
+	-find . -name "*.[od]" | xargs --no-run-if-empty rm
+	-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
+tags:
+	ctags * -R
+	cscope -b `find . -name '*.cc'` `find . -name '*.h'`

+# ---------------------------------------------------------------------------
+# 	Unit tests and tools
+# ---------------------------------------------------------------------------
 $(LIBRARY): $(LIBOBJECTS)
 	rm -f $@
 	$(AR) -rs $@ $(LIBOBJECTS)

-
 db_bench: db/db_bench.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) db/db_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

 db_stress: tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) tools/db_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

 db_repl_stress: tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

 db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3
+	$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3 $(COVERAGEFLAGS)

 db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet
+	$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet $(COVERAGEFLAGS)

 signal_test: util/signal_test.o $(LIBOBJECTS)
-	$(CXX) util/signal_test.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/signal_test.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 bloom_test: util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/bloom_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 c_test: db/c_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 cache_test: util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/cache_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) utilities/merge_operators/string_append/stringappend_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS)
+	$(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)

 corruption_test: db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/corruption_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 crc32c_test: util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/crc32c_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 db_test: db/db_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
+perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)

 ttl_test: utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) utilities/ttl/ttl_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

 dbformat_test: db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/dbformat_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 env_test: util/env_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/env_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 log_test: db/log_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/log_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 table_test: table/table_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) table/table_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 block_test: table/block_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) table/block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) table/block_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 skiplist_test: db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/skiplist_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 reduce_levels_test: tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 $(MEMENVLIBRARY) : $(MEMENVOBJECTS)
 	rm -f $@
 	$(AR) -rs $@ $(MEMENVOBJECTS)

 memenv_test : helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS)
-	$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) helpers/memenv/memenv_test.o $(MEMENVLIBRARY) $(LIBRARY) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 manual_compaction_test: util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/manual_compaction_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 rocksdb_shell: tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o tools/shell/ShellContext.h tools/shell/ShellState.h tools/shell/DBClientProxy.h $(LIBOBJECTS)
-	$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) tools/shell/ShellContext.o tools/shell/ShellState.o tools/shell/LeveldbShell.o tools/shell/DBClientProxy.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 DBClientProxy_test: tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY)
-	$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS)
+	$(CXX) tools/shell/test/DBClientProxyTest.o tools/shell/DBClientProxy.o $(LIBRARY) $(EXEC_LDFLAGS) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)

 filelock_test: util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/filelock_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 auto_roll_logger_test: util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS)
-	$(CXX) util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) util/auto_roll_logger_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 sst_dump: tools/sst_dump.o $(LIBOBJECTS)
-	$(CXX) tools/sst_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) tools/sst_dump.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

 ldb: tools/ldb.o $(LIBOBJECTS)
-	$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+	$(CXX) tools/ldb.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)

-.PHONY: tags
-tags:
-	ctags `find . -name '*.cc'` `find . -name '*.h'`
-	cscope -b `find . -name '*.cc'` `find . -name '*.h'`
+# ---------------------------------------------------------------------------
+#  	Platform-specific compilation
+# ---------------------------------------------------------------------------

 ifeq ($(PLATFORM), IOS)
 # For iOS, create universal object files to be used on both the simulator and
@@ -286,9 +308,9 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/versionCFBun

 .cc.o:
 	mkdir -p ios-x86/$(dir $@)
-	$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@
+	$(SIMULATORROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -c $< -o ios-x86/$@ $(COVERAGEFLAGS)
 	mkdir -p ios-arm/$(dir $@)
-	$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@
+	$(DEVICEROOT)/usr/bin/$(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -c $< -o ios-arm/$@ $(COVERAGEFLAGS)
 	lipo ios-x86/$@ ios-arm/$@ -create -output $@

 .c.o:
@@ -300,17 +322,26 @@ IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/versionCFBun

 else
 .cc.o:
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
+	$(CXX) $(CXXFLAGS) -c $< -o $@ $(COVERAGEFLAGS)

 .c.o:
-	$(CC) $(CFLAGS) $(PLATFORM_SHARED_CFLAGS) -c $< -o $@
+	$(CC) $(CFLAGS) -c $< -o $@
 endif

+# ---------------------------------------------------------------------------
+#  	Source files dependencies detection
+# ---------------------------------------------------------------------------
+
+# Add proper dependency support so changing a .h file forces a .cc file to
+# rebuild.
+
+# The .d file indicates .cc file's dependencies on .h files. We generate such
+# dependency by g++'s -MM option, whose output is a make dependency rule.
+# The sed command makes sure the "target" file in the generated .d file has
+# the correct path prefix.
 %.d: %.cc
-	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $*.d
-	@cp -f $*.d $*.d.tmp
-	@sed -e 's|.*:|$*.o:|' < $*.d.tmp > $*.d
-	@rm -f $*.d.tmp
+	$(CXX) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) -MM $< -o $@
+	@sed -i -e 's|.*:|$*.o:|' $@

 DEPFILES = $(filter-out util/build_version.d,$(SOURCES:.cc=.d))

@@ -319,4 +350,3 @@ depend: $(DEPFILES)
 ifneq ($(MAKECMDGOALS),clean)
 -include $(DEPFILES)
 endif
-
@@ -1,4 +0,0 @@
-This directory stores the tests that failed valgrind and the times associated
-with the failed runs.
-"make valgrind_check" can be invoked to call valgrind on the rocksdb tests and
-generate files in this directory
@@ -1,30 +0,0 @@
-#!/bin/sh
-#
-# Record the version of the source that we are compiling.
-# We keep a record of the git revision in util/version.cc. This source file
-# is then built as a regular source file as part of the compilation process.
-# One can run "strings executable_filename | grep _build_" to find the version of
-# the source that we used to build the executable file.
-#
-
-# create git version file
-VFILE=$(mktemp)
-trap "rm $VFILE" EXIT
-
-# check to see if git is in the path
-which git > /dev/null
-
-if [ "$?" = 0 ]; then
-  env -i git rev-parse HEAD | awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";"} END {}' >  ${VFILE}
-else
-  echo "git not found"| awk ' BEGIN {print "#include \"build_version.h\""} {print "const char * leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";"} END {}' > ${VFILE}
-fi
-
-date | awk 'BEGIN {} {print "const char * leveldb_build_git_datetime = \"leveldb_build_git_datetime:"$0"\";"} END {} ' >> ${VFILE}
-echo "const char * leveldb_build_compile_date = __DATE__;" >> ${VFILE}
-echo "const char * leveldb_build_compile_time = __TIME__;" >> ${VFILE}
-
-OUTFILE=util/build_version.cc
-if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
-    cp $VFILE $OUTFILE
-fi
@@ -65,7 +65,7 @@ PLATFORM_SHARED_CFLAGS="-fPIC"
 PLATFORM_SHARED_VERSIONED=true

 # generic port files (working on all platform by #ifdef) go directly in /port
-GENERIC_PORT_FILES=`find port -name '*.cc' | tr "\n" " "`
+GENERIC_PORT_FILES=`cd $ROCKSDB_ROOT; find port -name '*.cc' | tr "\n" " "`

 # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
 case "$TARGET_OS" in
@@ -78,7 +78,7 @@ case "$TARGET_OS" in
        ;;
    Linux)
        PLATFORM=OS_LINUX
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX -fPIC"
+        COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
        if [ -z "$USE_CLANG" ]; then
            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
        fi
@@ -146,8 +146,8 @@ fi
 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
 PRUNE_BENCH="-name *_bench.cc -prune"
-PORTABLE_FILES=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
-PORTABLE_CPP=`find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
+PORTABLE_FILES=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
+PORTABLE_CPP=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
 set +f # re-enable globbing

 # The sources consist of the portable files, plus the platform-specific port
@@ -0,0 +1,42 @@
+#!/bin/sh
+#
+# Record the version of the source that we are compiling.
+# We keep a record of the git revision in util/version.cc. This source file
+# is then built as a regular source file as part of the compilation process.
+# One can run "strings executable_filename | grep _build_" to find the version of
+# the source that we used to build the executable file.
+#
+
+# create git version file
+VFILE=$ROCKSDB_ROOT/util/build_version.cc.tmp
+trap "rm $VFILE" EXIT
+
+# check to see if git is in the path
+which git > /dev/null
+
+if [ "$?" = 0 ]; then
+  env -i git rev-parse HEAD |
+  awk '
+  BEGIN {
+    print "#include \"build_version.h\"\n"
+  } 
+  { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
+  ' >  ${VFILE}
+else
+  echo "git not found" |
+  awk '
+    BEGIN {
+      print "#include \"build_version.h\"" 
+    } 
+    { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
+    ' > ${VFILE}
+fi
+
+echo "const char* leveldb_build_git_datetime = \"leveldb_build_git_datetime:$(date)\";" >> ${VFILE}
+echo "const char* leveldb_build_compile_date = __DATE__;" >> ${VFILE}
+echo "const char* leveldb_build_compile_time = __TIME__;" >> ${VFILE}
+
+OUTFILE=$ROCKSDB_ROOT/util/build_version.cc
+if [ ! -e $OUTFILE ] || ! cmp -s $VFILE $OUTFILE; then
+    cp $VFILE $OUTFILE
+fi
@@ -14,8 +14,8 @@ fi
 # On the production build servers, set data and stat
 # files/directories not in /tmp or else the tempdir cleaning
 # scripts will make you very unhappy.
-DATA_DIR=${DATA_DIR:-$(mktemp --tmpdir -d leveldb_XXXX)}
-STAT_FILE=${STAT_FILE:-$(mktemp --tmpdir -u leveldb_test_stats_XXXX)}
+DATA_DIR=${DATA_DIR:-$(mktemp -t -d rocksdb_XXXX)}
+STAT_FILE=${STAT_FILE:-$(mktemp -t -u rocksdb_test_stats_XXXX)}

 function cleanup {
  rm -rf $DATA_DIR
@@ -34,7 +34,8 @@ function send_to_ods {
    echo >&2 "ERROR: Key $key doesn't have a value."
    return
  fi
-  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=$key&value=$value"
+  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=$key&value=$value" \
+    --connect-timeout 60
 }

 make clean
@@ -2,7 +2,7 @@
 #A shell script for Jenknis to run valgrind on rocksdb tests
 #Returns 0 on success when there are no failed tests 

-VALGRIND_DIR=VALGRIND_LOGS
+VALGRIND_DIR=build_tools/VALGRIND_LOGS
 make clean
 make -j$(nproc) valgrind_check
 NUM_FAILED_TESTS=$((`wc -l $VALGRIND_DIR/valgrind_failed_tests | awk '{print $1}'` - 1))
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# Exit on error.
+set -e
+
+if [ -n "$USE_CLANG" ]; then
+  echo "Error: Coverage test is supported only for gcc."
+  exit 1
+fi
+
+ROOT=".."
+# Fetch right version of gcov
+if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
+  source $ROOT/build_tools/fbcode.gcc471.sh
+  GCOV=$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.7.1/cc6c9dc/bin/gcov
+else
+  GCOV=$(which gcov)
+fi
+
+COVERAGE_DIR="$PWD/COVERAGE_REPORT"
+mkdir -p $COVERAGE_DIR
+
+# Find all gcno files to generate the coverage report
+
+GCNO_FILES=`find $ROOT -name "*.gcno"`
+$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
+  # Parse the raw gcov report to more human readable form.
+  python $ROOT/coverage/parse_gcov_output.py |
+  # Write the output to both stdout and report file.
+  tee $COVERAGE_DIR/coverage_report_all.txt &&
+echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
+
+# TODO: we also need to get the files of the latest commits.
+# Get the most recently committed files.
+LATEST_FILES=`
+  git show --pretty="format:" --name-only HEAD |
+  grep -v "^$" |
+  paste -s -d,`
+RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
+
+echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
+$GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
+  python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
+  tee -a $RECENT_REPORT &&
+echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
+
+# Generate the html report. If we cannot find lcov in this machine, we'll simply
+# skip this step.
+echo "Generating the html coverage report..."
+
+LCOV=$(which lcov || true 2>/dev/null)
+if [ -z $LCOV ]
+then
+  echo "Skip: Cannot find lcov to generate the html report."
+  exit 0
+fi
+
+(cd $ROOT; lcov --no-external \
+     --capture  \
+     --directory $PWD \
+     --gcov-tool $GCOV \
+     --output-file $COVERAGE_DIR/coverage.info &>/dev/null)
+
+genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR &>/dev/null
+
+echo "HTML Coverage report is generated in $COVERAGE_DIR"
@@ -0,0 +1,118 @@
+import optparse
+import re
+import sys
+
+from optparse import OptionParser
+
+# the gcov report follows certain pattern. Each file will have two lines
+# of report, from which we can extract the file name, total lines and coverage
+# percentage.
+def parse_gcov_report(gcov_input):
+    per_file_coverage = {}
+    total_coverage = None
+
+    for line in sys.stdin:
+        line = line.strip()
+
+        # --First line of the coverage report (with file name in it)?
+        match_obj = re.match("^File '(.*)'$", line)
+        if match_obj:
+            # fetch the file name from the first line of the report.
+            current_file = match_obj.group(1)
+            continue
+
+        # -- Second line of the file report (with coverage percentage)
+        match_obj = re.match("^Lines executed:(.*)% of (.*)", line)
+
+        if match_obj:
+            coverage = float(match_obj.group(1))
+            lines = int(match_obj.group(2))
+
+            if current_file is not None:
+                per_file_coverage[current_file] = (coverage, lines)
+                current_file = None
+            else:
+                # If current_file is not set, we reach the last line of report,
+                # which contains the summarized coverage percentage.
+                total_coverage = (coverage, lines)
+            continue
+
+        # If the line's pattern doesn't fall into the above categories. We
+        # can simply ignore them since they're either empty line or doesn't
+        # find executable lines of the given file.
+        current_file = None
+
+    return per_file_coverage, total_coverage
+
+def get_option_parser():
+    usage = "Parse the gcov output and generate more human-readable code " +\
+            "coverage report."
+    parser = OptionParser(usage)
+
+    parser.add_option(
+        "--interested-files", "-i",
+        dest="filenames",
+        help="Comma separated files names. if specified, we will display " +
+             "the coverage report only for interested source files. " +
+             "Otherwise we will display the coverage report for all " +
+             "source files."
+    )
+    return parser
+
+def display_file_coverage(per_file_coverage, total_coverage):
+    # To print out auto-adjustable column, we need to know the longest
+    # length of file names.
+    max_file_name_length = max(
+        len(fname) for fname in per_file_coverage.keys()
+    )
+
+    # -- Print header
+    # size of separator is determined by 3 column sizes:
+    # file name, coverage percentage and lines.
+    header_template = \
+        "%" + str(max_file_name_length) + "s\t%s\t%s"
+    separator = "-" * (max_file_name_length + 10 + 20)
+    print header_template % ("Filename", "Coverage", "Lines")
+    print separator
+
+    # -- Print body
+    # template for printing coverage report for each file.
+    record_template = "%" + str(max_file_name_length) + "s\t%5.2f%%\t%10d"
+
+    for fname, coverage_info in per_file_coverage.items():
+        coverage, lines = coverage_info
+        print record_template % (fname, coverage, lines)
+
+    # -- Print footer
+    if total_coverage:
+        print separator
+        print record_template % ("Total", total_coverage[0], total_coverage[1])
+
+def report_coverage():
+    parser = get_option_parser()
+    (options, args) = parser.parse_args()
+
+    interested_files = set()
+    if options.filenames is not None:
+        interested_files = set(f.strip() for f in options.filenames.split(','))
+
+    # To make things simple, right now we only read gcov report from the input
+    per_file_coverage, total_coverage = parse_gcov_report(sys.stdin)
+
+    # Check if we need to display coverage info for interested files.
+    if len(interested_files):
+        per_file_coverage = dict(
+            (fname, per_file_coverage[fname]) for fname in interested_files
+            if fname in per_file_coverage
+        )
+        # If we only interested in several files, it makes no sense to report
+        # the total_coverage
+        total_coverage = None
+
+    if not len(per_file_coverage):
+        print >> sys.stderr, "Cannot find coverage info for the given files."
+        return
+    display_file_coverage(per_file_coverage, total_coverage)
+
+if __name__ == "__main__":
+    report_coverage()
@@ -28,6 +28,7 @@ Status BuildTable(const std::string& dbname,
                  const SequenceNumber earliest_seqno_in_memtable) {
  Status s;
  meta->file_size = 0;
+  meta->smallest_seqno = meta->largest_seqno = 0;
  iter->SeekToFirst();

  // If the sequence number of the smallest entry in the memtable is
@@ -48,9 +49,12 @@ Status BuildTable(const std::string& dbname,
    TableBuilder* builder = new TableBuilder(options, file.get(), 0);

    // the first key is the smallest key
-    meta->smallest.DecodeFrom(iter->key());
+    Slice key = iter->key();
+    meta->smallest.DecodeFrom(key);
+    meta->smallest_seqno = GetInternalKeySeqno(key);
+    meta->largest_seqno = meta->smallest_seqno;

-    MergeHelper merge(user_comparator, options.merge_operator,
+    MergeHelper merge(user_comparator, options.merge_operator.get(),
                      options.info_log.get(),
                      true /* internal key corruption is not ok */);

@@ -135,12 +139,18 @@ Status BuildTable(const std::string& dbname,

      // The last key is the largest key
      meta->largest.DecodeFrom(Slice(prev_key));
+      SequenceNumber seqno = GetInternalKeySeqno(Slice(prev_key));
+      meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
+      meta->largest_seqno = std::max(meta->largest_seqno, seqno);

    } else {
      for (; iter->Valid(); iter->Next()) {
        Slice key = iter->key();
        meta->largest.DecodeFrom(key);
        builder->Add(key, iter->value());
+        SequenceNumber seqno = GetInternalKeySeqno(key);
+        meta->smallest_seqno = std::min(meta->smallest_seqno, seqno);
+        meta->largest_seqno = std::max(meta->largest_seqno, seqno);
      }
    }

@@ -9,6 +9,7 @@
 #include "db/db_impl.h"
 #include "db/version_set.h"
 #include "db/db_statistics.h"
+#include "leveldb/options.h"
 #include "leveldb/cache.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
@@ -24,6 +25,7 @@
 #include "util/string_util.h"
 #include "util/testutil.h"
 #include "hdfs/env_hdfs.h"
+#include "utilities/merge_operators.h"

 // Comma-separated list of operations to run in the specified order
 //   Actual benchmarks:
@@ -42,6 +44,9 @@
 //      readwhilewriting -- 1 writer, N threads doing random reads
 //      readrandomwriterandom - N threads doing random-read, random-write
 //      updaterandom  -- N threads doing read-modify-write for random keys
+//      appendrandom  -- N threads doing read-modify-write with growing values
+//      mergerandom   -- same as updaterandom/appendrandom using merge operator
+//                    -- must be used with FLAGS_merge_operator (see below)
 //      seekrandom    -- N random seeks
 //      crc32c        -- repeated crc32c of 4K of data
 //      acquireload   -- load N*1000 times
@@ -125,9 +130,9 @@ static int FLAGS_max_write_buffer_number = 0;
 // The minimum number of write buffers that will be merged together
 // before writing to storage. This is cheap because it is an
 // in-memory merge. If this feature is not enabled, then all these
-// write buffers are fushed to L0 as seperate files and this increases
+// write buffers are flushed to L0 as separate files and this increases
 // read amplification because a get request has to check in all of these
-// files. Also, an in-memory merge may result in writing lesser
+// files. Also, an in-memory merge may result in writing less
 // data to storage if there are duplicate records in each of these
 // individual write buffers.
 static int FLAGS_min_write_buffer_number_to_merge = 0;
@@ -137,6 +142,17 @@ static int FLAGS_min_write_buffer_number_to_merge = 0;
 // This is initialized to default value of 1 in "main" function.
 static int FLAGS_max_background_compactions = 0;

+// style of compaction: level-based vs universal
+static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel;
+
+// Percentage flexibility while comparing file size
+// (for universal compaction only).
+static int FLAGS_universal_size_ratio = 1;
+
+// The minimum number of files in a single compaction run
+// (for universal compaction only).
+static int FLAGS_compaction_universal_min_merge_width = 2;
+
 // Number of bytes to use as a cache of uncompressed data.
 // Negative means use default settings.
 static long FLAGS_cache_size = -1;
@@ -202,16 +218,16 @@ static bool FLAGS_get_approx = false;
 // The total number of levels
 static int FLAGS_num_levels = 7;

-// Target level-0 file size for compaction
+// Target file size at level-1
 static int FLAGS_target_file_size_base = 2 * 1048576;

-// A multiplier to compute targe level-N file size
+// A multiplier to compute target level-N file size (N >= 2)
 static int FLAGS_target_file_size_multiplier = 1;

 // Max bytes for level-1
 static uint64_t FLAGS_max_bytes_for_level_base = 10 * 1048576;

-// A multiplier to compute max bytes for level-N
+// A multiplier to compute max bytes for level-N (N >= 2)
 static int FLAGS_max_bytes_for_level_multiplier = 10;

 // A vector that specifies additional fanout per level
@@ -226,16 +242,19 @@ static int FLAGS_level0_slowdown_writes_trigger = 8;
 // Number of files in level-0 when compactions start
 static int FLAGS_level0_file_num_compaction_trigger = 4;

-// Ratio of reads to writes (expressed as a percentage)
-// for the ReadRandomWriteRandom workload. The default
-// setting is 9 gets for every 1 put.
+// Ratio of reads to reads/writes (expressed as percentage) for the
+// ReadRandomWriteRandom workload. The default value 90 means 90% operations
+// out of all reads and writes operations are reads. In other words, 9 gets
+// for every 1 put.
 static int FLAGS_readwritepercent = 90;

-// This percent of deletes are done (used in RandomWithVerify only)
-// Must be smaller than total writepercent (i.e 100 - FLAGS_readwritepercent)
+// Percentage of deletes out of reads/writes/deletes (used in RandomWithVerify
+// only). RandomWithVerify calculates writepercent as
+// (100 - FLAGS_readwritepercent - FLAGS_deletepercent), so FLAGS_deletepercent
+// must be smaller than (100 - FLAGS_readwritepercent)
 static int FLAGS_deletepercent = 2;

-// Option to disable compation triggered by read.
+// Option to disable compaction triggered by read.
 static int FLAGS_disable_seek_compaction = false;

 // Option to delete obsolete files periodically
@@ -243,12 +262,13 @@ static int FLAGS_disable_seek_compaction = false;
 // deleted after every compaction run.
 static uint64_t FLAGS_delete_obsolete_files_period_micros = 0;

-// Algorithm to use to compress the database
+// Algorithm used to compress the database
 static enum leveldb::CompressionType FLAGS_compression_type =
    leveldb::kSnappyCompression;

-// Allows compression for levels 0 and 1 to be disabled when
-// other levels are compressed
+// If non-negative, compression starts from this level. Levels with number
+// < FLAGS_min_level_to_compress are not compressed.
+// Otherwise, apply FLAGS_compression_type to all levels.
 static int FLAGS_min_level_to_compress = -1;

 static int FLAGS_table_cache_numshardbits = 4;
@@ -285,8 +305,8 @@ static bool FLAGS_read_only = false;
 // Do not auto trigger compactions
 static bool FLAGS_disable_auto_compactions = false;

-// Cap the size of data in levelK for a compaction run
-// that compacts Levelk with LevelK+1
+// Cap the size of data in level-K for a compaction run
+// that compacts Level-K with Level-(K+1) (for K >= 1)
 static int FLAGS_source_compaction_factor = 1;

 // Set the TTL for the WAL Files.
@@ -333,6 +353,11 @@ static auto FLAGS_bytes_per_sync =
 // On true, deletes use bloom-filter and drop the delete if key not present
 static bool FLAGS_filter_deletes = false;

+// The merge operator to use with the database.
+// If a new merge operator is specified, be sure to use fresh database
+// The possible merge operators are defined in utilities/merge_operators.h
+static std::string FLAGS_merge_operator = "";
+
 namespace leveldb {

 // Helper for quickly generating random data.
@@ -366,6 +391,7 @@ class RandomGenerator {
    return Slice(data_.data() + pos_ - len, len);
  }
 };
+
 static Slice TrimSpace(Slice s) {
  unsigned int start = 0;
  while (start < s.size() && isspace(s[start])) {
@@ -774,20 +800,20 @@ class Benchmark {
    delete db_;
    delete filter_policy_;
  }
-//this function will construct string format for key. e.g "%016d"
-void ConstructStrFormatForKey(char* str, int keySize)
-{
-  str[0] = '%';
-  str[1] = '0';
-  sprintf(str+2, "%dd%s", keySize, "%s");
-}

-unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
-{
-  unique_ptr<char []> keyInStr(new char[MAX_KEY_SIZE]);
-  snprintf(keyInStr.get(), MAX_KEY_SIZE, keyFormat_, v, suffix);
-  return keyInStr;
-}
+  //this function will construct string format for key. e.g "%016d"
+  void ConstructStrFormatForKey(char* str, int keySize) {
+    str[0] = '%';
+    str[1] = '0';
+    sprintf(str+2, "%dd%s", keySize, "%s");
+  }
+
+  unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "") {
+    unique_ptr<char []> keyInStr(new char[MAX_KEY_SIZE]);
+    snprintf(keyInStr.get(), MAX_KEY_SIZE, keyFormat_, v, suffix);
+    return keyInStr;
+  }
+
  void Run() {
    PrintHeader();
    Open();
@@ -804,7 +830,7 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
        benchmarks = sep + 1;
      }

-      // Reset parameters that may be overriddden bwlow
+      // Sanitize parameters
      num_ = FLAGS_num;
      reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
      writes_ = (FLAGS_writes < 0 ? FLAGS_num : FLAGS_writes);
@@ -879,6 +905,16 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
        method = &Benchmark::ReadRandomWriteRandom;
      } else if (name == Slice("updaterandom")) {
        method = &Benchmark::UpdateRandom;
+      } else if (name == Slice("appendrandom")) {
+        method = &Benchmark::AppendRandom;
+      } else if (name == Slice("mergerandom")) {
+        if (FLAGS_merge_operator.empty()) {
+          fprintf(stdout, "%-12s : skipped (--merge_operator is unknown)\n",
+                  name.ToString().c_str());
+          method = nullptr;
+        } else {
+          method = &Benchmark::MergeRandom;
+        }
      } else if (name == Slice("randomwithverify")) {
        method = &Benchmark::RandomWithVerify;
      } else if (name == Slice("compact")) {
@@ -1104,6 +1140,10 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
    options.min_write_buffer_number_to_merge =
      FLAGS_min_write_buffer_number_to_merge;
    options.max_background_compactions = FLAGS_max_background_compactions;
+    options.compaction_style = FLAGS_compaction_style;
+    options.compaction_options_universal.size_ratio = FLAGS_universal_size_ratio;
+    options.compaction_options_universal.min_merge_width =
+      FLAGS_compaction_universal_min_merge_width;
    options.block_size = FLAGS_block_size;
    options.filter_policy = filter_policy_;
    options.max_open_files = FLAGS_open_files;
@@ -1169,6 +1209,15 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
    options.use_adaptive_mutex = FLAGS_use_adaptive_mutex;
    options.bytes_per_sync = FLAGS_bytes_per_sync;

+    // merge operator options
+    options.merge_operator = MergeOperators::CreateFromStringId(
+        FLAGS_merge_operator);
+    if (options.merge_operator == nullptr && !FLAGS_merge_operator.empty()) {
+      fprintf(stderr, "invalid merge operator: %s\n",
+              FLAGS_merge_operator.c_str());
+      exit(1);
+    }
+
    Status s;
    if(FLAGS_read_only) {
      s = DB::OpenForReadOnly(options, FLAGS_db, &db_);
@@ -1364,7 +1413,7 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
      // Recalculate number of keys per group, and call MultiGet until done
      long num_keys;
      while(num_keys = std::min(keys_left, kpg), !duration.Done(num_keys)) {
-        found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num,"");
+        found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num, "");
        thread->stats.FinishedSingleOp(db_);
        keys_left -= num_keys;
      }
@@ -1418,7 +1467,7 @@ unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "")
    thread->stats.AddMessage(msg);
  }

-void ReadMissing(ThreadState* thread) {
+  void ReadMissing(ThreadState* thread) {
    FLAGS_warn_missing_keys = false;    // Never warn about missing keys

    Duration duration(FLAGS_duration, reads_);
@@ -1432,10 +1481,14 @@ void ReadMissing(ThreadState* thread) {
      long num_keys;
      long found;
      while(num_keys = std::min(keys_left, kpg), !duration.Done(num_keys)) {
-        found = MultiGetRandom(options, num_keys, thread->rand, FLAGS_num,".");
-        if (!found) {
+        found = MultiGetRandom(options, num_keys, thread->rand, FLAGS_num, ".");
+
+        // We should not find any key since the key we try to get has a
+        // different suffix
+        if (found) {
          assert(false);
        }
+
        thread->stats.FinishedSingleOp(db_);
        keys_left -= num_keys;
      }
@@ -1735,10 +1788,8 @@ void ReadMissing(ThreadState* thread) {
    thread->stats.AddMessage(msg);
  }

-  //
-  // This is diffferent from ReadWhileWriting because it does not use
+  // This is different from ReadWhileWriting because it does not use
  // an extra thread.
-  //
  void ReadRandomWriteRandom(ThreadState* thread) {
    if (FLAGS_use_multiget){
      // Separate function for multiget (for ease of reading)
@@ -1761,7 +1812,7 @@ void ReadMissing(ThreadState* thread) {
      const int k = thread->rand.Next() % FLAGS_num;
      unique_ptr<char []> key = GenerateKeyFromInt(k);
      if (get_weight == 0 && put_weight == 0) {
-        // one batch complated, reinitialize for next batch
+        // one batch completed, reinitialize for next batch
        get_weight = FLAGS_readwritepercent;
        put_weight = 100 - get_weight;
      }
@@ -1862,7 +1913,7 @@ void ReadMissing(ThreadState* thread) {
      assert(num_keys + num_put_keys <= keys_left);

      // Apply the MultiGet operations
-      found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num,"");
+      found += MultiGetRandom(options, num_keys, thread->rand, FLAGS_num, "");
      ++multigets_done;
      reads_done+=num_keys;
      thread->stats.FinishedSingleOp(db_);
@@ -1894,8 +1945,6 @@ void ReadMissing(ThreadState* thread) {

  //
  // Read-modify-write for random keys
-  //
-  // TODO: Implement MergeOperator tests here (Read-modify-write)
  void UpdateRandom(ThreadState* thread) {
    ReadOptions options(FLAGS_verify_checksum, true);
    RandomGenerator gen;
@@ -1942,6 +1991,100 @@ void ReadMissing(ThreadState* thread) {
    thread->stats.AddMessage(msg);
  }

+  // Read-modify-write for random keys.
+  // Each operation causes the key grow by value_size (simulating an append).
+  // Generally used for benchmarking against merges of similar type
+  void AppendRandom(ThreadState* thread) {
+    ReadOptions options(FLAGS_verify_checksum, true);
+    RandomGenerator gen;
+    std::string value;
+    long found = 0;
+
+    // The number of iterations is the larger of read_ or write_
+    Duration duration(FLAGS_duration, readwrites_);
+    while (!duration.Done(1)) {
+      const int k = thread->rand.Next() % FLAGS_num;
+      unique_ptr<char []> key = GenerateKeyFromInt(k);
+
+      if (FLAGS_use_snapshot) {
+        options.snapshot = db_->GetSnapshot();
+      }
+
+      if (FLAGS_get_approx) {
+        char key2[100];
+        snprintf(key2, sizeof(key2), "%016d", k + 1);
+        Slice skey2(key2);
+        Slice skey(key2);
+        Range range(skey, skey2);
+        uint64_t sizes;
+        db_->GetApproximateSizes(&range, 1, &sizes);
+      }
+
+      // Get the existing value
+      if (db_->Get(options, key.get(), &value).ok()) {
+        found++;
+      } else {
+        // If not existing, then just assume an empty string of data
+        value.clear();
+      }
+
+      if (FLAGS_use_snapshot) {
+        db_->ReleaseSnapshot(options.snapshot);
+      }
+
+      // Update the value (by appending data)
+      Slice operand = gen.Generate(value_size_);
+      if (value.size() > 0) {
+        // Use a delimeter to match the semantics for StringAppendOperator
+        value.append(1,',');
+      }
+      value.append(operand.data(), operand.size());
+
+      // Write back to the database
+      Status s = db_->Put(write_options_, key.get(), value);
+      if (!s.ok()) {
+        fprintf(stderr, "put error: %s\n", s.ToString().c_str());
+        exit(1);
+      }
+      thread->stats.FinishedSingleOp(db_);
+    }
+    char msg[100];
+    snprintf(msg, sizeof(msg), "( updates:%ld found:%ld)", readwrites_, found);
+    thread->stats.AddMessage(msg);
+  }
+
+  // Read-modify-write for random keys (using MergeOperator)
+  // The merge operator to use should be defined by FLAGS_merge_operator
+  // Adjust FLAGS_value_size so that the keys are reasonable for this operator
+  // Assumes that the merge operator is non-null (i.e.: is well-defined)
+  //
+  // For example, use FLAGS_merge_operator="uint64add" and FLAGS_value_size=8
+  // to simulate random additions over 64-bit integers using merge.
+  void MergeRandom(ThreadState* thread) {
+    RandomGenerator gen;
+
+    // The number of iterations is the larger of read_ or write_
+    Duration duration(FLAGS_duration, readwrites_);
+    while (!duration.Done(1)) {
+      const int k = thread->rand.Next() % FLAGS_num;
+      unique_ptr<char []> key = GenerateKeyFromInt(k);
+
+      Status s = db_->Merge(write_options_, key.get(),
+                            gen.Generate(value_size_));
+
+      if (!s.ok()) {
+        fprintf(stderr, "merge error: %s\n", s.ToString().c_str());
+        exit(1);
+      }
+      thread->stats.FinishedSingleOp(db_);
+    }
+
+    // Print some statistics
+    char msg[100];
+    snprintf(msg, sizeof(msg), "( updates:%ld)", readwrites_);
+    thread->stats.AddMessage(msg);
+  }
+
  void Compact(ThreadState* thread) {
    db_->CompactRange(nullptr, nullptr);
  }
@@ -1988,6 +2131,11 @@ int main(int argc, char** argv) {
  FLAGS_open_files = leveldb::Options().max_open_files;
  FLAGS_max_background_compactions =
    leveldb::Options().max_background_compactions;
+  FLAGS_compaction_style = leveldb::Options().compaction_style;
+  FLAGS_universal_size_ratio =
+    leveldb::Options().compaction_options_universal.size_ratio;
+  FLAGS_compaction_universal_min_merge_width =
+    leveldb::Options().compaction_options_universal.min_merge_width;
  // Compression test code above refers to FLAGS_block_size
  FLAGS_block_size = leveldb::Options().block_size;
  FLAGS_use_os_buffer = leveldb::EnvOptions().use_os_buffer;
@@ -2047,6 +2195,13 @@ int main(int argc, char** argv) {
    } else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk)
               == 1) {
      FLAGS_max_background_compactions = n;
+    } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
+      FLAGS_compaction_style = (leveldb::CompactionStyle)n;
+    } else if (sscanf(argv[i], "--universal_size_ratio=%d%c", &n, &junk) == 1) {
+      FLAGS_universal_size_ratio = n;
+    } else if (sscanf(argv[i], "--universal_min_merge_width=%d%c",
+               &n, &junk) == 1) {
+      FLAGS_compaction_universal_min_merge_width = n;
    } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
      FLAGS_cache_size = l;
    } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
@@ -2227,11 +2382,13 @@ int main(int argc, char** argv) {
    } else if (sscanf(argv[i], "--keys_per_multiget=%d%c",
               &n, &junk) == 1) {
      FLAGS_keys_per_multiget = n;
-    }  else if (sscanf(argv[i], "--bytes_per_sync=%ld%c", &l, &junk) == 1) {
+    } else if (sscanf(argv[i], "--bytes_per_sync=%ld%c", &l, &junk) == 1) {
      FLAGS_bytes_per_sync = l;
    } else if (sscanf(argv[i], "--filter_deletes=%d%c", &n, &junk)
               == 1 && (n == 0 || n ==1 )) {
      FLAGS_filter_deletes = n;
+    } else if (sscanf(argv[i], "--merge_operator=%s", buf) == 1) {
+      FLAGS_merge_operator = buf;
    } else {
      fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
      exit(1);
@@ -2,10 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.

-#include "db/db_impl.h"
-#include "db/filename.h"
+#include <algorithm>
 #include <string>
 #include <stdint.h>
+#include "db/db_impl.h"
+#include "db/filename.h"
 #include "db/version_set.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
@@ -66,4 +67,47 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
  return Status::OK();
 }

+Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
+  // First get sorted files in archive dir, then append sorted files from main
+  // dir to maintain sorted order
+
+  //  list wal files in archive dir.
+  Status s;
+  std::string archivedir = ArchivalDirectory(dbname_);
+  if (env_->FileExists(archivedir)) {
+    s = AppendSortedWalsOfType(archivedir, files, kArchivedLogFile);
+    if (!s.ok()) {
+      return s;
+    }
+  }
+  // list wal files in main db dir.
+  s = AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
+  if (!s.ok()) {
+    return s;
+  }
+  return s;
+}
+
+Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
+  Status s;
+  std::string archivedir = ArchivalDirectory(dbname_);
+  std::string files_not_deleted;
+  for (const auto& wal : files) {
+    /* Try deleting in archive dir. If fails, try deleting in main db dir.
+     * This is efficient because all except for very few wal files will be in
+     * archive. Checking for WalType is not much helpful because alive wal could
+       be archived now.
+     */
+    if (!env_->DeleteFile(archivedir + "/" + wal->Filename()).ok() &&
+        !env_->DeleteFile(dbname_ + "/" + wal->Filename()).ok()) {
+      files_not_deleted.append(wal->Filename());
+    }
+  }
+  if (!files_not_deleted.empty()) {
+    return Status::IOError("Deleted all requested files except: " +
+                           files_not_deleted);
+  }
+  return Status::OK();
+}
+
 }
@@ -23,10 +23,11 @@
 #include "db/memtable.h"
 #include "db/memtablelist.h"
 #include "db/merge_helper.h"
+#include "db/prefix_filter_iterator.h"
 #include "db/table_cache.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
-#include "db/transaction_log_iterator_impl.h"
+#include "db/transaction_log_impl.h"
 #include "leveldb/compaction_filter.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
@@ -76,6 +77,7 @@ struct DBImpl::CompactionState {
    uint64_t number;
    uint64_t file_size;
    InternalKey smallest, largest;
+    SequenceNumber smallest_seqno, largest_seqno;
  };
  std::vector<Output> outputs;
  std::list<uint64_t> allocated_file_numbers;
@@ -157,6 +159,10 @@ Options SanitizeOptions(const std::string& dbname,
  if (result.soft_rate_limit > result.hard_rate_limit) {
    result.soft_rate_limit = result.hard_rate_limit;
  }
+  if (result.compaction_filter &&
+      result.compaction_filter_factory->CreateCompactionFilter().get()) {
+    Log(result.info_log, "Both filter and factory specified. Using filter");
+  }
  return result;
 }

@@ -514,11 +520,11 @@ void DBImpl::DeleteObsoleteFiles() {
 void DBImpl::PurgeObsoleteWALFiles() {
  int64_t current_time;
  Status s = env_->GetCurrentTime(&current_time);
-  uint64_t now_micros = static_cast<uint64_t>(current_time);
+  uint64_t now_seconds = static_cast<uint64_t>(current_time);
  assert(s.ok());

  if (options_.WAL_ttl_seconds != ULONG_MAX && options_.WAL_ttl_seconds > 0) {
-    if (purge_wal_files_last_run_ + options_.WAL_ttl_seconds > now_micros) {
+    if (purge_wal_files_last_run_ + options_.WAL_ttl_seconds > now_seconds) {
      return;
    }
    std::vector<std::string> wal_files;
@@ -528,7 +534,7 @@ void DBImpl::PurgeObsoleteWALFiles() {
      uint64_t file_m_time;
      const std::string file_path = archival_dir + "/" + f;
      const Status s = env_->GetFileModificationTime(file_path, &file_m_time);
-      if (s.ok() && (now_micros - file_m_time > options_.WAL_ttl_seconds)) {
+      if (s.ok() && (now_seconds - file_m_time > options_.WAL_ttl_seconds)) {
        Status status = env_->DeleteFile(file_path);
        if (!status.ok()) {
          Log(options_.info_log,
@@ -538,7 +544,7 @@ void DBImpl::PurgeObsoleteWALFiles() {
      } // Ignore errors.
    }
  }
-  purge_wal_files_last_run_ = now_micros;
+  purge_wal_files_last_run_ = now_seconds;
 }

 // If externalTable is set, then apply recovered transactions
@@ -637,6 +643,8 @@ Status DBImpl::Recover(VersionEdit* edit, MemTable* external_table,
      } else {
        last_flushed_sequence_ = versions_->LastSequence();
      }
+      SetTickerCount(options_.statistics, SEQUENCE_NUMBER,
+                     versions_->LastSequence());
    }
  }

@@ -780,7 +788,8 @@ Status DBImpl::WriteLevel0TableForRecovery(MemTable* mem, VersionEdit* edit) {
  int level = 0;
  if (s.ok() && meta.file_size > 0) {
    edit->AddFile(level, meta.number, meta.file_size,
-                  meta.smallest, meta.largest);
+                  meta.smallest, meta.largest,
+                  meta.smallest_seqno, meta.largest_seqno);
  }

  CompactionStats stats;
@@ -854,11 +863,13 @@ Status DBImpl::WriteLevel0Table(std::vector<MemTable*> &mems, VersionEdit* edit,
    // insert files directly into higher levels because some other
    // threads could be concurrently producing compacted files for
    // that key range.
-    if (base != nullptr && options_.max_background_compactions <= 1) {
+    if (base != nullptr && options_.max_background_compactions <= 1 &&
+        options_.compaction_style == kCompactionStyleLevel) {
      level = base->PickLevelForMemTableOutput(min_user_key, max_user_key);
    }
    edit->AddFile(level, meta.number, meta.file_size,
-                  meta.smallest, meta.largest);
+                  meta.smallest, meta.largest,
+                  meta.smallest_seqno, meta.largest_seqno);
  }

  CompactionStats stats;
@@ -991,7 +1002,8 @@ void DBImpl::ReFitLevel(int level) {
    VersionEdit edit(NumberLevels());
    for (const auto& f : versions_->current()->files_[level]) {
      edit.DeleteFile(level, f->number);
-      edit.AddFile(to_level, f->number, f->file_size, f->smallest, f->largest);
+      edit.AddFile(to_level, f->number, f->file_size, f->smallest, f->largest,
+                   f->smallest_seqno, f->largest_seqno);
    }
    Log(options_.info_log, "Apply version edit:\n%s",
        edit.DebugString().data());
@@ -1034,34 +1046,24 @@ SequenceNumber DBImpl::GetLatestSequenceNumber() {
 Status DBImpl::GetUpdatesSince(SequenceNumber seq,
                               unique_ptr<TransactionLogIterator>* iter) {

-  //  Get All Log Files.
-  //  Sort Files
-  //  Get the first entry from each file.
+  if (seq > last_flushed_sequence_) {
+    return Status::IOError("Requested sequence not yet written in the db");
+  }
+  //  Get all sorted Wal Files.
  //  Do binary search and open files and find the seq number.

-  std::vector<LogFile> walFiles;
-  // list wal files in main db dir.
-  Status s = ListAllWALFiles(dbname_, &walFiles, kAliveLogFile);
+  std::unique_ptr<VectorLogPtr> wal_files(new VectorLogPtr);
+  Status s = GetSortedWalFiles(*wal_files);
  if (!s.ok()) {
    return s;
  }
-  //  list wal files in archive dir.
-  std::string archivedir = ArchivalDirectory(dbname_);
-  if (env_->FileExists(archivedir)) {
-    s = ListAllWALFiles(archivedir, &walFiles, kArchivedLogFile);
-    if (!s.ok()) {
-      return s;
-    }
-  }

-  if (walFiles.empty()) {
+  if (wal_files->empty()) {
    return Status::IOError(" NO WAL Files present in the db");
  }
  //  std::shared_ptr would have been useful here.

-  std::unique_ptr<std::vector<LogFile>> probableWALFiles(
-    new std::vector<LogFile>());
-  s = FindProbableWALFiles(&walFiles, probableWALFiles.get(), seq);
+  s = RetainProbableWalFiles(*wal_files, seq);
  if (!s.ok()) {
    return s;
  }
@@ -1070,90 +1072,61 @@ Status DBImpl::GetUpdatesSince(SequenceNumber seq,
                                   &options_,
                                   storage_options_,
                                   seq,
-                                   std::move(probableWALFiles),
+                                   std::move(wal_files),
                                   &last_flushed_sequence_));
  iter->get()->Next();
  return iter->get()->status();
 }

-Status DBImpl::FindProbableWALFiles(std::vector<LogFile>* const allLogs,
-                                    std::vector<LogFile>* const result,
-                                    const SequenceNumber target) {
-  assert(allLogs != nullptr);
-  assert(result != nullptr);
-
-  std::sort(allLogs->begin(), allLogs->end());
+Status DBImpl::RetainProbableWalFiles(VectorLogPtr& all_logs,
+                                      const SequenceNumber target) {
  long start = 0; // signed to avoid overflow when target is < first file.
-  long end = static_cast<long>(allLogs->size()) - 1;
+  long end = static_cast<long>(all_logs.size()) - 1;
  // Binary Search. avoid opening all files.
  while (end >= start) {
    long mid = start + (end - start) / 2;  // Avoid overflow.
-    WriteBatch batch;
-    Status s = ReadFirstRecord(allLogs->at(mid), &batch);
-    if (!s.ok()) {
-      if (CheckFileExistsAndEmpty(allLogs->at(mid))) {
-        allLogs->erase(allLogs->begin() + mid);
-        --end;
-        continue;
-      }
-      return s;
-    }
-    SequenceNumber currentSeqNum = WriteBatchInternal::Sequence(&batch);
-    if (currentSeqNum == target) {
-      start = mid;
+    SequenceNumber current_seq_num = all_logs.at(mid)->StartSequence();
+    if (current_seq_num == target) {
      end = mid;
      break;
-    } else if (currentSeqNum < target) {
+    } else if (current_seq_num < target) {
      start = mid + 1;
    } else {
      end = mid - 1;
    }
  }
-  size_t startIndex = std::max(0l, end); // end could be -ve.
-  for (size_t i = startIndex; i < allLogs->size(); ++i) {
-    result->push_back(allLogs->at(i));
-  }
-  if (result->empty()) {
-    return Status::IOError(
-        "No probable files. Check if the db contains log files");
-  }
+  size_t start_index = std::max(0l, end); // end could be -ve.
+  // The last wal file is always included
+  all_logs.erase(all_logs.begin(), all_logs.begin() + start_index);
  return Status::OK();
 }

-bool DBImpl::CheckFileExistsAndEmpty(const LogFile& file) {
-  if (file.type == kAliveLogFile) {
-    const std::string fname = LogFileName(dbname_, file.logNumber);
-    uint64_t file_size;
-    Status s = env_->GetFileSize(fname, &file_size);
-    if (s.ok() && file_size == 0) {
-      return true;
-    }
-  }
-  const std::string fname = ArchivedLogFileName(dbname_, file.logNumber);
+bool DBImpl::CheckWalFileExistsAndEmpty(const WalFileType type,
+                                        const uint64_t number) {
+  const std::string fname = (type == kAliveLogFile) ?
+    LogFileName(dbname_, number) : ArchivedLogFileName(dbname_, number);
  uint64_t file_size;
  Status s = env_->GetFileSize(fname, &file_size);
-  if (s.ok() && file_size == 0) {
-    return true;
-  }
-  return false;
+  return (s.ok() && (file_size == 0));
 }

-Status DBImpl::ReadFirstRecord(const LogFile& file, WriteBatch* const result) {
+Status DBImpl::ReadFirstRecord(const WalFileType type, const uint64_t number,
+                               WriteBatch* const result) {

-  if (file.type == kAliveLogFile) {
-    std::string fname = LogFileName(dbname_, file.logNumber);
+  if (type == kAliveLogFile) {
+    std::string fname = LogFileName(dbname_, number);
    Status status = ReadFirstLine(fname, result);
    if (!status.ok()) {
      //  check if the file got moved to archive.
-      std::string archivedFile = ArchivedLogFileName(dbname_, file.logNumber);
-      Status s = ReadFirstLine(archivedFile, result);
+      std::string archived_file = ArchivedLogFileName(dbname_, number);
+      Status s = ReadFirstLine(archived_file, result);
      if (!s.ok()) {
-        return Status::IOError("Log File Has been deleted");
+        return Status::IOError("Log File has been deleted");
      }
    }
    return Status::OK();
-  } else if (file.type == kArchivedLogFile) {
-    std::string fname = ArchivedLogFileName(dbname_, file.logNumber);
+  } else if (type == kArchivedLogFile) {
+    std::string fname = ArchivedLogFileName(dbname_, number);
    Status status = ReadFirstLine(fname, result);
    return status;
  }
@@ -1192,6 +1165,7 @@ Status DBImpl::ReadFirstLine(const std::string& fname,
                     0/*initial_offset*/);
  std::string scratch;
  Slice record;
+
  if (reader.ReadRecord(&record, &scratch) && status.ok()) {
    if (record.size() < 12) {
      reporter.Corruption(
@@ -1205,22 +1179,49 @@ Status DBImpl::ReadFirstLine(const std::string& fname,
  return Status::IOError("Error reading from file " + fname);
 }

-Status DBImpl::ListAllWALFiles(const std::string& path,
-                               std::vector<LogFile>* const logFiles,
-                               WalFileType logType) {
-  assert(logFiles != nullptr);
-  std::vector<std::string> allFiles;
-  const Status status = env_->GetChildren(path, &allFiles);
+struct CompareLogByPointer {
+  bool operator() (const unique_ptr<LogFile>& a,
+                   const unique_ptr<LogFile>& b) {
+    LogFileImpl* a_impl = dynamic_cast<LogFileImpl*>(a.get());
+    LogFileImpl* b_impl = dynamic_cast<LogFileImpl*>(b.get());
+    return *a_impl < *b_impl;
+  }
+};
+
+Status DBImpl::AppendSortedWalsOfType(const std::string& path,
+    VectorLogPtr& log_files, WalFileType log_type) {
+  std::vector<std::string> all_files;
+  const Status status = env_->GetChildren(path, &all_files);
  if (!status.ok()) {
    return status;
  }
-  for (const auto& f : allFiles) {
+  log_files.reserve(log_files.size() + all_files.size());
+  for (const auto& f : all_files) {
    uint64_t number;
    FileType type;
    if (ParseFileName(f, &number, &type) && type == kLogFile){
-      logFiles->push_back(LogFile(number, logType));
+
+      WriteBatch batch;
+      Status s = ReadFirstRecord(log_type, number, &batch);
+      if (!s.ok()) {
+        if (CheckWalFileExistsAndEmpty(log_type, number)) {
+          continue;
+        }
+        return s;
+      }
+
+      uint64_t size_bytes;
+      s = env_->GetFileSize(LogFileName(path, number), &size_bytes);
+      if (!s.ok()) {
+        return s;
+      }
+
+      log_files.push_back(std::move(unique_ptr<LogFile>(new LogFileImpl(
+        number, log_type, WriteBatchInternal::Sequence(&batch), size_bytes))));
    }
  }
+  CompareLogByPointer compare_log_files;
+  std::sort(log_files.begin(), log_files.end(), compare_log_files);
  return status;
 }

@@ -1233,13 +1234,17 @@ void DBImpl::TEST_CompactRange(int level, const Slice* begin,const Slice* end) {
  manual.level = level;
  manual.done = false;
  manual.in_progress = false;
-  if (begin == nullptr) {
+  // For universal compaction, we enforce every manual compaction to compact
+  // all files.
+  if (begin == nullptr ||
+      options_.compaction_style == kCompactionStyleUniversal) {
    manual.begin = nullptr;
  } else {
    begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek);
    manual.begin = &begin_storage;
  }
-  if (end == nullptr) {
+  if (end == nullptr ||
+      options_.compaction_style == kCompactionStyleUniversal) {
    manual.end = nullptr;
  } else {
    end_storage = InternalKey(*end, 0, static_cast<ValueType>(0));
@@ -1452,7 +1457,8 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
    FileMetaData* f = c->input(0, 0);
    c->edit()->DeleteFile(c->level(), f->number);
    c->edit()->AddFile(c->level() + 1, f->number, f->file_size,
-                       f->smallest, f->largest);
+                       f->smallest, f->largest,
+                       f->smallest_seqno, f->largest_seqno);
    status = versions_->LogAndApply(c->edit(), &mutex_);
    VersionSet::LevelSummaryStorage tmp;
    Log(options_.info_log, "Moved #%lld to level-%d %lld bytes %s: %s\n",
@@ -1492,6 +1498,18 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress,
    if (!status.ok()) {
      m->done = true;
    }
+    // For universal compaction:
+    //   Because universal compaction always happens at level 0, so one
+    //   compaction will pick up all overlapped files. No files will be
+    //   filtered out due to size limit and left for a successive compaction.
+    //   So we can safely conclude the current compaction.
+    //
+    //   Also note that, if we don't stop here, then the current compaction
+    //   writes a new file back to level 0, which will be used in successive
+    //   compaction. Hence the manual compaction will never finish.
+    if (options_.compaction_style == kCompactionStyleUniversal) {
+      m->done = true;
+    }
    if (!m->done) {
      // We only compacted part of the requested range.  Update *m
      // to the range that is left to be compacted.
@@ -1564,6 +1582,7 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
  out.number = file_number;
  out.smallest.Clear();
  out.largest.Clear();
+  out.smallest_seqno = out.largest_seqno = 0;
  compact->outputs.push_back(out);

  // Make the output file
@@ -1574,10 +1593,10 @@ Status DBImpl::OpenCompactionOutputFile(CompactionState* compact) {
    // Over-estimate slightly so we don't end up just barely crossing
    // the threshold.
    compact->outfile->SetPreallocationBlockSize(
-      1.1 * versions_->MaxFileSizeForLevel(compact->compaction->level() + 1));
+      1.1 * versions_->MaxFileSizeForLevel(compact->compaction->output_level()));

    compact->builder.reset(new TableBuilder(options_, compact->outfile.get(),
-                                            compact->compaction->level() + 1));
+                                            compact->compaction->output_level()));
  }
  return s;
 }
@@ -1668,8 +1687,10 @@ Status DBImpl::InstallCompactionResults(CompactionState* compact) {
  for (size_t i = 0; i < compact->outputs.size(); i++) {
    const CompactionState::Output& out = compact->outputs[i];
    compact->compaction->edit()->AddFile(
-        level + 1,
-        out.number, out.file_size, out.smallest, out.largest);
+        (options_.compaction_style == kCompactionStyleUniversal) ?
+          level : level + 1,
+        out.number, out.file_size, out.smallest, out.largest,
+        out.smallest_seqno, out.largest_seqno);
  }
  return versions_->LogAndApply(compact->compaction->edit(), &mutex_);
 }
@@ -1736,14 +1757,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
  }

  // Is this compaction producing files at the bottommost level?
-  bool bottommost_level = true;
-  for (int i = compact->compaction->level() + 2;
-       i < versions_->NumberLevels(); i++) {
-    if (versions_->NumLevelFiles(i) > 0) {
-      bottommost_level = false;
-      break;
-    }
-  }
+  bool bottommost_level = compact->compaction->BottomMostLevel();

  // Allocate the output file numbers before we release the lock
  AllocateCompactionOutputFileNumbers(compact);
@@ -1763,9 +1777,16 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
  SequenceNumber visible_in_snapshot = kMaxSequenceNumber;
  std::string compaction_filter_value;
  std::vector<char> delete_key; // for compaction filter
-  MergeHelper merge(user_comparator(), options_.merge_operator,
+  MergeHelper merge(user_comparator(), options_.merge_operator.get(),
                    options_.info_log.get(),
                    false /* internal key corruption is expected */);
+  auto compaction_filter = options_.compaction_filter;
+  std::unique_ptr<CompactionFilter> compaction_filter_from_factory = nullptr;
+  if (!compaction_filter) {
+    compaction_filter_from_factory = std::move(
+        options_.compaction_filter_factory->CreateCompactionFilter());
+    compaction_filter = compaction_filter_from_factory.get();
+  }
  for (; input->Valid() && !shutting_down_.Acquire_Load(); ) {
    // Prioritize immutable compaction work
    if (imm_.imm_flush_needed.NoBarrier_Load() != nullptr) {
@@ -1812,7 +1833,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
        visible_in_snapshot = kMaxSequenceNumber;

        // apply the compaction filter to the first occurrence of the user key
-        if (options_.compaction_filter &&
+        if (compaction_filter &&
            ikey.type == kTypeValue &&
            (visible_at_tip || ikey.sequence > latest_snapshot)) {
          // If the user has specified a compaction filter and the sequence
@@ -1823,7 +1844,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
          bool value_changed = false;
          compaction_filter_value.clear();
          bool to_delete =
-            options_.compaction_filter->Filter(compact->compaction->level(),
+            compaction_filter->Filter(compact->compaction->level(),
                                               ikey.user_key, value,
                                               &compaction_filter_value,
                                               &value_changed);
@@ -1885,7 +1906,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
        // object to minimize change to the existing flow. Turn out this
        // logic could also be nicely re-used for memtable flush purge
        // optimization in BuildTable.
-        merge.MergeUntil(input.get(), prev_snapshot, bottommost_level);
+        merge.MergeUntil(input.get(), prev_snapshot, bottommost_level,
+                         options_.statistics);
        current_entry_is_merging = true;
        if (merge.IsSuccess()) {
          // Successfully found Put/Delete/(end-of-key-range) while merging
@@ -1950,7 +1972,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
        // If this is the bottommost level (no files in lower levels)
        // and the earliest snapshot is larger than this seqno
        // then we can squash the seqno to zero.
-        if (bottommost_level && ikey.sequence < earliest_snapshot &&
+        if (options_.compaction_style == kCompactionStyleLevel &&
+            bottommost_level && ikey.sequence < earliest_snapshot &&
            ikey.type != kTypeMerge) {
          assert(ikey.type != kTypeDeletion);
          // make a copy because updating in place would cause problems
@@ -1970,11 +1993,19 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
            break;
          }
        }
+
+        SequenceNumber seqno = GetInternalKeySeqno(newkey);
        if (compact->builder->NumEntries() == 0) {
          compact->current_output()->smallest.DecodeFrom(newkey);
+          compact->current_output()->smallest_seqno = seqno;
+        } else {
+          compact->current_output()->smallest_seqno =
+            std::min(compact->current_output()->smallest_seqno, seqno);
        }
        compact->current_output()->largest.DecodeFrom(newkey);
        compact->builder->Add(newkey, value);
+        compact->current_output()->largest_seqno =
+          std::max(compact->current_output()->largest_seqno, seqno);

        // Close output file if it is big enough
        if (compact->builder->FileSize() >=
@@ -2070,7 +2101,7 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
      versions_->LevelSummary(&tmp),
      (stats.bytes_readn + stats.bytes_readnp1 + stats.bytes_written) /
          (double) stats.micros,
-      compact->compaction->level() + 1,
+      compact->compaction->output_level(),
      stats.files_in_leveln, stats.files_in_levelnp1, stats.files_out_levelnp1,
      stats.bytes_readn / 1048576.0,
      stats.bytes_readnp1 / 1048576.0,
@@ -2301,12 +2332,19 @@ bool DBImpl::KeyMayExist(const ReadOptions& options,

 Iterator* DBImpl::NewIterator(const ReadOptions& options) {
  SequenceNumber latest_snapshot;
-  Iterator* internal_iter = NewInternalIterator(options, &latest_snapshot);
-  return NewDBIterator(
-      &dbname_, env_, options_, user_comparator(), internal_iter,
-      (options.snapshot != nullptr
-       ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
-       : latest_snapshot));
+  Iterator* iter = NewInternalIterator(options, &latest_snapshot);
+  iter = NewDBIterator(
+             &dbname_, env_, options_, user_comparator(), iter,
+             (options.snapshot != nullptr
+              ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
+              : latest_snapshot));
+  if (options.prefix) {
+    // use extra wrapper to exclude any keys from the results which
+    // don't begin with the prefix
+    iter = new PrefixFilterIterator(iter, *options.prefix,
+                                    options_.prefix_extractor);
+  }
+  return iter;
 }

 const Snapshot* DBImpl::GetSnapshot() {
@@ -2401,6 +2439,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
          // have succeeded in memtable but Status reports error for all writes.
          throw std::runtime_error("In memory WriteBatch corruption!");
        }
+        RecordTick(options_.statistics, SEQUENCE_NUMBER, my_batch_count);
        versions_->SetLastSequence(last_sequence);
        last_flushed_sequence_ = current_sequence;
      }
@@ -10,15 +10,15 @@
 #include <set>
 #include <vector>
 #include "db/dbformat.h"
-#include "db/log_file.h"
 #include "db/log_writer.h"
 #include "db/snapshot.h"
 #include "leveldb/db.h"
 #include "leveldb/env.h"
+#include "leveldb/memtablerep.h"
+#include "leveldb/transaction_log.h"
 #include "port/port.h"
 #include "util/stats_logger.h"
 #include "memtablelist.h"
-#include "leveldb/memtablerep.h"

 #ifdef USE_SCRIBE
 #include "scribe/scribe_logger.h"
@@ -73,6 +73,8 @@ class DBImpl : public DB {
  virtual Status EnableFileDeletions();
  virtual Status GetLiveFiles(std::vector<std::string>&,
                              uint64_t* manifest_file_size);
+  virtual Status GetSortedWalFiles(VectorLogPtr& files);
+  virtual Status DeleteWalFiles(const VectorLogPtr& files);
  virtual SequenceNumber GetLatestSequenceNumber();
  virtual Status GetUpdatesSince(SequenceNumber seq_number,
                                 unique_ptr<TransactionLogIterator>* iter);
@@ -183,7 +185,7 @@ class DBImpl : public DB {
  void MaybeScheduleCompaction();
  static void BGWork(void* db);
  void BackgroundCall();
-  Status BackgroundCompaction(bool* madeProgress, DeletionState& deletion_state);
+  Status BackgroundCompaction(bool* madeProgress,DeletionState& deletion_state);
  void CleanupCompaction(CompactionState* compact);
  Status DoCompactionWork(CompactionState* compact);

@@ -208,19 +210,21 @@ class DBImpl : public DB {

  void PurgeObsoleteWALFiles();

-  Status ListAllWALFiles(const std::string& path,
-                         std::vector<LogFile>* logFiles,
-                         WalFileType type);
+  Status AppendSortedWalsOfType(const std::string& path,
+                                VectorLogPtr& log_files,
+                                WalFileType type);

-  //  Find's all the log files which contain updates with seq no.
-  //  Greater Than or Equal to the requested SequenceNumber
-  Status FindProbableWALFiles(std::vector<LogFile>* const allLogs,
-                              std::vector<LogFile>* const result,
-                              const SequenceNumber target);
+  // Requires: all_logs should be sorted with earliest log file first
+  // Retains all log files in all_logs which contain updates with seq no.
+  // Greater Than or Equal to the requested SequenceNumber.
+  Status RetainProbableWalFiles(VectorLogPtr& all_logs,
+                                const SequenceNumber target);
  //  return true if
-  bool CheckFileExistsAndEmpty(const LogFile& file);
+  bool CheckWalFileExistsAndEmpty(const WalFileType type,
+                                  const uint64_t number);

-  Status ReadFirstRecord(const LogFile& file, WriteBatch* const result);
+  Status ReadFirstRecord(const WalFileType type, const uint64_t number,
+                         WriteBatch* const result);

  Status ReadFirstLine(const std::string& fname, WriteBatch* const batch);

@@ -57,7 +57,7 @@ class DBIter: public Iterator {
        env_(env),
        logger_(options.info_log),
        user_comparator_(cmp),
-        user_merge_operator_(options.merge_operator),
+        user_merge_operator_(options.merge_operator.get()),
        iter_(iter),
        sequence_(s),
        direction_(kForward),
@@ -212,10 +212,10 @@ void DBIter::FindNextUserEntry(bool skipping) {
            SaveKey(ikey.user_key, &saved_key_);
            current_entry_is_merged_ = true;
            valid_ = true;
-            // Go to a different state machine
-            MergeValuesNewToOld();
-            // TODO: what if !iter_->Valid()
+            MergeValuesNewToOld();  // Go to a different state machine
            return;
+          case kTypeLogData:
+            assert(false);
            break;
        }
      }
@@ -232,7 +232,11 @@ void DBIter::FindNextUserEntry(bool skipping) {
 // POST: saved_value_ has the merged value for the user key
 //       iter_ points to the next entry (or invalid)
 void DBIter::MergeValuesNewToOld() {
-  // TODO: Is there a way to unite with MergeHelper or other similar code?
+  if (!user_merge_operator_) {
+    Log(logger_, "Options::merge_operator is null.");
+    throw std::logic_error("DBIter::MergeValuesNewToOld() with"
+                           " Options::merge_operator null");
+  }

  // Start the merge process by pushing the first operand
  std::deque<std::string> operands;
@@ -263,8 +267,8 @@ void DBIter::MergeValuesNewToOld() {
      // final result in saved_value_. We are done!
      // ignore corruption if there is any.
      const Slice value = iter_->value();
-      user_merge_operator_->Merge(ikey.user_key, &value, operands,
-                                  &saved_value_, logger_.get());
+      user_merge_operator_->FullMerge(ikey.user_key, &value, operands,
+                                      &saved_value_, logger_.get());
      // iter_ is positioned after put
      iter_->Next();
      return;
@@ -297,8 +301,8 @@ void DBIter::MergeValuesNewToOld() {
  // a deletion marker.
  // feed null as the existing value to the merge operator, such that
  // client can differentiate this scenario and do things accordingly.
-  user_merge_operator_->Merge(ikey.user_key, nullptr, operands,
-                              &saved_value_, logger_.get());
+  user_merge_operator_->FullMerge(saved_key_, nullptr, operands,
+                                  &saved_value_, logger_.get());
 }

 void DBIter::Prev() {
@@ -30,6 +30,11 @@ class DBStatistics: public Statistics {
    return allTickers_[tickerType].getCount();
  }

+  virtual void setTickerCount(Tickers tickerType, uint64_t count) {
+    assert(tickerType < TICKER_ENUM_MAX);
+    allTickers_[tickerType].setTickerCount(count);
+  }
+
  virtual void recordTick(Tickers tickerType, uint64_t count) {
    assert(tickerType < TICKER_ENUM_MAX);
    allTickers_[tickerType].recordTick(count);
@@ -6,6 +6,7 @@
 #include "db/dbformat.h"
 #include "port/port.h"
 #include "util/coding.h"
+#include "include/leveldb/perf_context.h"

 namespace leveldb {

@@ -53,6 +54,7 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
  //    decreasing sequence number
  //    decreasing type (though sequence# should be enough to disambiguate)
  int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
+  perf_context.user_key_comparison_count++;
  if (r == 0) {
    const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
    const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
@@ -25,7 +25,8 @@ class InternalKey;
 enum ValueType {
  kTypeDeletion = 0x0,
  kTypeValue = 0x1,
-  kTypeMerge = 0x2
+  kTypeMerge = 0x2,
+  kTypeLogData = 0x3
 };
 // kValueTypeForSeek defines the ValueType that should be passed when
 // constructing a ParsedInternalKey object for seeking to a particular
@@ -173,6 +174,15 @@ inline void UpdateInternalKey(char* internal_key,
  EncodeFixed64(seqtype, newval);
 }

+// Get the sequence number from the internal key
+inline uint64_t GetInternalKeySeqno(const Slice& internal_key) {
+  const size_t n = internal_key.size();
+  assert(n >= 8);
+  uint64_t num = DecodeFixed64(internal_key.data() + n - 8);
+  return num >> 8;
+}
+
+
 // A helper class useful for DBImpl::Get()
 class LookupKey {
 public:
@@ -46,10 +46,13 @@ extern Status WriteStringToFileSync(Env* env, const Slice& data,
 static std::string MakeFileName(const std::string& name, uint64_t number,
                                const char* suffix) {
  char buf[100];
-  snprintf(buf, sizeof(buf), "/%06llu.%s",
+  snprintf(buf, sizeof(buf), "%06llu.%s",
           static_cast<unsigned long long>(number),
           suffix);
-  return name + buf;
+  if (name.empty()) {
+    return buf;
+  }
+  return name + "/" + buf;
 }

 std::string LogFileName(const std::string& name, uint64_t number) {
@@ -1,48 +0,0 @@
-// Copyright 2008-present Facebook. All Rights Reserved.
-
-#ifndef STORAGE_LEVELDB_DB_LOG_FILE_H_
-#define STORAGE_LEVELDB_DB_LOG_FILE_H_
-
-namespace leveldb {
-
-enum  WalFileType {
-  kArchivedLogFile = 0,
-  kAliveLogFile = 1
-} ;
-
-class LogFile {
-
- public:
-  uint64_t logNumber;
-  WalFileType type;
-
-  LogFile(uint64_t logNum,WalFileType logType) :
-    logNumber(logNum),
-    type(logType) {}
-
-  LogFile(const LogFile& that) {
-   logNumber = that.logNumber;
-   type = that.type;
-  }
-
-  bool operator < (const LogFile& that) const {
-    return logNumber < that.logNumber;
-  }
-
-  std::string ToString() const {
-    char response[100];
-    const char* typeOfLog;
-    if (type == kAliveLogFile) {
-      typeOfLog = "Alive Log";
-    } else {
-      typeOfLog = "Archived Log";
-    }
-    sprintf(response,
-            "LogNumber  : %ld LogType : %s",
-            logNumber,
-            typeOfLog);
-    return std::string(response);
-  }
-};
-}  //  namespace leveldb
-#endif  // STORAGE_LEVELDB_DB_LOG_FILE_H_
@@ -139,7 +139,7 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
  assert(operands != nullptr);

  bool merge_in_progress = s->IsMergeInProgress();
-  auto merge_operator = options.merge_operator;
+  auto merge_operator = options.merge_operator.get();
  auto logger = options.info_log;
  std::string merge_result;

@@ -167,8 +167,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
          *s = Status::OK();
          if (merge_in_progress) {
            assert(merge_operator);
-            if (!merge_operator->Merge(key.user_key(), &v, *operands,
-                                       value, logger.get())) {
+            if (!merge_operator->FullMerge(key.user_key(), &v, *operands,
+                                           value, logger.get())) {
+              RecordTick(options.statistics, NUMBER_MERGE_FAILURES);
              *s = Status::Corruption("Error: Could not perform merge.");
            }
          } else {
@@ -180,8 +181,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
          if (merge_in_progress) {
            assert(merge_operator);
            *s = Status::OK();
-            if (!merge_operator->Merge(key.user_key(), nullptr, *operands,
-                                       value, logger.get())) {
+            if (!merge_operator->FullMerge(key.user_key(), nullptr, *operands,
+                                           value, logger.get())) {
+              RecordTick(options.statistics, NUMBER_MERGE_FAILURES);
              *s = Status::Corruption("Error: Could not perform merge.");
            }
          } else {
@@ -209,6 +211,9 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
          }
          break;
        }
+        case kTypeLogData:
+          assert(false);
+          break;
      }
    } else {
      // exit loop if user key does not match
@@ -8,10 +8,10 @@
 #include <string>
 #include <memory>
 #include <deque>
-#include "leveldb/db.h"
 #include "db/dbformat.h"
 #include "db/skiplist.h"
 #include "db/version_set.h"
+#include "leveldb/db.h"
 #include "leveldb/memtablerep.h"
 #include "util/arena_impl.h"

@@ -21,17 +21,15 @@ using std::list;

 // Increase reference count on all underling memtables
 void MemTableList::RefAll() {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end() ; ++it) {
-    (*it)->Ref();
+  for (auto &memtable : memlist_) {
+    memtable->Ref();
  }
 }

 // Drop reference count on all underling memtables
 void MemTableList::UnrefAll() {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end() ; ++it) {
-    (*it)->Unref();
+  for (auto &memtable : memlist_) {
+    memtable->Unref();
  }
 }

@@ -53,8 +51,7 @@ bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {

 // Returns the memtables that need to be flushed.
 void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
-  for (list<MemTable*>::reverse_iterator it = memlist_.rbegin();
-       it != memlist_.rend(); it++) {
+  for (auto it = memlist_.rbegin(); it != memlist_.rend(); it++) {
    MemTable* m = *it;
    if (!m->flush_in_progress_) {
      assert(!m->flush_completed_);
@@ -184,9 +181,8 @@ void MemTableList::Add(MemTable* m) {
 // Returns an estimate of the number of bytes of data in use.
 size_t MemTableList::ApproximateMemoryUsage() {
  size_t size = 0;
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end(); ++it) {
-    size += (*it)->ApproximateMemoryUsage();
+  for (auto &memtable : memlist_) {
+    size += memtable->ApproximateMemoryUsage();
  }
  return size;
 }
@@ -197,9 +193,8 @@ size_t MemTableList::ApproximateMemoryUsage() {
 bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s,
                       std::deque<std::string>* operands,
                       const Options& options) {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end(); ++it) {
-    if ((*it)->Get(key, value, s, operands, options)) {
+  for (auto &memtable : memlist_) {
+    if (memtable->Get(key, value, s, operands, options)) {
      return true;
    }
  }
@@ -207,9 +202,8 @@ bool MemTableList::Get(const LookupKey& key, std::string* value, Status* s,
 }

 void MemTableList::GetMemTables(std::vector<MemTable*>* output) {
-  for (list<MemTable*>::iterator it = memlist_.begin();
-       it != memlist_.end(); ++it) {
-    output->push_back(*it);
+  for (auto &memtable : memlist_) {
+    output->push_back(memtable);
  }
 }

@@ -15,7 +15,7 @@ namespace leveldb {
 //       operands_ stores the list of merge operands encountered while merging.
 //       keys_[i] corresponds to operands_[i] for each i.
 void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
-                             bool at_bottom) {
+                             bool at_bottom, shared_ptr<Statistics> stats) {
  // Get a copy of the internal key, before it's invalidated by iter->Next()
  // Also maintain the list of merge operands seen.
  keys_.clear();
@@ -67,9 +67,9 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
      //   => store result in operands_.back() (and update keys_.back())
      //   => change the entry type to kTypeValue for keys_.back()
      // We are done! Return a success if the merge passes.
-      success_ = user_merge_operator_->Merge(ikey.user_key, nullptr,
-                                             operands_, &merge_result,
-                                             logger_);
+      success_ = user_merge_operator_->FullMerge(ikey.user_key, nullptr,
+                                                 operands_, &merge_result,
+                                                 logger_);

      // We store the result in keys_.back() and operands_.back()
      // if nothing went wrong (i.e.: no operand corruption on disk)
@@ -79,6 +79,8 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
        UpdateInternalKey(&key[0], key.size(),
                          orig_ikey.sequence, orig_ikey.type);
        swap(operands_.back(), merge_result);
+      } else {
+        RecordTick(stats, NUMBER_MERGE_FAILURES);
      }

      // move iter to the next entry (before doing anything else)
@@ -93,9 +95,9 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
      //   => change the entry type to kTypeValue for keys_.back()
      // We are done! Success!
      const Slice value = iter->value();
-      success_ = user_merge_operator_->Merge(ikey.user_key, &value,
-                                             operands_, &merge_result,
-                                             logger_);
+      success_ = user_merge_operator_->FullMerge(ikey.user_key, &value,
+                                                 operands_, &merge_result,
+                                                 logger_);

      // We store the result in keys_.back() and operands_.back()
      // if nothing went wrong (i.e.: no operand corruption on disk)
@@ -105,6 +107,8 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
        UpdateInternalKey(&key[0], key.size(),
                          orig_ikey.sequence, orig_ikey.type);
        swap(operands_.back(), merge_result);
+      } else {
+        RecordTick(stats, NUMBER_MERGE_FAILURES);
      }

      // move iter to the next entry
@@ -166,9 +170,9 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
    assert(kTypeMerge == orig_ikey.type);
    assert(operands_.size() >= 1);
    assert(operands_.size() == keys_.size());
-    success_ = user_merge_operator_->Merge(ikey.user_key, nullptr,
-                                           operands_, &merge_result,
-                                           logger_);
+    success_ = user_merge_operator_->FullMerge(ikey.user_key, nullptr,
+                                               operands_, &merge_result,
+                                               logger_);

    if (success_) {
      std::string& key = keys_.back();  // The original key encountered
@@ -179,6 +183,7 @@ void MergeHelper::MergeUntil(Iterator* iter, SequenceNumber stop_before,
      // The final value() is always stored in operands_.back()
      swap(operands_.back(),merge_result);
    } else {
+      RecordTick(stats, NUMBER_MERGE_FAILURES);
      // Do nothing if not success_. Leave keys() and operands() as they are.
    }
  }
@@ -3,6 +3,7 @@

 #include "db/dbformat.h"
 #include "leveldb/slice.h"
+#include "leveldb/statistics.h"
 #include <string>
 #include <deque>

@@ -40,7 +41,7 @@ class MergeHelper {
  // at_bottom:   (IN) true if the iterator covers the bottem level, which means
  //                   we could reach the start of the history of this user key.
  void MergeUntil(Iterator* iter, SequenceNumber stop_before = 0,
-                  bool at_bottom = false);
+                  bool at_bottom = false, shared_ptr<Statistics> stats=nullptr);

  // Query the merge result
  // These are valid until the next MergeUntil call
@@ -12,7 +12,7 @@ namespace leveldb {
 // Given a "real" merge from the library, call the user's
 // associative merge function one-by-one on each of the operands.
 // NOTE: It is assumed that the client's merge-operator will handle any errors.
-bool AssociativeMergeOperator::Merge(
+bool AssociativeMergeOperator::FullMerge(
    const Slice& key,
    const Slice* existing_value,
    const std::deque<std::string>& operand_list,
@@ -16,14 +16,13 @@
 using namespace std;
 using namespace leveldb;

-auto mergeOperator = MergeOperators::CreateUInt64AddOperator();

 std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false) {
  DB* db;
  StackableDB* sdb;
  Options options;
  options.create_if_missing = true;
-  options.merge_operator = mergeOperator.get();
+  options.merge_operator = MergeOperators::CreateUInt64AddOperator();
  Status s;
  DestroyDB(dbname, Options());
  if (ttl) {
@@ -0,0 +1,12 @@
+#include "include/leveldb/perf_context.h"
+
+
+namespace leveldb {
+
+void PerfContext::Reset() {
+  user_key_comparison_count = 0;
+}
+
+__thread PerfContext perf_context;
+
+}
@@ -0,0 +1,81 @@
+#include <iostream>
+
+#include "leveldb/db.h"
+#include "leveldb/perf_context.h"
+#include "util/testharness.h"
+
+
+namespace leveldb {
+
+// Path to the database on file system
+const std::string kDbName = test::TmpDir() + "/perf_context_test";
+
+std::shared_ptr<DB> OpenDb() {
+    DB* db;
+    Options options;
+    options.create_if_missing = true;
+    options.write_buffer_size = 1000000000;     // give it a big memtable
+    Status s = DB::Open(options, kDbName,  &db);
+    ASSERT_OK(s);
+    return std::shared_ptr<DB>(db);
+}
+
+class PerfContextTest { };
+
+int kTotalKeys = 100;
+
+TEST(PerfContextTest, KeyComparisonCount) {
+
+  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
+
+  auto db = OpenDb();
+
+  WriteOptions write_options;
+  ReadOptions read_options;
+
+  uint64_t total_user_key_comparison_get = 0;
+  uint64_t total_user_key_comparison_put = 0;
+  uint64_t max_user_key_comparison_get = 0;
+
+  std::cout << "Inserting " << kTotalKeys << " key/value pairs\n...\n";
+
+  for (int i = 0; i < kTotalKeys; ++i) {
+    std::string key = "k" + std::to_string(i);
+    std::string value = "v" + std::to_string(i);
+
+    perf_context.Reset();
+    db->Put(write_options, key, value);
+    total_user_key_comparison_put += perf_context.user_key_comparison_count;
+
+    perf_context.Reset();
+    db->Get(read_options, key, &value);
+    total_user_key_comparison_get += perf_context.user_key_comparison_count;
+    max_user_key_comparison_get =
+      std::max(max_user_key_comparison_get,
+               perf_context.user_key_comparison_count);
+  }
+
+  std::cout << "total user key comparison get: "
+            << total_user_key_comparison_get << "\n"
+            << "total user key comparison put: "
+            << total_user_key_comparison_put << "\n"
+            << "max user key comparison get: "
+            << max_user_key_comparison_get << "\n"
+            << "avg user key comparison get:"
+            << total_user_key_comparison_get/kTotalKeys << "\n";
+
+}
+
+
+}
+
+
+int main(int argc, char** argv) {
+
+  if (argc > 1) {
+    leveldb::kTotalKeys = std::stoi(argv[1]);
+  }
+
+  leveldb::test::RunAllTests();
+  return 0;
+}
@@ -0,0 +1,76 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Wrap an underlying iterator, but exclude any results not starting
+// with a given prefix.  Seeking to keys not beginning with the prefix
+// is invalid, and SeekToLast is not implemented (that would be
+// non-trivial), but otherwise this iterator will behave just like the
+// underlying iterator would if there happened to be no non-matching
+// keys in the dataset.
+
+#ifndef STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
+#define STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
+
+#include "leveldb/iterator.h"
+
+namespace leveldb {
+
+class PrefixFilterIterator : public Iterator {
+ private:
+  Iterator* iter_;
+  const Slice &prefix_;
+  const SliceTransform *prefix_extractor_;
+  Status status_;
+
+ public:
+  PrefixFilterIterator(Iterator* iter,
+                       const Slice &prefix,
+                       const SliceTransform* prefix_extractor)
+                             : iter_(iter), prefix_(prefix),
+                               prefix_extractor_(prefix_extractor),
+                               status_(Status::OK()) {
+    if (prefix_extractor == nullptr) {
+      status_ = Status::InvalidArgument("A prefix filter may not be used "
+                                        "unless a function is also defined "
+                                        "for extracting prefixes");
+    } else if (!prefix_extractor_->InRange(prefix)) {
+      status_ = Status::InvalidArgument("Must provide a slice for prefix which"
+                                        "is a prefix for some key");
+    }
+  }
+  ~PrefixFilterIterator() {
+    delete iter_;
+  }
+  Slice key() const { return iter_->key(); }
+  Slice value() const { return iter_->value(); }
+  Status status() const {
+    if (!status_.ok()) {
+      return status_;
+    }
+    return iter_->status();
+  }
+  void Next() { iter_->Next(); }
+  void Prev() { iter_->Prev(); }
+  void Seek(const Slice& k) {
+    if (prefix_extractor_->Transform(k) == prefix_) {
+      iter_->Seek(k);
+    } else {
+      status_ = Status::InvalidArgument("Seek must begin with target prefix");
+    }
+  }
+  void SeekToFirst() {
+    Seek(prefix_);
+  }
+  void SeekToLast() {
+    status_ = Status::NotSupported("SeekToLast is incompatible with prefixes");
+  }
+  bool Valid() const {
+    return (status_.ok() && iter_->Valid() &&
+            prefix_extractor_->Transform(iter_->key()) == prefix_);
+  }
+};
+
+}  // namespace leveldb
+
+#endif
@@ -88,6 +88,7 @@ class Repairer {
 private:
  struct TableInfo {
    FileMetaData meta;
+    SequenceNumber min_sequence;
    SequenceNumber max_sequence;
  };

@@ -264,6 +265,7 @@ class Repairer {
          ReadOptions(), storage_options_, t->meta.number, t->meta.file_size);
      bool empty = true;
      ParsedInternalKey parsed;
+      t->min_sequence = 0;
      t->max_sequence = 0;
      for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
        Slice key = iter->key();
@@ -280,6 +282,9 @@ class Repairer {
          t->meta.smallest.DecodeFrom(key);
        }
        t->meta.largest.DecodeFrom(key);
+        if (parsed.sequence < t->min_sequence) {
+          t->min_sequence = parsed.sequence;
+        }
        if (parsed.sequence > t->max_sequence) {
          t->max_sequence = parsed.sequence;
        }
@@ -320,7 +325,8 @@ class Repairer {
      // TODO(opt): separate out into multiple levels
      const TableInfo& t = tables_[i];
      edit_->AddFile(0, t.meta.number, t.meta.file_size,
-                    t.meta.smallest, t.meta.largest);
+                    t.meta.smallest, t.meta.largest,
+                    t.min_sequence, t.max_sequence);
    }

    //fprintf(stderr, "NewDescriptor:\n%s\n", edit_.DebugString().c_str());
@@ -1,6 +1,5 @@
-#include "db/transaction_log_iterator_impl.h"
+#include "db/transaction_log_impl.h"
 #include "db/write_batch_internal.h"
-#include "db/filename.h"

 namespace leveldb {

@@ -8,39 +7,39 @@ TransactionLogIteratorImpl::TransactionLogIteratorImpl(
                           const std::string& dbname,
                           const Options* options,
                           const EnvOptions& soptions,
-                           SequenceNumber& seq,
-                           std::unique_ptr<std::vector<LogFile>> files,
+                           const SequenceNumber seq,
+                           std::unique_ptr<VectorLogPtr> files,
                           SequenceNumber const * const lastFlushedSequence) :
-  dbname_(dbname),
-  options_(options),
-  soptions_(soptions),
-  startingSequenceNumber_(seq),
-  files_(std::move(files)),
-  started_(false),
-  isValid_(false),
-  currentFileIndex_(0),
-  lastFlushedSequence_(lastFlushedSequence) {
+    dbname_(dbname),
+    options_(options),
+    soptions_(soptions),
+    startingSequenceNumber_(seq),
+    files_(std::move(files)),
+    started_(false),
+    isValid_(false),
+    currentFileIndex_(0),
+    lastFlushedSequence_(lastFlushedSequence) {
+  assert(startingSequenceNumber_ <= *lastFlushedSequence_);
  assert(files_.get() != nullptr);
-  assert(lastFlushedSequence_);

  reporter_.env = options_->env;
  reporter_.info_log = options_->info_log.get();
 }

 Status TransactionLogIteratorImpl::OpenLogFile(
-  const LogFile& logFile,
+  const LogFile* logFile,
  unique_ptr<SequentialFile>* file) {
  Env* env = options_->env;
-  if (logFile.type == kArchivedLogFile) {
-    std::string fname = ArchivedLogFileName(dbname_, logFile.logNumber);
+  if (logFile->Type() == kArchivedLogFile) {
+    std::string fname = ArchivedLogFileName(dbname_, logFile->LogNumber());
    return env->NewSequentialFile(fname, file, soptions_);
  } else {
-    std::string fname = LogFileName(dbname_, logFile.logNumber);
+    std::string fname = LogFileName(dbname_, logFile->LogNumber());
    Status status = env->NewSequentialFile(fname, file, soptions_);
    if (!status.ok()) {
      //  If cannot open file in DB directory.
      //  Try the archive dir, as it could have moved in the meanwhile.
-      fname = ArchivedLogFileName(dbname_, logFile.logNumber);
+      fname = ArchivedLogFileName(dbname_, logFile->LogNumber());
      status = env->NewSequentialFile(fname, file, soptions_);
      if (!status.ok()) {
        return Status::IOError(" Requested file not present in the dir");
@@ -67,7 +66,7 @@ bool TransactionLogIteratorImpl::Valid() {
 }

 void TransactionLogIteratorImpl::Next() {
-  LogFile currentLogFile = files_.get()->at(currentFileIndex_);
+  LogFile* currentLogFile = files_.get()->at(currentFileIndex_).get();

 //  First seek to the given seqNo. in the current file.
  std::string scratch;
@@ -129,7 +128,7 @@ void TransactionLogIteratorImpl::Next() {
    if (openNextFile) {
      if (currentFileIndex_ < files_.get()->size() - 1) {
        ++currentFileIndex_;
-        Status status = OpenLogReader(files_.get()->at(currentFileIndex_));
+        Status status =OpenLogReader(files_.get()->at(currentFileIndex_).get());
        if (!status.ok()) {
          isValid_ = false;
          currentStatus_ = status;
@@ -157,7 +156,7 @@ void TransactionLogIteratorImpl::UpdateCurrentWriteBatch(const Slice& record) {
  currentStatus_ = Status::OK();
 }

-Status TransactionLogIteratorImpl::OpenLogReader(const LogFile& logFile) {
+Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
  unique_ptr<SequentialFile> file;
  Status status = OpenLogFile(logFile, &file);
  if (!status.ok()) {
@@ -7,9 +7,9 @@
 #include "leveldb/env.h"
 #include "leveldb/options.h"
 #include "leveldb/types.h"
-#include "leveldb/transaction_log_iterator.h"
-#include "db/log_file.h"
+#include "leveldb/transaction_log.h"
 #include "db/log_reader.h"
+#include "db/filename.h"

 namespace leveldb {

@@ -21,13 +21,45 @@ struct LogReporter : public log::Reader::Reporter {
  }
 };

+class LogFileImpl : public LogFile {
+ public:
+  LogFileImpl(uint64_t logNum, WalFileType logType, SequenceNumber startSeq,
+              uint64_t sizeBytes) :
+    logNumber_(logNum),
+    type_(logType),
+    startSequence_(startSeq),
+    sizeFileBytes_(sizeBytes) {
+  }
+
+  std::string Filename() const { return LogFileName("", logNumber_); }
+
+  uint64_t LogNumber() const { return logNumber_; }
+
+  WalFileType Type() const { return type_; }
+
+  SequenceNumber StartSequence() const { return startSequence_; }
+
+  uint64_t SizeFileBytes() const { return sizeFileBytes_; }
+
+  bool operator < (const LogFile& that) const {
+    return LogNumber() < that.LogNumber();
+  }
+
+ private:
+  uint64_t logNumber_;
+  WalFileType type_;
+  SequenceNumber startSequence_;
+  uint64_t sizeFileBytes_;
+
+};
+
 class TransactionLogIteratorImpl : public TransactionLogIterator {
 public:
  TransactionLogIteratorImpl(const std::string& dbname,
                             const Options* options,
                             const EnvOptions& soptions,
-                             SequenceNumber& seqNum,
-                             std::unique_ptr<std::vector<LogFile>> files,
+                             const SequenceNumber seqNum,
+                             std::unique_ptr<VectorLogPtr> files,
                             SequenceNumber const * const lastFlushedSequence);

  virtual bool Valid();
@@ -42,22 +74,22 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
  const std::string& dbname_;
  const Options* options_;
  const EnvOptions& soptions_;
-  const uint64_t startingSequenceNumber_;
-  std::unique_ptr<std::vector<LogFile>> files_;
+  const SequenceNumber startingSequenceNumber_;
+  std::unique_ptr<VectorLogPtr> files_;
  bool started_;
  bool isValid_;  // not valid when it starts of.
  Status currentStatus_;
  size_t currentFileIndex_;
  std::unique_ptr<WriteBatch> currentBatch_;
  unique_ptr<log::Reader> currentLogReader_;
-  Status OpenLogFile(const LogFile& logFile, unique_ptr<SequentialFile>* file);
+  Status OpenLogFile(const LogFile* logFile, unique_ptr<SequentialFile>* file);
  LogReporter reporter_;
  SequenceNumber const * const lastFlushedSequence_;
  // represents the sequence number being read currently.
  SequenceNumber currentSequence_;

  void UpdateCurrentWriteBatch(const Slice& record);
-  Status OpenLogReader(const LogFile& file);
+  Status OpenLogReader(const LogFile* file);
 };


@@ -20,7 +20,10 @@ enum Tag {
  kDeletedFile          = 6,
  kNewFile              = 7,
  // 8 was used for large value refs
-  kPrevLogNumber        = 9
+  kPrevLogNumber        = 9,
+
+  // these are new formats divergent from open source leveldb
+  kNewFile2             = 100  // store smallest & largest seqno
 };

 void VersionEdit::Clear() {
@@ -76,12 +79,14 @@ void VersionEdit::EncodeTo(std::string* dst) const {

  for (size_t i = 0; i < new_files_.size(); i++) {
    const FileMetaData& f = new_files_[i].second;
-    PutVarint32(dst, kNewFile);
+    PutVarint32(dst, kNewFile2);
    PutVarint32(dst, new_files_[i].first);  // level
    PutVarint64(dst, f.number);
    PutVarint64(dst, f.file_size);
    PutLengthPrefixedSlice(dst, f.smallest.Encode());
    PutLengthPrefixedSlice(dst, f.largest.Encode());
+    PutVarint64(dst, f.smallest_seqno);
+    PutVarint64(dst, f.largest_seqno);
  }
 }

@@ -201,6 +206,22 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
        }
        break;

+      case kNewFile2:
+        if (GetLevel(&input, &level, &msg) &&
+            GetVarint64(&input, &f.number) &&
+            GetVarint64(&input, &f.file_size) &&
+            GetInternalKey(&input, &f.smallest) &&
+            GetInternalKey(&input, &f.largest) &&
+            GetVarint64(&input, &f.smallest_seqno) &&
+            GetVarint64(&input, &f.largest_seqno) ) {
+          new_files_.push_back(std::make_pair(level, f));
+        } else {
+          if (!msg) {
+            msg = "new-file2 entry";
+          }
+        }
+        break;
+
      default:
        msg = "unknown tag";
        break;
@@ -218,7 +239,7 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
  return result;
 }

-std::string VersionEdit::DebugString() const {
+std::string VersionEdit::DebugString(bool hex_key) const {
  std::string r;
  r.append("VersionEdit {");
  if (has_comparator_) {
@@ -245,7 +266,7 @@ std::string VersionEdit::DebugString() const {
    r.append("\n  CompactPointer: ");
    AppendNumberTo(&r, compact_pointers_[i].first);
    r.append(" ");
-    r.append(compact_pointers_[i].second.DebugString());
+    r.append(compact_pointers_[i].second.DebugString(hex_key));
  }
  for (DeletedFileSet::const_iterator iter = deleted_files_.begin();
       iter != deleted_files_.end();
@@ -264,9 +285,9 @@ std::string VersionEdit::DebugString() const {
    r.append(" ");
    AppendNumberTo(&r, f.file_size);
    r.append(" ");
-    r.append(f.smallest.DebugString());
+    r.append(f.smallest.DebugString(hex_key));
    r.append(" .. ");
-    r.append(f.largest.DebugString());
+    r.append(f.largest.DebugString(hex_key));
  }
  r.append("\n}\n");
  return r;
@@ -22,6 +22,8 @@ struct FileMetaData {
  InternalKey smallest;       // Smallest internal key served by table
  InternalKey largest;        // Largest internal key served by table
  bool being_compacted;       // Is this file undergoing compaction?
+  SequenceNumber smallest_seqno;// The smallest seqno in this file
+  SequenceNumber largest_seqno; // The largest seqno in this file

  FileMetaData() : refs(0), allowed_seeks(1 << 30), file_size(0),
                   being_compacted(false) { }
@@ -67,12 +69,17 @@ class VersionEdit {
  void AddFile(int level, uint64_t file,
               uint64_t file_size,
               const InternalKey& smallest,
-               const InternalKey& largest) {
+               const InternalKey& largest,
+               const SequenceNumber& smallest_seqno,
+               const SequenceNumber& largest_seqno) {
    FileMetaData f;
    f.number = file;
    f.file_size = file_size;
    f.smallest = smallest;
    f.largest = largest;
+    f.smallest_seqno = smallest_seqno;
+    f.largest_seqno = largest_seqno;
+    assert(smallest_seqno <= largest_seqno);
    new_files_.push_back(std::make_pair(level, f));
  }

@@ -89,7 +96,7 @@ class VersionEdit {
  void EncodeTo(std::string* dst) const;
  Status DecodeFrom(const Slice& src);

-  std::string DebugString() const;
+  std::string DebugString(bool hex_key = false) const;

 private:
  friend class VersionSet;
@@ -27,7 +27,9 @@ TEST(VersionEditTest, EncodeDecode) {
    TestEncodeDecode(edit);
    edit.AddFile(3, kBig + 300 + i, kBig + 400 + i,
                 InternalKey("foo", kBig + 500 + i, kTypeValue),
-                 InternalKey("zoo", kBig + 600 + i, kTypeDeletion));
+                 InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
+                 kBig + 500 + i,
+                 kBig + 600 + i);
    edit.DeleteFile(4, kBig + 700 + i);
    edit.SetCompactPointer(i, InternalKey("x", kBig + 900 + i, kTypeValue));
  }
@@ -5,6 +5,7 @@
 #include "db/version_set.h"

 #include <algorithm>
+#include <climits>
 #include <stdio.h>
 #include "db/filename.h"
 #include "db/log_reader.h"
@@ -22,8 +23,8 @@

 namespace leveldb {

-static int64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
-  int64_t sum = 0;
+static uint64_t TotalFileSize(const std::vector<FileMetaData*>& files) {
+  uint64_t sum = 0;
  for (size_t i = 0; i < files.size() && files[i]; i++) {
    sum += files[i]->file_size;
  }
@@ -244,6 +245,7 @@ struct Saver {
  std::deque<std::string>* merge_operands;  // the merge operations encountered
  Logger* logger;
  bool didIO;    // did we do any disk io?
+  shared_ptr<Statistics> statistics;
 };
 }

@@ -284,8 +286,9 @@ static bool SaveValue(void* arg, const Slice& ikey, const Slice& v, bool didIO){
          } else if (kMerge == s->state) {
            assert(s->merge_operator != nullptr);
            s->state = kFound;
-            if (!s->merge_operator->Merge(s->user_key, &v, *ops,
-                                          s->value, s->logger)) {
+            if (!s->merge_operator->FullMerge(s->user_key, &v, *ops,
+                                              s->value, s->logger)) {
+              RecordTick(s->statistics, NUMBER_MERGE_FAILURES);
              s->state = kCorrupt;
            }
          } else {
@@ -298,8 +301,9 @@ static bool SaveValue(void* arg, const Slice& ikey, const Slice& v, bool didIO){
            s->state = kDeleted;
          } else if (kMerge == s->state) {
            s->state = kFound;
-            if (!s->merge_operator->Merge(s->user_key, nullptr, *ops,
-                                          s->value, s->logger)) {
+            if (!s->merge_operator->FullMerge(s->user_key, nullptr, *ops,
+                                              s->value, s->logger)) {
+              RecordTick(s->statistics, NUMBER_MERGE_FAILURES);
              s->state = kCorrupt;
            }
          } else {
@@ -326,6 +330,10 @@ static bool SaveValue(void* arg, const Slice& ikey, const Slice& v, bool didIO){
            }
          }
          return true;
+
+        case kTypeLogData:
+          assert(false);
+          break;
      }
    }
  }
@@ -338,6 +346,14 @@ static bool SaveValue(void* arg, const Slice& ikey, const Slice& v, bool didIO){
 static bool NewestFirst(FileMetaData* a, FileMetaData* b) {
  return a->number > b->number;
 }
+static bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b) {
+  if (a->smallest_seqno > b->smallest_seqno) {
+    assert(a->largest_seqno > b->largest_seqno);
+    return true;
+  }
+  assert(a->largest_seqno <= b->largest_seqno);
+  return false;
+}

 Version::Version(VersionSet* vset, uint64_t version_number)
    : vset_(vset), next_(this), prev_(this), refs_(0),
@@ -365,7 +381,7 @@ void Version::Get(const ReadOptions& options,
  Slice user_key = k.user_key();
  const Comparator* ucmp = vset_->icmp_.user_comparator();

-  auto merge_operator = db_options.merge_operator;
+  auto merge_operator = db_options.merge_operator.get();
  auto logger = db_options.info_log;

  assert(status->ok() || status->IsMergeInProgress());
@@ -382,6 +398,7 @@ void Version::Get(const ReadOptions& options,
  saver.merge_operands = operands;
  saver.logger = logger.get();
  saver.didIO = false;
+  saver.statistics = db_options.statistics;

  stats->seek_file = nullptr;
  stats->seek_file_level = -1;
@@ -434,7 +451,11 @@ void Version::Get(const ReadOptions& options,
    if (important_files.empty()) continue;

    if (level == 0) {
-      std::sort(important_files.begin(), important_files.end(), NewestFirst);
+      if (vset_->options_->compaction_style == kCompactionStyleUniversal) {
+        std::sort(important_files.begin(), important_files.end(), NewestFirstBySeqNo);
+      } else {
+        std::sort(important_files.begin(), important_files.end(), NewestFirst);
+      }
    } else {
      // Sanity check to make sure that the files are correctly sorted
 #ifndef NDEBUG
@@ -500,10 +521,11 @@ void Version::Get(const ReadOptions& options,
  if (kMerge == saver.state) {
    // merge_operands are in saver and we hit the beginning of the key history
    // do a final merge of nullptr and operands;
-    if (merge_operator->Merge(user_key, nullptr, *saver.merge_operands,
-                              value, logger.get())) {
+    if (merge_operator->FullMerge(user_key, nullptr, *saver.merge_operands,
+                                  value, logger.get())) {
      *status = Status::OK();
    } else {
+      RecordTick(db_options.statistics, NUMBER_MERGE_FAILURES);
      *status = Status::Corruption("could not perform end-of-key merge for ",
                                   user_key);
    }
@@ -565,7 +587,7 @@ int Version::PickLevelForMemTableOutput(
        break;
      }
      GetOverlappingInputs(level + 2, &start, &limit, &overlaps);
-      const int64_t sum = TotalFileSize(overlaps);
+      const uint64_t sum = TotalFileSize(overlaps);
      if (sum > vset_->MaxGrandParentOverlapBytes(level)) {
        break;
      }
@@ -1109,7 +1131,10 @@ void VersionSet::Init(int num_levels) {
  int target_file_size_multiplier = options_->target_file_size_multiplier;
  int max_bytes_multiplier = options_->max_bytes_for_level_multiplier;
  for (int i = 0; i < num_levels; i++) {
-    if (i > 1) {
+    if (i == 0 && options_->compaction_style == kCompactionStyleUniversal) {
+      max_file_size_[i] = ULLONG_MAX;
+      level_max_bytes_[i] = options_->max_bytes_for_level_base;
+    } else if (i > 1) {
      max_file_size_[i] = max_file_size_[i-1] * target_file_size_multiplier;
      level_max_bytes_[i] = level_max_bytes_[i-1] * max_bytes_multiplier *
        options_->max_bytes_for_level_multiplier_additional[i-1];
@@ -1496,7 +1521,7 @@ Status VersionSet::DumpManifest(Options& options, std::string& dscname,
      // Write out each individual edit
      if (verbose) {
        printf("*************************Edit[%d] = %s\n",
-                count, edit.DebugString().c_str());
+                count, edit.DebugString(hex).c_str());
      }
      count++;

@@ -1656,17 +1681,32 @@ void VersionSet::Finalize(Version* v,
  }
 }

-// a static compator used to sort files based on their size
-static bool compareSize(const VersionSet::Fsize& first,
+// A static compator used to sort files based on their size
+// In normal mode: descending size
+static bool compareSizeDescending(const VersionSet::Fsize& first,
  const VersionSet::Fsize& second) {
  return (first.file->file_size > second.file->file_size);
 }
+// A static compator used to sort files based on their seqno
+// In universal style : descending seqno
+static bool compareSeqnoDescending(const VersionSet::Fsize& first,
+  const VersionSet::Fsize& second) {
+  if (first.file->smallest_seqno > second.file->smallest_seqno) {
+    assert(first.file->largest_seqno > second.file->largest_seqno);
+    return true;
+  }
+  assert(first.file->largest_seqno <= second.file->largest_seqno);
+  return false;
+}

 // sort all files in level1 to level(n-1) based on file size
 void VersionSet::UpdateFilesBySize(Version* v) {

  // No need to sort the highest level because it is never compacted.
-  for (int level = 0; level < NumberLevels()-1; level++) {
+  int max_level = (options_->compaction_style == kCompactionStyleUniversal) ?
+                  NumberLevels() : NumberLevels() - 1;
+
+  for (int level = 0; level < max_level; level++) {

    const std::vector<FileMetaData*>& files = v->files_[level];
    std::vector<int>& files_by_size = v->files_by_size_[level];
@@ -1680,12 +1720,18 @@ void VersionSet::UpdateFilesBySize(Version* v) {
    }

    // sort the top number_of_files_to_sort_ based on file size
-    int num = Version::number_of_files_to_sort_;
-    if (num > (int)temp.size()) {
-      num = temp.size();
+    if (options_->compaction_style == kCompactionStyleUniversal) {
+      int num = temp.size();
+      std::partial_sort(temp.begin(),  temp.begin() + num,
+                        temp.end(), compareSeqnoDescending);
+    } else {
+      int num = Version::number_of_files_to_sort_;
+      if (num > (int)temp.size()) {
+        num = temp.size();
+      }
+      std::partial_sort(temp.begin(),  temp.begin() + num,
+                        temp.end(), compareSizeDescending);
    }
-    std::partial_sort(temp.begin(),  temp.begin() + num,
-                      temp.end(), compareSize);
    assert(temp.size() == files.size());

    // initialize files_by_size_
@@ -1718,7 +1764,8 @@ Status VersionSet::WriteSnapshot(log::Writer* log) {
    const std::vector<FileMetaData*>& files = current_->files_[level];
    for (size_t i = 0; i < files.size(); i++) {
      const FileMetaData* f = files[i];
-      edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest);
+      edit.AddFile(level, f->number, f->file_size, f->smallest, f->largest,
+                   f->smallest_seqno, f->largest_seqno);
    }
  }

@@ -1762,6 +1809,23 @@ const char* VersionSet::LevelDataSizeSummary(
  return scratch->buffer;
 }

+const char* VersionSet::LevelFileSummary(
+    FileSummaryStorage* scratch, int level) const {
+  int len = snprintf(scratch->buffer, sizeof(scratch->buffer), "files_size[");
+  for (unsigned int i = 0; i < current_->files_[level].size(); i++) {
+    FileMetaData* f = current_->files_[level][i];
+    int sz = sizeof(scratch->buffer) - len;
+    int ret = snprintf(scratch->buffer + len, sz, "#%ld(seq=%ld,sz=%ld,%d) ",
+                       f->number, f->smallest_seqno,
+                       f->file_size, f->being_compacted);
+    if (ret < 0 || ret >= sz)
+      break;
+    len += ret;
+  }
+  snprintf(scratch->buffer + len, sizeof(scratch->buffer) - len, "]");
+  return scratch->buffer;
+}
+
 // Opens the mainfest file and reads all records
 // till it finds the record we are looking for.
 bool VersionSet::ManifestContains(const std::string& record) const {
@@ -1867,14 +1931,14 @@ int64_t VersionSet::NumLevelBytes(int level) const {
 }

 int64_t VersionSet::MaxNextLevelOverlappingBytes() {
-  int64_t result = 0;
+  uint64_t result = 0;
  std::vector<FileMetaData*> overlaps;
  for (int level = 1; level < NumberLevels() - 1; level++) {
    for (size_t i = 0; i < current_->files_[level].size(); i++) {
      const FileMetaData* f = current_->files_[level][i];
      current_->GetOverlappingInputs(level+1, &f->smallest, &f->largest,
                                     &overlaps);
-      const int64_t sum = TotalFileSize(overlaps);
+      const uint64_t sum = TotalFileSize(overlaps);
      if (sum > result) {
        result = sum;
      }
@@ -1970,13 +2034,13 @@ uint64_t VersionSet::MaxFileSizeForLevel(int level) {
  return max_file_size_[level];
 }

-int64_t VersionSet::ExpandedCompactionByteSizeLimit(int level) {
+uint64_t VersionSet::ExpandedCompactionByteSizeLimit(int level) {
  uint64_t result = MaxFileSizeForLevel(level);
  result *= options_->expanded_compaction_factor;
  return result;
 }

-int64_t VersionSet::MaxGrandParentOverlapBytes(int level) {
+uint64_t VersionSet::MaxGrandParentOverlapBytes(int level) {
  uint64_t result = MaxFileSizeForLevel(level);
  result *= options_->max_grandparent_overlap_factor;
  return result;
@@ -2059,6 +2123,183 @@ void VersionSet::SizeBeingCompacted(std::vector<uint64_t>& sizes) {
  }
 }

+Compaction* VersionSet::PickCompactionUniversal(int level, double score) {
+  assert (level == 0);
+
+  // percentage flexibilty while comparing file sizes
+  uint64_t ratio = options_->compaction_options_universal.size_ratio;
+  unsigned int min_merge_width =
+    options_->compaction_options_universal.min_merge_width;
+  unsigned int max_merge_width =
+    options_->compaction_options_universal.max_merge_width;
+
+  if ((current_->files_[level].size() <=
+      (unsigned int)options_->level0_file_num_compaction_trigger)) {
+    Log(options_->info_log, "Universal: nothing to do\n");
+    return nullptr;
+  }
+  VersionSet::FileSummaryStorage tmp;
+  Log(options_->info_log, "Universal: candidate files(%lu): %s\n",
+      current_->files_[level].size(),
+      LevelFileSummary(&tmp, 0));
+
+  Compaction* c = nullptr;
+  c = new Compaction(level, level, MaxFileSizeForLevel(level),
+                     LLONG_MAX, NumberLevels());
+  c->score_ = score;
+
+  // The files are sorted from newest first to oldest last.
+  std::vector<int>& file_by_time = current_->files_by_size_[level];
+  FileMetaData* f = nullptr;
+  bool done = false;
+  assert(file_by_time.size() == current_->files_[level].size());
+
+  unsigned int max_files_to_compact = std::min(max_merge_width, UINT_MAX);
+
+  // Make two pass. The first pass considers a candidate file
+  // only if it is smaller than the total size accumulated so far.
+  // The second pass does not look at the slope of the
+  // file-size  curve to decide what to pick for compaction.
+  for (int iter = 0; !done && iter < 2; iter++) {
+
+    for (unsigned int loop = 0; loop < file_by_time.size(); ) {
+
+      // Skip files that are already being compacted
+      for (f = nullptr; loop < file_by_time.size(); loop++) {
+        int index = file_by_time[loop];
+        f = current_->files_[level][index];
+
+        if (!f->being_compacted) {
+          break;
+        }
+        Log(options_->info_log, "Universal: file %ld[%d] being compacted, skipping",
+            f->number, loop);
+        f = nullptr;
+      }
+
+      // This file is not being compacted. Consider it as the
+      // first candidate to be compacted.
+      unsigned int candidate_count = 1;
+      uint64_t candidate_size =  f != nullptr? f->file_size : 0;
+      if (f != nullptr) {
+        Log(options_->info_log, "Universal: Possible candidate file %ld[%d] %s.",
+            f->number, loop, iter == 0? "" : "forced ");
+      }
+
+      // Check if the suceeding files need compaction.
+      for (unsigned int i = loop+1;
+           candidate_count < max_files_to_compact && i < file_by_time.size();
+           i++) {
+        int index = file_by_time[i];
+        FileMetaData* f = current_->files_[level][index];
+        if (f->being_compacted) {
+          break;
+        }
+        // If this is the first iteration, then we pick files if the
+        // total candidate file size (increased by the specified ratio)
+        // is still larger than the next candidate file.
+        if (iter == 0) {
+          uint64_t sz = (candidate_size * (100 + ratio)) /100;
+          if (sz < f->file_size) {
+            break;
+          }
+        }
+        candidate_count++;
+        candidate_size += f->file_size;
+      }
+
+      // Found a series of consecutive files that need compaction.
+      if (candidate_count >= (unsigned int)min_merge_width) {
+        for (unsigned int i = loop; i < loop + candidate_count; i++) {
+          int index = file_by_time[i];
+          FileMetaData* f = current_->files_[level][index];
+          c->inputs_[0].push_back(f);
+          Log(options_->info_log, "Universal: Picking file %ld[%d] with size %ld %s",
+              f->number, i, f->file_size,
+              (iter == 0 ? "" : "forced"));
+        }
+        done = true;
+        break;
+      } else {
+        for (unsigned int i = loop;
+             i < loop + candidate_count && i < file_by_time.size(); i++) {
+         int index = file_by_time[i];
+         FileMetaData* f = current_->files_[level][index];
+         Log(options_->info_log, "Universal: Skipping file %ld[%d] with size %ld %d %s",
+             f->number, i, f->file_size, f->being_compacted,
+              (iter == 0 ? "" : "forced"));
+        }
+      }
+      loop += candidate_count;
+    }
+    assert(done || c->inputs_[0].size() == 0);
+
+    // If we are unable to find a normal compaction run and we are still
+    // above the compaction threshold, iterate again to pick compaction
+    // candidates, this time without considering their size differences.
+    if (!done) {
+      int files_not_in_compaction = 0;
+      for (unsigned int i = 0; i < current_->files_[level].size(); i++) {
+        f = current_->files_[level][i];
+        if (!f->being_compacted) {
+          files_not_in_compaction++;
+        }
+      }
+      int expected_num_files = files_not_in_compaction +
+                compactions_in_progress_[level].size();
+      if (expected_num_files <=
+          options_->level0_file_num_compaction_trigger + 1) {
+        done = true;     // nothing more to do
+      } else {
+        max_files_to_compact = std::min((int)max_merge_width,
+          expected_num_files - options_->level0_file_num_compaction_trigger);
+        Log(options_->info_log, "Universal: second loop with maxfiles %d",
+            max_files_to_compact);
+      }
+    }
+  }
+  if (c->inputs_[0].size() <= 1) {
+    Log(options_->info_log, "Universal: only %ld files, nothing to do.\n",
+        c->inputs_[0].size());
+    delete c;
+    return nullptr;
+  }
+
+  // validate that all the chosen files are non overlapping in time
+  FileMetaData* newerfile __attribute__((unused)) = nullptr;
+  for (unsigned int i = 0; i < c->inputs_[0].size(); i++) {
+    FileMetaData* f = c->inputs_[0][i];
+    assert (f->smallest_seqno <= f->largest_seqno);
+    assert(newerfile == nullptr ||
+           newerfile->smallest_seqno > f->largest_seqno);
+    newerfile = f;
+  }
+
+  // Is the earliest file part of this compaction?
+  int last_index = file_by_time[file_by_time.size()-1];
+  FileMetaData* last_file = current_->files_[level][last_index];
+  if (c->inputs_[0][c->inputs_[0].size()-1] == last_file) {
+    c->bottommost_level_ = true;
+  }
+
+  // update statistics
+  if (options_->statistics != nullptr) {
+    options_->statistics->measureTime(NUM_FILES_IN_SINGLE_COMPACTION,
+                                      c->inputs_[0].size());
+  }
+
+  c->input_version_ = current_;
+  c->input_version_->Ref();
+
+  // mark all the files that are being compacted
+  c->MarkFilesBeingCompacted(true);
+
+  // remember this currently undergoing compaction
+  compactions_in_progress_[level].insert(c);
+
+  return c;
+}
+
 Compaction* VersionSet::PickCompactionBySize(int level, double score) {
  Compaction* c = nullptr;

@@ -2072,7 +2313,7 @@ Compaction* VersionSet::PickCompactionBySize(int level, double score) {

  assert(level >= 0);
  assert(level+1 < NumberLevels());
-  c = new Compaction(level, MaxFileSizeForLevel(level+1),
+  c = new Compaction(level, level+1, MaxFileSizeForLevel(level+1),
      MaxGrandParentOverlapBytes(level), NumberLevels());
  c->score_ = score;

@@ -2142,6 +2383,13 @@ Compaction* VersionSet::PickCompaction() {
  current_->vset_->SizeBeingCompacted(size_being_compacted);
  Finalize(current_, size_being_compacted);

+  // In universal style of compaction, compact L0 files back into L0.
+  if (options_->compaction_style ==  kCompactionStyleUniversal) {
+    int level = 0;
+    c = PickCompactionUniversal(level, current_->compaction_score_[level]);
+    return c;
+  }
+
  // We prefer compactions triggered by too much data in a level over
  // the compactions triggered by seeks.
  //
@@ -2169,9 +2417,9 @@ Compaction* VersionSet::PickCompaction() {
    // Only allow one level 0 compaction at a time.
    // Do not pick this file if its parents at level+1 are being compacted.
    if (level != 0 || compactions_in_progress_[0].empty()) {
-      if (!ParentRangeInCompaction(&f->smallest, &f->largest, level,
-                                   &parent_index)) {
-        c = new Compaction(level, MaxFileSizeForLevel(level+1),
+      if(!ParentRangeInCompaction(&f->smallest, &f->largest, level,
+                                  &parent_index)) {
+        c = new Compaction(level, level+1, MaxFileSizeForLevel(level+1),
                MaxGrandParentOverlapBytes(level), NumberLevels(), true);
        c->inputs_[0].push_back(f);
        c->parent_index_ = parent_index;
@@ -2212,13 +2460,15 @@ Compaction* VersionSet::PickCompaction() {
    assert(!c->inputs_[0].empty());
  }

-
  // Setup "level+1" files (inputs_[1])
  SetupOtherInputs(c);

  // mark all the files that are being compacted
  c->MarkFilesBeingCompacted(true);

+  // Is this compaction creating a file at the bottommost level
+  c->SetupBottomMostLevel(false);
+
  // remember this currently undergoing compaction
  compactions_in_progress_[level].insert(c);

@@ -2331,10 +2581,10 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
    std::vector<FileMetaData*> expanded0;
    current_->GetOverlappingInputs(level, &all_start, &all_limit, &expanded0,
                                   c->base_index_, nullptr);
-    const int64_t inputs0_size = TotalFileSize(c->inputs_[0]);
-    const int64_t inputs1_size = TotalFileSize(c->inputs_[1]);
-    const int64_t expanded0_size = TotalFileSize(expanded0);
-    int64_t limit = ExpandedCompactionByteSizeLimit(level);
+    const uint64_t inputs0_size = TotalFileSize(c->inputs_[0]);
+    const uint64_t inputs1_size = TotalFileSize(c->inputs_[1]);
+    const uint64_t expanded0_size = TotalFileSize(expanded0);
+    uint64_t limit = ExpandedCompactionByteSizeLimit(level);
    if (expanded0.size() > c->inputs_[0].size() &&
        inputs1_size + expanded0_size < limit &&
        !FilesInCompaction(expanded0) &&
@@ -2392,6 +2642,13 @@ Compaction* VersionSet::CompactRange(
    const InternalKey* begin,
    const InternalKey* end) {
  std::vector<FileMetaData*> inputs;
+
+  // All files are 'overlapping' in universal style compaction.
+  // We have to compact the entire range in one shot.
+  if (options_->compaction_style == kCompactionStyleUniversal) {
+    begin = nullptr;
+    end = nullptr;
+  }
  current_->GetOverlappingInputs(level, begin, end, &inputs);
  if (inputs.empty()) {
    return nullptr;
@@ -2414,7 +2671,10 @@ Compaction* VersionSet::CompactRange(
      }
    }
  }
-  Compaction* c = new Compaction(level, MaxFileSizeForLevel(level+1),
+  int out_level = (options_->compaction_style == kCompactionStyleUniversal) ?
+                  level : level+1;
+
+  Compaction* c = new Compaction(level, out_level, MaxFileSizeForLevel(out_level),
    MaxGrandParentOverlapBytes(level), NumberLevels());

  c->inputs_[0] = inputs;
@@ -2432,13 +2692,17 @@ Compaction* VersionSet::CompactRange(
  // upon other files because manual compactions are processed when
  // the system has a max of 1 background compaction thread.
  c->MarkFilesBeingCompacted(true);
+
+  // Is this compaction creating a file at the bottommost level
+  c->SetupBottomMostLevel(true);
  return c;
 }

-Compaction::Compaction(int level, uint64_t target_file_size,
+Compaction::Compaction(int level, int out_level, uint64_t target_file_size,
  uint64_t max_grandparent_overlap_bytes, int number_levels,
  bool seek_compaction)
    : level_(level),
+      out_level_(out_level),
      max_output_file_size_(target_file_size),
      maxGrandParentOverlapBytes_(max_grandparent_overlap_bytes),
      input_version_(nullptr),
@@ -2450,6 +2714,7 @@ Compaction::Compaction(int level, uint64_t target_file_size,
      base_index_(-1),
      parent_index_(-1),
      score_(0),
+      bottommost_level_(false),
      level_ptrs_(std::vector<size_t>(number_levels)) {
  edit_ = new VersionEdit(number_levels_);
  for (int i = 0; i < number_levels_; i++) {
@@ -2482,6 +2747,10 @@ void Compaction::AddInputDeletions(VersionEdit* edit) {
 }

 bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
+  if (input_version_->vset_->options_->compaction_style ==
+      kCompactionStyleUniversal) {
+    return bottommost_level_;
+  }
  // Maybe use binary search to find right entry instead of linear search?
  const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
  for (int lvl = level_ + 2; lvl < number_levels_; lvl++) {
@@ -2540,6 +2809,31 @@ void Compaction::MarkFilesBeingCompacted(bool value) {
  }
 }

+// Is this compaction producing files at the bottommost level?
+void Compaction::SetupBottomMostLevel(bool isManual) {
+  if (input_version_->vset_->options_->compaction_style  ==
+         kCompactionStyleUniversal) {
+    // If universal compaction style is used and manual
+    // compaction is occuring, then we are guaranteed that
+    // all files will be picked in a single compaction
+    // run. We can safely set bottommost_level_ = true.
+    // If it is not manual compaction, then bottommost_level_
+    // is already set when the Compaction was created.
+    if (isManual) {
+      bottommost_level_ = true;
+    }
+    return;
+  }
+  bottommost_level_ = true;
+  int num_levels = input_version_->vset_->NumberLevels();
+  for (int i = level() + 2; i < num_levels; i++) {
+    if (input_version_->vset_->NumLevelFiles(i) > 0) {
+      bottommost_level_ = false;
+      break;
+    }
+  }
+}
+
 void Compaction::ReleaseInputs() {
  if (input_version_ != nullptr) {
    input_version_->Unref();
@@ -352,6 +352,9 @@ class VersionSet {
  struct LevelSummaryStorage {
    char buffer[100];
  };
+  struct FileSummaryStorage {
+    char buffer[1000];
+  };
  const char* LevelSummary(LevelSummaryStorage* scratch) const;

  // printf contents (for debugging)
@@ -362,6 +365,10 @@ class VersionSet {
  // of files per level.  Uses *scratch as backing store.
  const char* LevelDataSizeSummary(LevelSummaryStorage* scratch) const;

+  // Return a human-readable short (single-line) summary of files
+  // in a specified level.  Uses *scratch as backing store.
+  const char* LevelFileSummary(FileSummaryStorage* scratch, int level) const;
+
  // Return the size of the current manifest file
  const uint64_t ManifestFileSize() { return current_->offset_manifest_file_; }

@@ -371,6 +378,9 @@ class VersionSet {
  // function will return nullptr.
  Compaction* PickCompactionBySize(int level, double score);

+  // Pick files to compact in Universal mode
+  Compaction* PickCompactionUniversal(int level, double score);
+
  // Free up the files that were participated in a compaction
  void ReleaseCompactionFiles(Compaction* c, Status status);

@@ -426,9 +436,9 @@ class VersionSet {

  bool ManifestContains(const std::string& record) const;

-  int64_t ExpandedCompactionByteSizeLimit(int level);
+  uint64_t ExpandedCompactionByteSizeLimit(int level);

-  int64_t MaxGrandParentOverlapBytes(int level);
+  uint64_t MaxGrandParentOverlapBytes(int level);

  Env* const env_;
  const std::string dbname_;
@@ -503,9 +513,12 @@ class Compaction {
  ~Compaction();

  // Return the level that is being compacted.  Inputs from "level"
-  // and "level+1" will be merged to produce a set of "level+1" files.
+  // will be merged.
  int level() const { return level_; }

+  // Outputs will go to this level
+  int output_level() const { return out_level_; }
+
  // Return the object that holds the edits to the descriptor done
  // by this compaction.
  VersionEdit* edit() { return edit_; }
@@ -544,17 +557,21 @@ class Compaction {
  // Return the score that was used to pick this compaction run.
  double score() const { return score_; }

+  // Is this compaction creating a file in the bottom most level?
+  bool BottomMostLevel() { return bottommost_level_; }
+
 private:
  friend class Version;
  friend class VersionSet;

-  explicit Compaction(int level, uint64_t target_file_size,
+  explicit Compaction(int level, int out_level, uint64_t target_file_size,
    uint64_t max_grandparent_overlap_bytes, int number_levels,
    bool seek_compaction = false);

  int level_;
+  int out_level_; // levels to which output files are stored
  uint64_t max_output_file_size_;
-  int64_t maxGrandParentOverlapBytes_;
+  uint64_t maxGrandParentOverlapBytes_;
  Version* input_version_;
  VersionEdit* edit_;
  int number_levels_;
@@ -569,13 +586,14 @@ class Compaction {
  std::vector<FileMetaData*> grandparents_;
  size_t grandparent_index_;  // Index in grandparent_starts_
  bool seen_key_;             // Some output key has been seen
-  int64_t overlapped_bytes_;  // Bytes of overlap between current output
+  uint64_t overlapped_bytes_;  // Bytes of overlap between current output
                              // and grandparent files
  int base_index_;   // index of the file in files_[level_]
  int parent_index_; // index of some file with same range in files_[level_+1]
  double score_;     // score that was used to pick this compaction.

-  // State for implementing IsBaseLevelForKey
+  // Is this compaction creating a file in the bottom most level?
+  bool bottommost_level_;

  // level_ptrs_ holds indices into input_version_->levels_: our state
  // is that we are positioned at one of the file ranges for each
@@ -586,6 +604,9 @@ class Compaction {
  // mark (or clear) all files that are being compacted
  void MarkFilesBeingCompacted(bool);

+  // Initialize whether compaction producing files at the bottommost level
+  void SetupBottomMostLevel(bool isManual);
+
  // In case of compaction error, reset the nextIndex that is used
  // to pick up the next file to be compacted from files_by_size_
  void ResetNextCompactionIndex();
@@ -43,6 +43,11 @@ void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
  throw std::runtime_error("Handler::Merge not implemented!");
 }

+void WriteBatch::Handler::LogData(const Slice& blob) {
+  // If the user has not specified something to do with blobs, then we ignore
+  // them.
+}
+
 void WriteBatch::Clear() {
  rep_.clear();
  rep_.resize(kHeader);
@@ -59,10 +64,9 @@ Status WriteBatch::Iterate(Handler* handler) const {
  }

  input.remove_prefix(kHeader);
-  Slice key, value;
+  Slice key, value, blob;
  int found = 0;
  while (!input.empty()) {
-    found++;
    char tag = input[0];
    input.remove_prefix(1);
    switch (tag) {
@@ -70,6 +74,7 @@ Status WriteBatch::Iterate(Handler* handler) const {
        if (GetLengthPrefixedSlice(&input, &key) &&
            GetLengthPrefixedSlice(&input, &value)) {
          handler->Put(key, value);
+          found++;
        } else {
          return Status::Corruption("bad WriteBatch Put");
        }
@@ -77,6 +82,7 @@ Status WriteBatch::Iterate(Handler* handler) const {
      case kTypeDeletion:
        if (GetLengthPrefixedSlice(&input, &key)) {
          handler->Delete(key);
+          found++;
        } else {
          return Status::Corruption("bad WriteBatch Delete");
        }
@@ -85,10 +91,18 @@ Status WriteBatch::Iterate(Handler* handler) const {
        if (GetLengthPrefixedSlice(&input, &key) &&
            GetLengthPrefixedSlice(&input, &value)) {
          handler->Merge(key, value);
+          found++;
        } else {
          return Status::Corruption("bad WriteBatch Merge");
        }
        break;
+      case kTypeLogData:
+        if (GetLengthPrefixedSlice(&input, &blob)) {
+          handler->LogData(blob);
+        } else {
+          return Status::Corruption("bad WriteBatch Blob");
+        }
+        break;
      default:
        return Status::Corruption("unknown WriteBatch tag");
    }
@@ -136,6 +150,10 @@ void WriteBatch::Merge(const Slice& key, const Slice& value) {
  PutLengthPrefixedSlice(&rep_, value);
 }

+void WriteBatch::PutLogData(const Slice& blob) {
+  rep_.push_back(static_cast<char>(kTypeLogData));
+  PutLengthPrefixedSlice(&rep_, blob);
+}

 namespace {
 class MemTableInserter : public WriteBatch::Handler {
@@ -50,13 +50,16 @@ static std::string PrintContents(WriteBatch* b) {
        state.append(")");
        count++;
        break;
+      case kTypeLogData:
+        assert(false);
+        break;
    }
    state.append("@");
    state.append(NumberToString(ikey.sequence));
  }
  delete iter;
  if (!s.ok()) {
-    state.append("ParseError()");
+    state.append(s.ToString());
  } else if (count != WriteBatchInternal::Count(b)) {
    state.append("CountMismatch()");
  }
@@ -97,7 +100,7 @@ TEST(WriteBatchTest, Corruption) {
  WriteBatchInternal::SetContents(&batch,
                                  Slice(contents.data(),contents.size()-1));
  ASSERT_EQ("Put(foo, bar)@200"
-            "ParseError()",
+            "Corruption: bad WriteBatch Delete",
            PrintContents(&batch));
 }

@@ -131,6 +134,50 @@ TEST(WriteBatchTest, Append) {
  ASSERT_EQ(4, b1.Count());
 }

+TEST(WriteBatchTest, Blob) {
+  WriteBatch batch;
+  batch.Put(Slice("k1"), Slice("v1"));
+  batch.Put(Slice("k2"), Slice("v2"));
+  batch.Put(Slice("k3"), Slice("v3"));
+  batch.PutLogData(Slice("blob1"));
+  batch.Delete(Slice("k2"));
+  batch.PutLogData(Slice("blob2"));
+  batch.Merge(Slice("foo"), Slice("bar"));
+  ASSERT_EQ(5, batch.Count());
+  ASSERT_EQ("Merge(foo, bar)@4"
+            "Put(k1, v1)@0"
+            "Delete(k2)@3"
+            "Put(k2, v2)@1"
+            "Put(k3, v3)@2",
+            PrintContents(&batch));
+
+  struct Handler : public WriteBatch::Handler {
+    std::string seen;
+    virtual void Put(const Slice& key, const Slice& value) {
+      seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
+    }
+    virtual void Merge(const Slice& key, const Slice& value) {
+      seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
+    }
+    virtual void LogData(const Slice& blob) {
+      seen += "LogData(" + blob.ToString() + ")";
+    }
+    virtual void Delete(const Slice& key) {
+      seen += "Delete(" + key.ToString() + ")";
+    }
+  } handler;
+  batch.Iterate(&handler);
+  ASSERT_EQ(
+            "Put(k1, v1)"
+            "Put(k2, v2)"
+            "Put(k3, v3)"
+            "LogData(blob1)"
+            "Delete(k2)"
+            "LogData(blob2)"
+            "Merge(foo, bar)",
+            handler.seen);
+}
+
 }  // namespace leveldb

 int main(int argc, char** argv) {
@@ -1,2 +0,0 @@
-ar: creating libmemenv.a
-ar: creating libleveldb.a
@@ -41,6 +41,31 @@ class CompactionFilter {
  virtual const char* Name() const = 0;
 };

+// Each compaction will create a new CompactionFilter allowing the
+// application to know about different campactions
+class CompactionFilterFactory {
+  public:
+    virtual ~CompactionFilterFactory() { };
+    virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter() = 0;
+
+    // Returns a name that identifies this compaction filter factory.
+    virtual const char* Name() const = 0;
+};
+
+// Default implementaion of CompactionFilterFactory which does not
+// return any filter
+class DefaultCompactionFilterFactory : public CompactionFilterFactory {
+  public:
+    virtual std::unique_ptr<CompactionFilter>
+    CreateCompactionFilter() override {
+      return std::unique_ptr<CompactionFilter>(nullptr);
+    }
+
+    virtual const char* Name() const override {
+      return "DefaultCompactionFilterFactory";
+    }
+};
+
 }  // namespace leveldb

 #endif  // STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
@@ -12,7 +12,7 @@
 #include "leveldb/iterator.h"
 #include "leveldb/options.h"
 #include "leveldb/types.h"
-#include "leveldb/transaction_log_iterator.h"
+#include "leveldb/transaction_log.h"

 namespace leveldb {

@@ -232,6 +232,14 @@ class DB {
  virtual Status GetLiveFiles(std::vector<std::string>&,
                              uint64_t* manifest_file_size) = 0;

+  // Retrieve the sorted list of all wal files with earliest file first
+  virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
+
+  // Delete wal files in files. These can be either live or archived.
+  // Returns Status::OK if all files could be deleted, otherwise Status::IOError
+  // which contains information about files that could not be deleted.
+  virtual Status DeleteWalFiles(const VectorLogPtr& files) = 0;
+
  // The sequence number of the most recent transaction.
  virtual SequenceNumber GetLatestSequenceNumber() = 0;

@@ -29,10 +29,11 @@ class Logger;
 //    into rocksdb); numeric addition and string concatenation are examples;
 //
 //  b) MergeOperator - the generic class for all the more abstract / complex
-//    operations; one method to merge a Put/Delete value with a merge operand;
-//    and another method (PartialMerge) that merges two operands together.
-//    this is especially useful if your key values have a complex structure,
-//    but you would still like to support client-specific incremental updates.
+//    operations; one method (FullMerge) to merge a Put/Delete value with a
+//    merge operand; and another method (PartialMerge) that merges two
+//    operands together. this is especially useful if your key values have a
+//    complex structure but you would still like to support client-specific
+//    incremental updates.
 //
 // AssociativeMergeOperator is simpler to implement. MergeOperator is simply
 // more powerful.
@@ -60,11 +61,11 @@ class MergeOperator {
  // internal corruption. This will be treated as an error by the library.
  //
  // Also make use of the *logger for error messages.
-  virtual bool Merge(const Slice& key,
-                     const Slice* existing_value,
-                     const std::deque<std::string>& operand_list,
-                     std::string* new_value,
-                     Logger* logger) const = 0;
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_list,
+                         std::string* new_value,
+                         Logger* logger) const = 0;

  // This function performs merge(left_op, right_op)
  // when both the operands are themselves merge operation types
@@ -85,7 +86,7 @@ class MergeOperator {
  // TODO: Presently there is no way to differentiate between error/corruption
  // and simply "return false". For now, the client should simply return
  // false in any case it cannot perform partial-merge, regardless of reason.
-  // If there is corruption in the data, handle it in the above Merge() function,
+  // If there is corruption in the data, handle it in the FullMerge() function,
  // and return false there.
  virtual bool PartialMerge(const Slice& key,
                            const Slice& left_operand,
@@ -128,11 +129,11 @@ class AssociativeMergeOperator : public MergeOperator {

 private:
  // Default implementations of the MergeOperator functions
-  virtual bool Merge(const Slice& key,
-                     const Slice* existing_value,
-                     const std::deque<std::string>& operand_list,
-                     std::string* new_value,
-                     Logger* logger) const override;
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_list,
+                         std::string* new_value,
+                         Logger* logger) const override;

  virtual bool PartialMerge(const Slice& key,
                            const Slice& left_operand,
@@ -12,8 +12,11 @@
 #include <stdint.h>
 #include "leveldb/slice.h"
 #include "leveldb/statistics.h"
+#include "leveldb/universal_compaction.h"
 #include "leveldb/memtablerep.h"

+#include "leveldb/slice_transform.h"
+
 namespace leveldb {

 class Cache;
@@ -24,6 +27,7 @@ class Logger;
 class MergeOperator;
 class Snapshot;
 class CompactionFilter;
+class CompactionFilterFactory;

 using std::shared_ptr;

@@ -40,6 +44,11 @@ enum CompressionType {
  kBZip2Compression = 0x3
 };

+enum CompactionStyle {
+  kCompactionStyleLevel       = 0x0, // level based compaction style
+  kCompactionStyleUniversal   = 0x1  // Universal compaction style
+};
+
 // Compression options for different compression algorithms like Zlib
 struct CompressionOptions {
  int window_bits;
@@ -76,10 +85,15 @@ struct Options {
  // for the first time. It's necessary to specify a merge operator when
  // openning the DB in this case.
  // Default: nullptr
-  const MergeOperator* merge_operator;
+  shared_ptr<MergeOperator> merge_operator;

+  // The client must provide compaction_filter_factory if it requires a new
+  // compaction filter to be used for different compaction processes
  // Allows an application to modify/delete a key-value during background
  // compaction.
+  // Ideally, client should specify only one of filter or factory.
+  // compaction_filter takes precedence over compaction_filter_factory if
+  // client specifies both.
  // Default: nullptr
  const CompactionFilter* compaction_filter;

@@ -212,6 +226,28 @@ struct Options {
  // Default: nullptr
  const FilterPolicy* filter_policy;

+  // If non-nullptr, use the specified function to determine the
+  // prefixes for keys.  These prefixes will be placed in the filter.
+  // Depending on the workload, this can reduce the number of read-IOP
+  // cost for scans when a prefix is passed via ReadOptions to
+  // db.NewIterator().  For prefix filtering to work properly,
+  // "prefix_extractor" and "comparator" must be such that the following
+  // properties hold:
+  //
+  // 1) key.starts_with(prefix(key))
+  // 2) Compare(prefix(key), key) <= 0.
+  // 3) If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0
+  // 4) prefix(prefix(key)) == prefix(key)
+  //
+  // Default: nullptr
+  const SliceTransform* prefix_extractor;
+
+  // If true, place whole keys in the filter (not just prefixes).
+  // This must generally be true for gets to be efficient.
+  //
+  // Default: true
+  bool whole_key_filtering;
+
  // Number of levels for this database
  int num_levels;

@@ -377,9 +413,10 @@ struct Options {
  // The default value is MAX_INT so that roll-over does not take place.
  uint64_t max_manifest_file_size;

-  // Disable block cache. If this is set to false,
+  // Disable block cache. If this is set to true,
  // then no block cache should be used, and the block_cache should
  // point to a nullptr object.
+  // Default: false
  bool no_block_cache;

  // Number of shards used for table cache.
@@ -483,6 +520,12 @@ struct Options {
  // Default: 0
  uint64_t bytes_per_sync;

+  // The compaction style. Default: kCompactionStyleLevel
+  CompactionStyle compaction_style;
+
+  // The options needed to support Universal Style compactions
+  CompactionOptionsUniversal compaction_options_universal;
+
  // Use KeyMayExist API to filter deletes when this is true.
  // If KeyMayExist returns false, i.e. the key definitely does not exist, then
  // the delete is a noop. KeyMayExist only incurs in-memory look up.
@@ -495,6 +538,10 @@ struct Options {
  // MemTableRep.
  std::shared_ptr<MemTableRepFactory> memtable_factory;

+  // This is a factory that provides compaction filter objects which allow
+  // an application to modify/delete a key-value during background compaction.
+  // Default: a factory that doesn't provide any object
+  std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
 };

 // Options that control read operations
@@ -516,14 +563,28 @@ struct ReadOptions {
  // Default: nullptr
  const Snapshot* snapshot;

+  // If "prefix" is non-nullptr, and ReadOptions is being passed to
+  // db.NewIterator, only return results when the key begins with this
+  // prefix.  This field is ignored by other calls (e.g., Get).
+  // Options.prefix_extractor must also be set, and
+  // prefix_extractor.InRange(prefix) must be true.  The iterator
+  // returned by NewIterator when this option is set will behave just
+  // as if the underlying store did not contain any non-matching keys,
+  // with two exceptions.  Seek() only accepts keys starting with the
+  // prefix, and SeekToLast() is not supported.  prefix filter with this
+  // option will sometimes reduce the number of read IOPs.
+  // Default: nullptr
+  const Slice* prefix;
+
  ReadOptions()
      : verify_checksums(false),
        fill_cache(true),
-        snapshot(nullptr) {
+        snapshot(nullptr),
+        prefix(nullptr) {
  }
  ReadOptions(bool cksum, bool cache) :
              verify_checksums(cksum), fill_cache(cache),
-              snapshot(nullptr) {
+              snapshot(nullptr), prefix(nullptr) {
  }
 };

@@ -0,0 +1,24 @@
+#ifndef STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
+#define STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
+
+#include <stdint.h>
+
+namespace leveldb {
+
+// A thread local context for gathering performance counter efficiently
+// and transparently.
+
+struct PerfContext {
+
+  void Reset(); // reset all performance counters to zero
+
+
+  uint64_t user_key_comparison_count; // total number of user key comparisons
+};
+
+extern __thread PerfContext perf_context;
+
+}
+
+
+#endif
@@ -0,0 +1,41 @@
+// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Class for specifying user-defined functions which perform a
+// transformation on a slice.  It is not required that every slice
+// belong to the domain and/or range of a function.  Subclasses should
+// define InDomain and InRange to determine which slices are in either
+// of these sets respectively.
+
+#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
+#define STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
+
+#include <string>
+
+namespace leveldb {
+
+class Slice;
+
+class SliceTransform {
+ public:
+  virtual ~SliceTransform() {};
+
+  // Return the name of this transformation.
+  virtual const char* Name() const = 0;
+
+  // transform a src in domain to a dst in the range
+  virtual Slice Transform(const Slice& src) const = 0;
+
+  // determine whether this is a valid src upon the function applies
+  virtual bool InDomain(const Slice& src) const = 0;
+
+  // determine whether dst=Transform(src) for some src
+  virtual bool InRange(const Slice& dst) const = 0;
+};
+
+extern const SliceTransform* NewFixedPrefixTransform(size_t prefix_len);
+
+}
+
+#endif  // STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
@@ -59,8 +59,10 @@ enum Tickers {
  NUMBER_MULTIGET_BYTES_READ = 20,

  NUMBER_FILTERED_DELETES = 21,
+  NUMBER_MERGE_FAILURES = 22,
+  SEQUENCE_NUMBER = 23,

-  TICKER_ENUM_MAX = 22
+  TICKER_ENUM_MAX = 24
 };

 const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
@@ -85,7 +87,9 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
  { NUMBER_MULTIGET_CALLS, "rocksdb.number.multiget.get" },
  { NUMBER_MULTIGET_KEYS_READ, "rocksdb.number.multiget.keys.read" },
  { NUMBER_MULTIGET_BYTES_READ, "rocksdb.number.multiget.bytes.read" },
-  { NUMBER_FILTERED_DELETES, "rocksdb.number.deletes.filtered" }
+  { NUMBER_FILTERED_DELETES, "rocksdb.number.deletes.filtered" },
+  { NUMBER_MERGE_FAILURES, "rocksdb.number.merge.failures" },
+  { SEQUENCE_NUMBER, "rocksdb.sequence.number" }
 };

 /**
@@ -115,7 +119,8 @@ enum Histograms {
  STALL_L0_NUM_FILES_COUNT = 14,
  HARD_RATE_LIMIT_DELAY_COUNT = 15,
  SOFT_RATE_LIMIT_DELAY_COUNT = 16,
-  HISTOGRAM_ENUM_MAX = 17
+  NUM_FILES_IN_SINGLE_COMPACTION = 17,
+  HISTOGRAM_ENUM_MAX = 18
 };

 const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
@@ -135,7 +140,8 @@ const std::vector<std::pair<Histograms, std::string>> HistogramsNameMap = {
  { STALL_MEMTABLE_COMPACTION_COUNT, "rocksdb.memtable.compaction.count"},
  { STALL_L0_NUM_FILES_COUNT, "rocksdb.num.files.stall.count"},
  { HARD_RATE_LIMIT_DELAY_COUNT, "rocksdb.hard.rate.limit.delay.count"},
-  { SOFT_RATE_LIMIT_DELAY_COUNT, "rocksdb.soft.rate.limit.delay.count"}
+  { SOFT_RATE_LIMIT_DELAY_COUNT, "rocksdb.soft.rate.limit.delay.count"},
+  { NUM_FILES_IN_SINGLE_COMPACTION, "rocksdb.numfiles.in.singlecompaction" }
 };

 struct HistogramData {
@@ -174,11 +180,11 @@ class Ticker {
 public:
  Ticker() : count_(0) { }

- inline void recordTick() {
-    count_++;
+  inline void setTickerCount(uint64_t count) {
+    count_ = count;
  }

-  inline void recordTick(int count) {
+  inline void recordTick(int count = 1) {
    count_ += count;
  }

@@ -196,6 +202,7 @@ class Statistics {

  virtual long getTickerCount(Tickers tickerType) = 0;
  virtual void recordTick(Tickers tickerType, uint64_t count = 0) = 0;
+  virtual void setTickerCount(Tickers tickerType, uint64_t count) = 0;
  virtual void measureTime(Histograms histogramType, uint64_t time) = 0;

  virtual void histogramData(Histograms type, HistogramData * const data) = 0;
@@ -216,6 +223,17 @@ inline void RecordTick(std::shared_ptr<Statistics> statistics,
    statistics->recordTick(ticker, count);
  }
 }
+
+inline void SetTickerCount(std::shared_ptr<Statistics> statistics,
+                           Tickers ticker,
+                           uint64_t count) {
+  assert(HistogramsNameMap.size() == HISTOGRAM_ENUM_MAX);
+  assert(TickersNameMap.size() == TICKER_ENUM_MAX);
+  if (statistics) {
+    statistics->setTickerCount(ticker, count);
+  }
+}
+
 }  // namespace leveldb

 #endif  // STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
@@ -3,10 +3,46 @@
 #define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_

 #include "leveldb/status.h"
+#include "leveldb/types.h"
 #include "leveldb/write_batch.h"

 namespace leveldb {

+class LogFile;
+typedef std::vector<std::unique_ptr<LogFile>> VectorLogPtr;
+
+enum  WalFileType {
+  /* Indicates that WAL file is in archive directory. WAL files are moved from
+   * the main db directory to archive directory once they are not live and stay
+   * there for a duration of WAL_ttl_seconds which can be set in Options
+   */
+  kArchivedLogFile = 0,
+
+  /* Indicates that WAL file is live and resides in the main db directory */
+  kAliveLogFile = 1
+} ;
+
+class LogFile {
+ public:
+  LogFile() {}
+  virtual ~LogFile() {}
+
+  // Returns log file's name excluding the db path
+  virtual std::string Filename() const = 0;
+
+  // Primary identifier for log file.
+  // This is directly proportional to creation time of the log file
+  virtual uint64_t LogNumber() const = 0;
+
+  // Log file can be either alive or archived
+  virtual WalFileType Type() const = 0;
+
+  // Starting sequence number of writebatch written in this log file
+  virtual SequenceNumber StartSequence() const = 0;
+
+  // Size of log file on disk in Bytes
+  virtual uint64_t SizeFileBytes() const = 0;
+};

 struct BatchResult {
  SequenceNumber sequence;
@@ -7,7 +7,7 @@ namespace leveldb {

 // Define all public custom types here.

-//  Represents a sequence number in a WAL file.
+// Represents a sequence number in a WAL file.
 typedef uint64_t SequenceNumber;

 }  //  namespace leveldb
@@ -0,0 +1,57 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
+#define STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
+
+#include <stddef.h>
+#include <string>
+#include <memory>
+#include <vector>
+#include <stdint.h>
+#include <climits>
+#include "leveldb/slice.h"
+#include "leveldb/statistics.h"
+
+namespace leveldb {
+
+//
+// Algorithm used to make a compaction request stop picking new files
+// into a single compaction run
+//
+enum CompactionStopStyle {
+  kCompactionStopStyleSimilarSize, // pick files of similar size
+  kCompactionStopStyleTotalSize    // total size of picked files > next file
+};
+
+class CompactionOptionsUniversal {
+ public:
+
+  // Percentage flexibilty while comparing file size. If the candidate file(s)
+  // size is 1% smaller than the next file's size, then include next file into
+  // this candidate set. // Default: 1
+  unsigned int size_ratio;
+
+  // The minimum number of files in a single compaction run. Default: 2
+  unsigned int min_merge_width;
+
+  // The maximum number of files in a single compaction run. Default: INT_MAX
+  unsigned int max_merge_width;
+
+  // The algorithm used to stop picking files into a single compaction run
+  // Default: kCompactionStopStyleTotalSize
+  CompactionStopStyle stop_style;
+
+  // Default set of parameters
+  CompactionOptionsUniversal() :
+    size_ratio(1),
+    min_merge_width(2),
+    max_merge_width(UINT_MAX),
+    stop_style(kCompactionStopStyleTotalSize) {
+  }
+};
+
+}  // namespace leveldb
+
+#endif  // STORAGE_ROCKSDB_UNIVERSAL_COMPACTION_OPTIONS_H
@@ -43,6 +43,17 @@ class WriteBatch {
  // If the database contains a mapping for "key", erase it.  Else do nothing.
  void Delete(const Slice& key);

+  // Append a blob of arbitrary to the records in this batch. The blob will be
+  // stored in the transaction log but not in any other file. In particular, it
+  // will not be persisted to the SST files. When iterating over this
+  // WriteBatch, WriteBatch::Handler::LogData will be called with the contents
+  // of the blob as it is encountered. Blobs, puts, deletes, and merges will be
+  // encountered in the same order in thich they were inserted.
+  //
+  // Example application: add timestamps to the transaction log for use in
+  // replication.
+  void PutLogData(const Slice& blob);
+
  // Clear all updates buffered in this batch.
  void Clear();

@@ -51,10 +62,12 @@ class WriteBatch {
   public:
    virtual ~Handler();
    virtual void Put(const Slice& key, const Slice& value) = 0;
-    // Merge is not pure virtual. Otherwise, we would break existing
-    // clients of Handler on a source code level.
-    // The default implementation simply throws a runtime exception.
+    // Merge and LogData are not pure virtual. Otherwise, we would break
+    // existing clients of Handler on a source code level. The default
+    // implementation of Merge simply throws a runtime exception.
    virtual void Merge(const Slice& key, const Slice& value);
+    // The default implementation of LogData does nothing.
+    virtual void LogData(const Slice& blob);
    virtual void Delete(const Slice& key) = 0;
  };
  Status Iterate(Handler* handler) const;
@@ -66,7 +79,7 @@ class WriteBatch {
  int Count() const;

  // Constructor with a serialized string object
-  WriteBatch(std::string rep): rep_(rep) {}
+  explicit WriteBatch(std::string rep): rep_(rep) {}

 private:
  friend class WriteBatchInternal;
@@ -60,7 +60,7 @@ class StackableDB : public DB {
                           const Slice& key,
                           std::string* value,
                           bool* value_found = nullptr) override {
-    return KeyMayExist(options, key, value, value_found);
+    return sdb_->KeyMayExist(options, key, value, value_found);
  }

  virtual Status Delete(const WriteOptions& wopts, const Slice& key) override {
@@ -139,6 +139,15 @@ class StackableDB : public DB {
    return sdb_->GetLatestSequenceNumber();
  }

+  virtual Status GetSortedWalFiles(VectorLogPtr& files) override {
+    return sdb_->GetSortedWalFiles(files);
+  }
+
+  virtual Status DeleteWalFiles(const VectorLogPtr& files)
+    override{
+      return sdb_->DeleteWalFiles(files);
+  }
+
  virtual Status GetUpdatesSince(SequenceNumber seq_number,
                                 unique_ptr<TransactionLogIterator>* iter)
    override {
@@ -4,6 +4,7 @@

 #include "table/filter_block.h"

+#include "db/dbformat.h"
 #include "leveldb/filter_policy.h"
 #include "util/coding.h"

@@ -15,9 +16,11 @@ namespace leveldb {
 static const size_t kFilterBaseLg = 11;
 static const size_t kFilterBase = 1 << kFilterBaseLg;

-FilterBlockBuilder::FilterBlockBuilder(const FilterPolicy* policy)
-    : policy_(policy) {
-}
+FilterBlockBuilder::FilterBlockBuilder(const Options& opt)
+                 : policy_(opt.filter_policy),
+                   prefix_extractor_(opt.prefix_extractor),
+                   whole_key_filtering_(opt.whole_key_filtering),
+                   comparator_(opt.comparator){}

 void FilterBlockBuilder::StartBlock(uint64_t block_offset) {
  uint64_t filter_index = (block_offset / kFilterBase);
@@ -27,10 +30,47 @@ void FilterBlockBuilder::StartBlock(uint64_t block_offset) {
  }
 }

+bool FilterBlockBuilder::SamePrefix(const Slice &key1,
+                                    const Slice &key2) const {
+  if (!prefix_extractor_->InDomain(key1) &&
+      !prefix_extractor_->InDomain(key2)) {
+    return true;
+  } else if (!prefix_extractor_->InDomain(key1) ||
+             !prefix_extractor_->InDomain(key2)) {
+    return false;
+  } else {
+    return (prefix_extractor_->Transform(key1) ==
+            prefix_extractor_->Transform(key2));
+  }
+}
+
 void FilterBlockBuilder::AddKey(const Slice& key) {
-  Slice k = key;
-  start_.push_back(keys_.size());
-  keys_.append(k.data(), k.size());
+  Slice prev;
+  if (start_.size() > 0) {
+    size_t prev_start = start_[start_.size() - 1];
+    const char* base = entries_.data() + prev_start;
+    size_t length = entries_.size() - prev_start;
+    prev = Slice(base, length);
+  }
+
+  if (whole_key_filtering_) {
+    start_.push_back(entries_.size());
+    entries_.append(key.data(), key.size());
+  }
+
+  if (prefix_extractor_ && prefix_extractor_->InDomain(key)) {
+    // this assumes prefix(prefix(key)) == prefix(key), as the last
+    // entry in entries_ may be either a key or prefix, and we use
+    // prefix(last entry) to get the prefix of the last key.
+    if (prev.size() == 0 || ! SamePrefix(key, prev)) {
+      Slice prefix = prefix_extractor_->Transform(key);
+      InternalKey internal_prefix_tmp(prefix, 0, kTypeValue);
+      Slice internal_prefix = internal_prefix_tmp.Encode();
+      assert(comparator_->Compare(internal_prefix, key) <= 0);
+      start_.push_back(entries_.size());
+      entries_.append(internal_prefix.data(), internal_prefix.size());
+    }
+  }
 }

 Slice FilterBlockBuilder::Finish() {
@@ -50,34 +90,35 @@ Slice FilterBlockBuilder::Finish() {
 }

 void FilterBlockBuilder::GenerateFilter() {
-  const size_t num_keys = start_.size();
-  if (num_keys == 0) {
+  const size_t num_entries = start_.size();
+  if (num_entries == 0) {
    // Fast path if there are no keys for this filter
    filter_offsets_.push_back(result_.size());
    return;
  }

  // Make list of keys from flattened key structure
-  start_.push_back(keys_.size());  // Simplify length computation
-  tmp_keys_.resize(num_keys);
-  for (size_t i = 0; i < num_keys; i++) {
-    const char* base = keys_.data() + start_[i];
+  start_.push_back(entries_.size());  // Simplify length computation
+  tmp_entries_.resize(num_entries);
+  for (size_t i = 0; i < num_entries; i++) {
+    const char* base = entries_.data() + start_[i];
    size_t length = start_[i+1] - start_[i];
-    tmp_keys_[i] = Slice(base, length);
+    tmp_entries_[i] = Slice(base, length);
  }

  // Generate filter for current set of keys and append to result_.
  filter_offsets_.push_back(result_.size());
-  policy_->CreateFilter(&tmp_keys_[0], num_keys, &result_);
+  policy_->CreateFilter(&tmp_entries_[0], num_entries, &result_);

-  tmp_keys_.clear();
-  keys_.clear();
+  tmp_entries_.clear();
+  entries_.clear();
  start_.clear();
 }

-FilterBlockReader::FilterBlockReader(const FilterPolicy* policy,
-                                     const Slice& contents)
-    : policy_(policy),
+FilterBlockReader::FilterBlockReader(const Options& opt, const Slice& contents)
+    : policy_(opt.filter_policy),
+      prefix_extractor_(opt.prefix_extractor),
+      whole_key_filtering_(opt.whole_key_filtering),
      data_(nullptr),
      offset_(nullptr),
      num_(0),
@@ -92,16 +133,32 @@ FilterBlockReader::FilterBlockReader(const FilterPolicy* policy,
  num_ = (n - 5 - last_word) / 4;
 }

-bool FilterBlockReader::KeyMayMatch(uint64_t block_offset, const Slice& key) {
+bool FilterBlockReader::KeyMayMatch(uint64_t block_offset,
+                                    const Slice& key) {
+  if (!whole_key_filtering_) {
+    return true;
+  }
+  return MayMatch(block_offset, key);
+}
+
+bool FilterBlockReader::PrefixMayMatch(uint64_t block_offset,
+                                       const Slice& prefix) {
+  if (!prefix_extractor_) {
+    return true;
+  }
+  return MayMatch(block_offset, prefix);
+}
+
+bool FilterBlockReader::MayMatch(uint64_t block_offset, const Slice& entry) {
  uint64_t index = block_offset >> base_lg_;
  if (index < num_) {
    uint32_t start = DecodeFixed32(offset_ + index*4);
    uint32_t limit = DecodeFixed32(offset_ + index*4 + 4);
    if (start <= limit && limit <= (offset_ - data_)) {
      Slice filter = Slice(data_ + start, limit - start);
-      return policy_->KeyMayMatch(key, filter);
+      return policy_->KeyMayMatch(entry, filter);
    } else if (start == limit) {
-      // Empty filters do not match any keys
+      // Empty filters do not match any entries
      return false;
    }
  }
@@ -13,7 +13,9 @@
 #include <stdint.h>
 #include <string>
 #include <vector>
+#include "leveldb/options.h"
 #include "leveldb/slice.h"
+#include "leveldb/slice_transform.h"
 #include "util/hash.h"

 namespace leveldb {
@@ -28,20 +30,25 @@ class FilterPolicy;
 //      (StartBlock AddKey*)* Finish
 class FilterBlockBuilder {
 public:
-  explicit FilterBlockBuilder(const FilterPolicy*);
+  explicit FilterBlockBuilder(const Options& opt);

  void StartBlock(uint64_t block_offset);
  void AddKey(const Slice& key);
  Slice Finish();

 private:
+  bool SamePrefix(const Slice &key1, const Slice &key2) const;
  void GenerateFilter();

  const FilterPolicy* policy_;
-  std::string keys_;              // Flattened key contents
-  std::vector<size_t> start_;     // Starting index in keys_ of each key
-  std::string result_;            // Filter data computed so far
-  std::vector<Slice> tmp_keys_;   // policy_->CreateFilter() argument
+  const SliceTransform* prefix_extractor_;
+  bool whole_key_filtering_;
+  const Comparator* comparator_;
+
+  std::string entries_;         // Flattened entry contents
+  std::vector<size_t> start_;   // Starting index in entries_ of each entry
+  std::string result_;          // Filter data computed so far
+  std::vector<Slice> tmp_entries_; // policy_->CreateFilter() argument
  std::vector<uint32_t> filter_offsets_;

  // No copying allowed
@@ -52,15 +59,20 @@ class FilterBlockBuilder {
 class FilterBlockReader {
 public:
 // REQUIRES: "contents" and *policy must stay live while *this is live.
-  FilterBlockReader(const FilterPolicy* policy, const Slice& contents);
+  FilterBlockReader(const Options& opt, const Slice& contents);
  bool KeyMayMatch(uint64_t block_offset, const Slice& key);
+  bool PrefixMayMatch(uint64_t block_offset, const Slice& prefix);

 private:
  const FilterPolicy* policy_;
+  const SliceTransform* prefix_extractor_;
+  bool whole_key_filtering_;
  const char* data_;    // Pointer to filter data (at block-start)
  const char* offset_;  // Pointer to beginning of offset array (at block-end)
  size_t num_;          // Number of entries in offset array
  size_t base_lg_;      // Encoding parameter (see kFilterBaseLg in .cc file)
+
+  bool MayMatch(uint64_t block_offset, const Slice& entry);
 };

 }
@@ -41,19 +41,25 @@ class TestHashFilter : public FilterPolicy {
 class FilterBlockTest {
 public:
  TestHashFilter policy_;
+  Options options_;
+
+  FilterBlockTest() {
+    options_ = Options();
+    options_.filter_policy = &policy_;
+  }
 };

 TEST(FilterBlockTest, EmptyBuilder) {
-  FilterBlockBuilder builder(&policy_);
+  FilterBlockBuilder builder(options_);
  Slice block = builder.Finish();
  ASSERT_EQ("\\x00\\x00\\x00\\x00\\x0b", EscapeString(block));
-  FilterBlockReader reader(&policy_, block);
+  FilterBlockReader reader(options_, block);
  ASSERT_TRUE(reader.KeyMayMatch(0, "foo"));
  ASSERT_TRUE(reader.KeyMayMatch(100000, "foo"));
 }

 TEST(FilterBlockTest, SingleChunk) {
-  FilterBlockBuilder builder(&policy_);
+  FilterBlockBuilder builder(options_);
  builder.StartBlock(100);
  builder.AddKey("foo");
  builder.AddKey("bar");
@@ -63,7 +69,7 @@ TEST(FilterBlockTest, SingleChunk) {
  builder.StartBlock(300);
  builder.AddKey("hello");
  Slice block = builder.Finish();
-  FilterBlockReader reader(&policy_, block);
+  FilterBlockReader reader(options_, block);
  ASSERT_TRUE(reader.KeyMayMatch(100, "foo"));
  ASSERT_TRUE(reader.KeyMayMatch(100, "bar"));
  ASSERT_TRUE(reader.KeyMayMatch(100, "box"));
@@ -74,7 +80,7 @@ TEST(FilterBlockTest, SingleChunk) {
 }

 TEST(FilterBlockTest, MultiChunk) {
-  FilterBlockBuilder builder(&policy_);
+  FilterBlockBuilder builder(options_);

  // First filter
  builder.StartBlock(0);
@@ -94,7 +100,7 @@ TEST(FilterBlockTest, MultiChunk) {
  builder.AddKey("hello");

  Slice block = builder.Finish();
-  FilterBlockReader reader(&policy_, block);
+  FilterBlockReader reader(options_, block);

  // Check first filter
  ASSERT_TRUE(reader.KeyMayMatch(0, "foo"));
@@ -4,6 +4,8 @@

 #include "table/table.h"

+#include "db/dbformat.h"
+
 #include "leveldb/cache.h"
 #include "leveldb/comparator.h"
 #include "leveldb/env.h"
@@ -207,7 +209,7 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
  if (block.heap_allocated) {
    rep_->filter_data = block.data.data();     // Will need to delete later
  }
-  rep_->filter = new FilterBlockReader(rep_->options.filter_policy, block.data);
+  rep_->filter = new FilterBlockReader(rep_->options, block.data);
 }

 Table::~Table() {
@@ -318,7 +320,66 @@ Iterator* Table::BlockReader(void* arg,
  return BlockReader(arg, options, index_value, nullptr, for_compaction);
 }

+// This will be broken if the user specifies an unusual implementation
+// of Options.comparator, or if the user specifies an unusual
+// definition of prefixes in Options.filter_policy.  In particular, we
+// require the following three properties:
+//
+// 1) key.starts_with(prefix(key))
+// 2) Compare(prefix(key), key) <= 0.
+// 3) If Compare(key1, key2) <= 0, then Compare(prefix(key1), prefix(key2)) <= 0
+bool Table::PrefixMayMatch(const Slice& internal_prefix) const {
+  FilterBlockReader* filter = rep_->filter;
+  bool may_match = true;
+  Status s;
+
+  if (filter == nullptr) {
+    return true;
+  }
+
+  Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
+  iiter->Seek(internal_prefix);
+  if (! iiter->Valid()) {
+    // we're past end of file
+    may_match = false;
+  } else if (iiter->key().starts_with(internal_prefix)) {
+    // we need to check for this subtle case because our only
+    // guarantee is that "the key is a string >= last key in that data
+    // block" according to the doc/table_format.txt spec.
+    //
+    // Suppose iiter->key() starts with the desired prefix; it is not
+    // necessarily the case that the corresponding data block will
+    // contain the prefix, since iiter->key() need not be in the
+    // block.  However, the next data block may contain the prefix, so
+    // we return true to play it safe.
+    may_match = true;
+  } else {
+    // iiter->key() does NOT start with the desired prefix.  Because
+    // Seek() finds the first key that is >= the seek target, this
+    // means that iiter->key() > prefix.  Thus, any data blocks coming
+    // after the data block corresponding to iiter->key() cannot
+    // possibly contain the key.  Thus, the corresponding data block
+    // is the only one which could potentially contain the prefix.
+    Slice handle_value = iiter->value();
+    BlockHandle handle;
+    s = handle.DecodeFrom(&handle_value);
+    assert(s.ok());
+    may_match = filter->PrefixMayMatch(handle.offset(), internal_prefix);
+  }
+  delete iiter;
+  return may_match;
+}
+
 Iterator* Table::NewIterator(const ReadOptions& options) const {
+  if (options.prefix) {
+    InternalKey internal_prefix(*options.prefix, 0, kTypeValue);
+    if (!PrefixMayMatch(internal_prefix.Encode())) {
+      // nothing in this file can match the prefix, so we should not
+      // bother doing I/O to this file when iterating.
+      return NewEmptyIterator();
+    }
+  }
+
  return NewTwoLevelIterator(
      rep_->index_block->NewIterator(rep_->options.comparator),
      &Table::BlockReader, const_cast<Table*>(this), options, rep_->soptions);
@@ -47,6 +47,8 @@ class Table {

  ~Table();

+  bool PrefixMayMatch(const Slice& prefix) const;
+
  // Returns a new iterator over the table contents.
  // The result of NewIterator() is initially invalid (caller must
  // call one of the Seek methods on the iterator before using it).
@@ -55,7 +55,7 @@ struct TableBuilder::Rep {
        num_entries(0),
        closed(false),
        filter_block(opt.filter_policy == nullptr ? nullptr
-                     : new FilterBlockBuilder(opt.filter_policy)),
+                     : new FilterBlockBuilder(opt)),
        pending_index_entry(false) {
    index_block_options.block_restart_interval = 1;
  }
@@ -1,5 +1,6 @@
 #! /usr/bin/env python
 import os
+import re
 import sys
 import time
 import random
@@ -8,18 +9,16 @@ import logging
 import tempfile
 import subprocess

-# This python script runs and kills db_stress multiple times with
-# test-batches-snapshot ON,
-# total operations much less than the total keys, and
-# a high read percentage.
-# This checks consistency in case of unsafe crashes in  Rocksdb
+# This script runs and kills db_stress multiple times. It checks consistency
+# in case of unsafe crashes in Rocksdb.

 def main(argv):
    try:
        opts, args = getopt.getopt(argv, "hd:t:i:o:b:")
    except getopt.GetoptError:
        print("db_crashtest.py -d <duration_test> -t <#threads> "
-              "-i <interval for one run> -o <ops_per_thread>\n")
+              "-i <interval for one run> -o <ops_per_thread> "
+              "-b <write_buffer_size>\n")
        sys.exit(2)

    # default values, will be overridden by cmdline args
@@ -36,15 +35,15 @@ def main(argv):
                  " -t <#threads> -i <interval for one run>"
                  " -o <ops_per_thread> -b <write_buffer_size>\n")
            sys.exit()
-        elif opt == ("-d"):
+        elif opt == "-d":
            duration = int(arg)
-        elif opt == ("-t"):
+        elif opt == "-t":
            threads = int(arg)
-        elif opt == ("-i"):
+        elif opt == "-i":
            interval = int(arg)
-        elif opt == ("-o"):
+        elif opt == "-o":
            ops_per_thread = int(arg)
-        elif opt == ("-b"):
+        elif opt == "-b":
            write_buf_size = int(arg)
        else:
            print("db_crashtest.py -d <duration_test>"
@@ -54,8 +53,6 @@ def main(argv):

    exit_time = time.time() + duration

-    dirpath = tempfile.mkdtemp()
-
    print("Running blackbox-crash-test with \ninterval_between_crash="
          + str(interval) + "\ntotal-duration=" + str(duration)
          + "\nthreads=" + str(threads) + "\nops_per_thread="
@@ -64,59 +61,75 @@ def main(argv):

    while time.time() < exit_time:
        run_had_errors = False
-        additional_opts = ' --disable_seek_compaction=' + \
-                          str(random.randint(0, 1)) + \
-                          ' --mmap_read=' + str(random.randint(0, 1)) + \
-                          ' --block_size=16384 ' + \
-                          ' --cache_size=1048576 ' + \
-                          ' --open_files=500000 ' + \
-                          ' --verify_checksum=1 ' + \
-                          ' --sync=' + str(random.randint(0, 1)) + \
-                          ' --disable_wal=0 ' + \
-                          ' --disable_data_sync=' + \
-                          str(random.randint(0, 1)) + \
-                          ' --target_file_size_base=2097152 ' + \
-                          ' --target_file_size_multiplier=2 ' + \
-                          ' --max_write_buffer_number=3 ' + \
-                          ' --max_background_compactions=20 ' + \
-                          ' --max_bytes_for_level_base=10485760 ' + \
-                          ' --filter_deletes=' + str(random.randint(0, 1))
        killtime = time.time() + interval
-        child = subprocess.Popen(['./db_stress \
-                        --test_batches_snapshots=1 \
-                        --ops_per_thread=0' + str(ops_per_thread) + ' \
-                        --threads=0' + str(threads) + ' \
-                        --write_buffer_size=' + str(write_buf_size) + '\
-                        --destroy_db_initially=0 \
-                        --reopen=0 \
-                        --readpercent=50 \
-                        --db=' + dirpath + '\
-                        --max_key=100000000 ' + additional_opts],
-                                 stderr=subprocess.PIPE, shell=True)
-        print("Running db_stress with pid=%d and additional options=\n"
-              % child.pid + additional_opts + "\n")
-        time.sleep(interval)
-        while True:
-            if time.time() > killtime:
-                if child.poll() is not None:
-                    print("WARNING: db_stress ended before kill\n")
-                else:
-                    child.kill()
-                    print("KILLED %d\n" % child.pid)
-                    time.sleep(1)  # time to stabilize after a kill

-                while True:
-                    line = child.stderr.readline().strip()
-                    if line != '':
-                        run_had_errors = True
-                        print('***' + line + '^')
-                    else:
-                        break
-                if run_had_errors:
-                    sys.exit(2)
+        cmd = re.sub('\s+', ' ', """
+            ./db_stress
+            --test_batches_snapshots=1
+            --ops_per_thread=%s
+            --threads=%s
+            --write_buffer_size=%s
+            --destroy_db_initially=0
+            --reopen=0
+            --readpercent=50
+            --prefixpercent=5
+            --writepercent=40
+            --delpercent=5
+            --db=%s
+            --max_key=100000000
+            --disable_seek_compaction=%s
+            --mmap_read=%s
+            --block_size=16384
+            --cache_size=1048576
+            --open_files=500000
+            --verify_checksum=1
+            --sync=%s
+            --disable_wal=0
+            --disable_data_sync=%s
+            --target_file_size_base=2097152
+            --target_file_size_multiplier=2
+            --max_write_buffer_number=3
+            --max_background_compactions=20
+            --max_bytes_for_level_base=10485760
+            --filter_deletes=%s
+            """ % (ops_per_thread,
+                   threads,
+                   write_buf_size,
+                   tempfile.mkdtemp(),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   random.randint(0, 1)))
+
+        child = subprocess.Popen([cmd],
+                                 stderr=subprocess.PIPE, shell=True)
+        print("Running db_stress with pid=%d: %s\n\n"
+              % (child.pid, cmd))
+
+        while time.time() < killtime:
+            time.sleep(10)
+
+        if child.poll() is not None:
+            print("WARNING: db_stress ended before kill: exitcode=%d\n"
+                  % child.returncode)
+        else:
+            child.kill()
+            print("KILLED %d\n" % child.pid)
+            time.sleep(1)  # time to stabilize after a kill
+
+        while True:
+            line = child.stderr.readline().strip()
+            if line != '':
+                run_had_errors = True
+                print('***' + line + '^')
+            else:
                break

-            time.sleep(1)  # time to stabilize before the next run
+        if run_had_errors:
+            sys.exit(2)
+
+        time.sleep(1)  # time to stabilize before the next run

 if __name__ == "__main__":
    sys.exit(main(sys.argv[1:]))
@@ -1,5 +1,6 @@
 #! /usr/bin/env python
 import os
+import re
 import sys
 import time
 import random
@@ -8,24 +9,22 @@ import logging
 import tempfile
 import subprocess

-# This python script runs db_stress multiple times with kill_random_test
-# that causes leveldb to crash at various points in code.
-# It also has test-batches-snapshot ON so that basic atomic/consistency
-# checks can be performed.
-#
+# This python script runs db_stress multiple times. Some runs with
+# kill_random_test that causes leveldb to crash at various points in code.
+
 def main(argv):
    try:
        opts, args = getopt.getopt(argv, "hd:t:k:o:b:")
    except getopt.GetoptError:
        print str(getopt.GetoptError)
        print "db_crashtest2.py -d <duration_test> -t <#threads> " \
-            "-k <kills with prob 1/k> -o <ops_per_thread> "\
-            "-b <write_buffer_size>\n"
+              "-k <kills with prob 1/k> -o <ops_per_thread> "\
+              "-b <write_buffer_size>\n"
        sys.exit(2)

    # default values, will be overridden by cmdline args
    kill_random_test = 97  # kill with probability 1/97 by default
-    duration = 6000  # total time for this script to test db_stress
+    duration = 10000  # total time for this script to test db_stress
    threads = 32
    ops_per_thread = 200000
    write_buf_size = 4 * 1024 * 1024
@@ -33,90 +32,101 @@ def main(argv):
    for opt, arg in opts:
        if opt == '-h':
            print "db_crashtest2.py -d <duration_test> -t <#threads> " \
-                "-k <kills with prob 1/k> -o <ops_per_thread> "\
-                "-b <write_buffer_size>\n"
+                  "-k <kills with prob 1/k> -o <ops_per_thread> " \
+                  "-b <write_buffer_size>\n"
            sys.exit()
-        elif opt == ("-d"):
+        elif opt == "-d":
            duration = int(arg)
-        elif opt == ("-t"):
+        elif opt == "-t":
            threads = int(arg)
-        elif opt == ("-k"):
+        elif opt == "-k":
            kill_random_test = int(arg)
-        elif opt == ("-i"):
-            interval = int(arg)
-        elif opt == ("-o"):
+        elif opt == "-o":
            ops_per_thread = int(arg)
-        elif opt == ("-b"):
+        elif opt == "-b":
            write_buf_size = int(arg)
        else:
            print "unrecognized option " + str(opt) + "\n"
            print "db_crashtest2.py -d <duration_test> -t <#threads> " \
-                "-k <kills with prob 1/k> -o <ops_per_thread> " \
-                "-b <write_buffer_size>\n"
+                  "-k <kills with prob 1/k> -o <ops_per_thread> " \
+                  "-b <write_buffer_size>\n"
            sys.exit(2)

    exit_time = time.time() + duration

-    dirpath = tempfile.mkdtemp()
+    print "Running whitebox-crash-test with \ntotal-duration=" + str(duration) \
+          + "\nthreads=" + str(threads) + "\nops_per_thread=" \
+          + str(ops_per_thread) + "\nwrite_buffer_size=" \
+          + str(write_buf_size) + "\n"

-    print("Running whitebox-crash-test with \ntotal-duration=" + str(duration)
-          + "\nthreads=" + str(threads) + "\nops_per_thread="
-          + str(ops_per_thread) + "\nwrite_buffer_size="
-          + str(write_buf_size) + "\n")
-
-    # kill in every alternate run. toggle tracks which run we are doing.
-    toggle = True
+    total_check_mode = 3
+    check_mode = 0

    while time.time() < exit_time:
-        run_had_errors = False
-        additional_opts = ' --disable_seek_compaction=' + \
-                          str(random.randint(0, 1)) + \
-                          ' --mmap_read=' + str(random.randint(0, 1)) + \
-                          ' --block_size=16384 ' + \
-                          ' --cache_size=1048576 ' + \
-                          ' --open_files=500000 ' + \
-                          ' --verify_checksum=1 ' + \
-                          ' --sync=' + str(random.randint(0, 1)) + \
-                          ' --disable_wal=0 ' + \
-                          ' --disable_data_sync=' + \
-                          str(random.randint(0, 1)) + \
-                          ' --target_file_size_base=2097152 ' + \
-                          ' --target_file_size_multiplier=2 ' + \
-                          ' --max_write_buffer_number=3 ' + \
-                          ' --max_background_compactions=20 ' + \
-                          ' --max_bytes_for_level_base=10485760 ' + \
-                          ' --filter_deletes=' + str(random.randint(0, 1))
-        print ("Running db_stress with additional options=\n"
-               + additional_opts + "\n")
-
-        if toggle:
-            # since we are going to kill anyway, use more ops per thread
-            new_ops_per_thread = 100 * ops_per_thread
-            killoption = '--kill_random_test=' + str(kill_random_test)
+        killoption = ""
+        if check_mode == 0:
+            # run with kill_random_test
+            killoption = " --kill_random_test=" + str(kill_random_test)
+            # use large ops per thread since we will kill it anyway
+            additional_opts = "--ops_per_thread=" + \
+                              str(100 * ops_per_thread) + killoption
+        elif check_mode == 1:
+            # normal run with universal compaction mode
+            additional_opts = "--ops_per_thread=" + str(ops_per_thread) + \
+                              " --compaction_style=1"
        else:
-            new_ops_per_thread = ops_per_thread
-            killoption = ''
+            # nomral run
+            additional_opts = "--ops_per_thread=" + str(ops_per_thread)

-        toggle = not toggle
+        cmd = re.sub('\s+', ' ', """
+            ./db_stress
+            --test_batches_snapshots=%s
+            --threads=%s
+            --write_buffer_size=%s
+            --destroy_db_initially=0
+            --reopen=0
+            --readpercent=50
+            --prefixpercent=5
+            --writepercent=40
+            --delpercent=5
+            --db=%s
+            --max_key=100000000
+            --disable_seek_compaction=%s
+            --mmap_read=%s
+            --block_size=16384
+            --cache_size=1048576
+            --open_files=500000
+            --verify_checksum=1
+            --sync=%s
+            --disable_wal=0
+            --disable_data_sync=%s
+            --target_file_size_base=2097152
+            --target_file_size_multiplier=2
+            --max_write_buffer_number=3
+            --max_background_compactions=20
+            --max_bytes_for_level_base=10485760
+            --filter_deletes=%s
+            %s
+            """ % (random.randint(0, 1),
+                   threads,
+                   write_buf_size,
+                   tempfile.mkdtemp(),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   random.randint(0, 1),
+                   additional_opts))

-        cmd = ['./db_stress \
-                --test_batches_snapshots=1 \
-                --ops_per_thread=0' + str(new_ops_per_thread) + ' \
-                --threads=0' + str(threads) + ' \
-                --write_buffer_size=' + str(write_buf_size) + ' \
-                --destroy_db_initially=0 ' + killoption + ' \
-                --reopen=0 \
-                --readpercent=50 \
-                --db=' + dirpath + ' \
-                --max_key=100000000 ' + additional_opts]
+        print "Running:" + cmd + "\n"

-        popen = subprocess.Popen(cmd, stdout=subprocess.PIPE,
+        popen = subprocess.Popen([cmd], stdout=subprocess.PIPE,
                                 stderr=subprocess.STDOUT,
                                 shell=True)
        stdoutdata, stderrdata = popen.communicate()
        retncode = popen.returncode
-        msg = ("kill option = {0}, exitcode = {1}".format(
-               killoption, retncode))
+        msg = ("check_mode={0}, kill option={1}, exitcode={2}\n".format(
+               check_mode, killoption, retncode))
        print msg
        print stdoutdata

@@ -143,6 +153,9 @@ def main(argv):
        if (stdoutdata.find('fail') >= 0):
            print "TEST FAILED. Output has 'fail'!!!\n"
            sys.exit(2)
+
+        check_mode = (check_mode + 1) % total_check_mode
+
        time.sleep(1)  # time to stabilize after a kill

 if __name__ == "__main__":
@@ -1,4 +1,3 @@
-
 #include <cstdio>

 #include "db/write_batch_internal.h"
@@ -19,7 +18,6 @@ using namespace leveldb;
 struct DataPumpThread {
  size_t no_records;
  DB* db; // Assumption DB is Open'ed already.
-  volatile bool is_running;
 };

 static std::string RandomString(Random* rnd, int len) {
@@ -33,59 +31,50 @@ static void DataPumpThreadBody(void* arg) {
  DB* db = t->db;
  Random rnd(301);
  size_t i = 0;
-  t->is_running = true;
-  while( i < t->no_records ) {
-    db->Put(WriteOptions(),
-            Slice(RandomString(&rnd, 50)),
-            Slice(RandomString(&rnd, 500)));
-    ++i;
+  while(i++ < t->no_records) {
+    if(!db->Put(WriteOptions(), Slice(RandomString(&rnd, 500)),
+                Slice(RandomString(&rnd, 500))).ok()) {
+      fprintf(stderr, "Error in put\n");
+      exit(1);
+    }
  }
-  t->is_running = false;
 }

 struct ReplicationThread {
  port::AtomicPointer stop;
  DB* db;
-  volatile SequenceNumber latest;
  volatile size_t no_read;
-  volatile bool has_more;
 };

 static void ReplicationThreadBody(void* arg) {
  ReplicationThread* t = reinterpret_cast<ReplicationThread*>(arg);
  DB* db = t->db;
  unique_ptr<TransactionLogIterator> iter;
-  SequenceNumber currentSeqNum = 0;
+  SequenceNumber currentSeqNum = 1;
  while (t->stop.Acquire_Load() != nullptr) {
-    if (!iter) {
-      db->GetUpdatesSince(currentSeqNum, &iter);
-      fprintf(stdout, "Refreshing iterator\n");
-      iter->Next();
-      while(iter->Valid()) {
-        BatchResult res = iter->GetBatch();
-        if (res.sequence != currentSeqNum +1
-            && res.sequence != currentSeqNum) {
-          fprintf(stderr,
-                  "Missed a seq no. b/w %ld and %ld\n",
-                  currentSeqNum,
-                  res.sequence);
-          exit(1);
-        }
-        currentSeqNum = res.sequence;
-        t->latest = res.sequence;
-        iter->Next();
-        t->no_read++;
+    iter.reset();
+    Status s;
+    while(!db->GetUpdatesSince(currentSeqNum, &iter).ok()) {
+      if (t->stop.Acquire_Load() == nullptr) {
+        return;
+      }
+    }
+    fprintf(stderr, "Refreshing iterator\n");
+    for(;iter->Valid(); iter->Next(), t->no_read++, currentSeqNum++) {
+      BatchResult res = iter->GetBatch();
+      if (res.sequence != currentSeqNum) {
+        fprintf(stderr, "Missed a seq no. b/w %ld and %ld\n", currentSeqNum,
+                        res.sequence);
+        exit(1);
      }
    }
-    iter.reset();
  }
 }

-
 int main(int argc, const char** argv) {

-  long FLAGS_num_inserts = 1000;
-  long FLAGS_WAL_ttl_seconds = 1000;
+  uint64_t FLAGS_num_inserts = 1000;
+  uint64_t FLAGS_WAL_ttl_seconds = 1000;
  char junk;
  long l;

@@ -108,36 +97,34 @@ int main(int argc, const char** argv) {
  options.create_if_missing = true;
  options.WAL_ttl_seconds = FLAGS_WAL_ttl_seconds;
  DB* db;
+  DestroyDB(default_db_path, options);

  Status s = DB::Open(options, default_db_path, &db);

  if (!s.ok()) {
    fprintf(stderr, "Could not open DB due to %s\n", s.ToString().c_str());
+    exit(1);
  }

  DataPumpThread dataPump;
  dataPump.no_records = FLAGS_num_inserts;
  dataPump.db = db;
-  dataPump.is_running = true;
  env->StartThread(DataPumpThreadBody, &dataPump);

  ReplicationThread replThread;
  replThread.db = db;
  replThread.no_read = 0;
-  replThread.has_more = true;
  replThread.stop.Release_Store(env); // store something to make it non-null.

  env->StartThread(ReplicationThreadBody, &replThread);
-  while(dataPump.is_running) {
-    continue;
-  }
+  while(replThread.no_read < FLAGS_num_inserts);
  replThread.stop.Release_Store(nullptr);
-  if ( replThread.no_read < dataPump.no_records ) {
+  if (replThread.no_read < dataPump.no_records) {
    // no. read should be => than inserted.
    fprintf(stderr, "No. of Record's written and read not same\nRead : %ld"
-            " Written : %ld", replThread.no_read, dataPump.no_records);
+            " Written : %ld\n", replThread.no_read, dataPump.no_records);
    exit(1);
  }
+  fprintf(stderr, "Successful!\n");
  exit(0);
-  fprintf(stdout, "ALL IS FINE");
 }
@@ -2,17 +2,18 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 //
-//The test uses an array to compare against values written to the database.
-//Keys written to the array are in 1:1 correspondence to the actual values in
-//the database according to the formula in the functino GenerateValue
+// The test uses an array to compare against values written to the database.
+// Keys written to the array are in 1:1 correspondence to the actual values in
+// the database according to the formula in the function GenerateValue.

-//Space is reserved in the array from 0 to FLAGS_max_key and values are randomly
-//written/deleted/read from those positions. During verification we compare all
-//the positions in the array. Therefore to shorten/elongate the amount of time
-//that this test runs for, you should change the settings:
-//FLAGS_max_key, FLAGS_ops_per_thread, (sometimes also FLAGS_threads)
-//NOTE that if FLAGS_test_batches_snapshots is set, the test behaves a little
-//differently. See comment header for the flag.
+// Space is reserved in the array from 0 to FLAGS_max_key and values are
+// randomly written/deleted/read from those positions. During verification we
+// compare all the positions in the array. To shorten/elongate the running
+// time, you could change the settings: FLAGS_max_key, FLAGS_ops_per_thread,
+// (sometimes also FLAGS_threads).
+//
+// NOTE that if FLAGS_test_batches_snapshots is set, the test will have
+// different behavior. See comment of the flag for details.

 #include <sys/types.h>
 #include <stdio.h>
@@ -26,6 +27,7 @@
 #include "leveldb/write_batch.h"
 #include "leveldb/statistics.h"
 #include "port/port.h"
+#include "util/coding.h"
 #include "util/crc32c.h"
 #include "util/histogram.h"
 #include "util/mutexlock.h"
@@ -34,6 +36,7 @@
 #include "util/logging.h"
 #include "utilities/ttl/db_ttl.h"
 #include "hdfs/env_hdfs.h"
+#include "utilities/merge_operators.h"

 static const long KB = 1024;

@@ -41,16 +44,17 @@ static const long KB = 1024;
 static uint32_t FLAGS_seed = 2341234;

 // Max number of key/values to place in database
-static long FLAGS_max_key = 2 * KB * KB * KB;
+static long FLAGS_max_key = 1 * KB * KB * KB;

-// If set, the test uses MultiGet, MultiPut and MultiDelete that
-// do a different kind of validation during the test itself,
-// rather than at the end. This is meant to solve the following
-// problems at the expense of doing less degree of validation.
+// If set, the test uses MultiGet(), MultiPut() and MultiDelete() which
+// read/write/delete multiple keys in a batch. In this mode, we do not verify
+// db content by comparing the content with the pre-allocated array. Instead,
+// we do partial verification inside MultiGet() by checking various values in
+// a batch. Benefit of this mode:
 // (a) No need to acquire mutexes during writes (less cache flushes
 //     in multi-core leading to speed up)
 // (b) No long validation at the end (more speed up)
-// (c) Also test snapshot and atomicity of batch writes
+// (c) Test snapshot and atomicity of batch writes
 static bool FLAGS_test_batches_snapshots = false;

 // Number of concurrent threads to run.
@@ -58,7 +62,7 @@ static int FLAGS_threads = 32;

 // Opens the db with this ttl value if this is not -1
 // Carefully specify a large value such that verifications on deleted
-//  values don't fail
+// values don't fail
 static int FLAGS_ttl = -1;

 // Size of each value will be this number times rand_int(1,3) bytes
@@ -88,6 +92,9 @@ static int FLAGS_max_write_buffer_number = 0;
 // This is initialized to default value of 1 in "main" function.
 static int FLAGS_max_background_compactions = 0;

+// This is initialized to default value of false
+static leveldb::CompactionStyle FLAGS_compaction_style = leveldb::kCompactionStyleLevel;
+
 // Number of bytes to use as a cache of uncompressed data.
 static long FLAGS_cache_size = 2 * KB * KB * KB;

@@ -132,16 +139,16 @@ extern int leveldb_kill_odds;
 // If true, do not write WAL for write.
 static bool FLAGS_disable_wal = false;

-// Target level-0 file size for compaction
+// Target level-1 file size for compaction
 static int FLAGS_target_file_size_base = 64 * KB;

-// A multiplier to compute targe level-N file size
+// A multiplier to compute targe level-N file size (N >= 2)
 static int FLAGS_target_file_size_multiplier = 1;

-// Max bytes for level-0
+// Max bytes for level-1
 static uint64_t FLAGS_max_bytes_for_level_base = 256 * KB;

-// A multiplier to compute max bytes for level-N
+// A multiplier to compute max bytes for level-N (N >= 2)
 static int FLAGS_max_bytes_for_level_multiplier = 2;

 // Number of files in level-0 that will trigger put stop.
@@ -153,8 +160,14 @@ static int FLAGS_level0_slowdown_writes_trigger = 8;
 // Ratio of reads to total workload (expressed as a percentage)
 static unsigned int FLAGS_readpercent = 10;

+// Ratio of prefix iterators to total workload (expressed as a percentage)
+static unsigned int FLAGS_prefixpercent = 25;
+
 // Ratio of deletes to total workload (expressed as a percentage)
-static unsigned int FLAGS_delpercent = 30;
+static unsigned int FLAGS_writepercent = 50;
+
+// Ratio of deletes to total workload (expressed as a percentage)
+static unsigned int FLAGS_delpercent = 15;

 // Option to disable compation triggered by read.
 static int FLAGS_disable_seek_compaction = false;
@@ -186,8 +199,24 @@ static bool FLAGS_filter_deletes = false;
 // Level0 compaction start trigger
 static int FLAGS_level0_file_num_compaction_trigger = 0;

+// On true, replaces all writes with a Merge that behaves like a Put
+static bool FLAGS_use_merge_put = false;
+
 namespace leveldb {

+// convert long to a big-endian slice key
+static std::string Key(long val) {
+  std::string little_endian_key;
+  std::string big_endian_key;
+  PutFixed64(&little_endian_key, val);
+  assert(little_endian_key.size() == sizeof(val));
+  big_endian_key.resize(sizeof(val));
+  for (int i=0; i<(int)sizeof(val); i++) {
+    big_endian_key[i] = little_endian_key[sizeof(val) - 1 - i];
+  }
+  return big_endian_key;
+}
+
 class StressTest;
 namespace {

@@ -197,9 +226,11 @@ class Stats {
  double finish_;
  double seconds_;
  long done_;
+  long gets_;
+  long prefixes_;
  long writes_;
  long deletes_;
-  long gets_;
+  long iterator_size_sums_;
  long founds_;
  long errors_;
  int next_report_;
@@ -214,9 +245,11 @@ class Stats {
    next_report_ = 100;
    hist_.Clear();
    done_ = 0;
+    gets_ = 0;
+    prefixes_ = 0;
    writes_ = 0;
    deletes_ = 0;
-    gets_ = 0;
+    iterator_size_sums_ = 0;
    founds_ = 0;
    errors_ = 0;
    bytes_ = 0;
@@ -229,9 +262,11 @@ class Stats {
  void Merge(const Stats& other) {
    hist_.Merge(other.hist_);
    done_ += other.done_;
+    gets_ += other.gets_;
+    prefixes_ += other.prefixes_;
    writes_ += other.writes_;
    deletes_ += other.deletes_;
-    gets_ += other.gets_;
+    iterator_size_sums_ += other.iterator_size_sums_;
    founds_ += other.founds_;
    errors_ += other.errors_;
    bytes_ += other.bytes_;
@@ -274,15 +309,20 @@ class Stats {
    bytes_ += nbytes;
  }

-  void AddDeletes(int n) {
-    deletes_ += n;
-  }
-
  void AddGets(int ngets, int nfounds) {
    founds_ += nfounds;
    gets_ += ngets;
  }

+  void AddPrefixes(int nprefixes, int count) {
+    prefixes_ += nprefixes;
+    iterator_size_sums_ += count;
+  }
+
+  void AddDeletes(int n) {
+    deletes_ += n;
+  }
+
  void AddErrors(int n) {
    errors_ += n;
  }
@@ -306,7 +346,11 @@ class Stats {
            "", bytes_mb, rate, (100*writes_)/done_, done_);
    fprintf(stdout, "%-12s: Wrote %ld times\n", "", writes_);
    fprintf(stdout, "%-12s: Deleted %ld times\n", "", deletes_);
-    fprintf(stdout, "%-12s: %ld/%ld gets found the key\n", "", founds_, gets_);
+    fprintf(stdout, "%-12s: %ld read and %ld found the key\n", "",
+            gets_, founds_);
+    fprintf(stdout, "%-12s: Prefix scanned %ld times\n", "", prefixes_);
+    fprintf(stdout, "%-12s: Iterator size sum is %ld\n", "",
+            iterator_size_sums_);
    fprintf(stdout, "%-12s: Got errors %ld times\n", "", errors_);

    if (FLAGS_histogram) {
@@ -489,6 +533,9 @@ class StressTest {
        filter_policy_(FLAGS_bloom_bits >= 0
                       ? NewBloomFilterPolicy(FLAGS_bloom_bits)
                       : nullptr),
+        prefix_extractor_(NewFixedPrefixTransform(
+                          FLAGS_test_batches_snapshots ?
+                          sizeof(long) : sizeof(long)-1)),
        db_(nullptr),
        num_times_reopened_(0) {
    if (FLAGS_destroy_db_initially) {
@@ -506,6 +553,7 @@ class StressTest {
  ~StressTest() {
    delete db_;
    delete filter_policy_;
+    delete prefix_extractor_;
  }

  void Run() {
@@ -633,7 +681,11 @@ class StressTest {
      keys[i] += key.ToString();
      values[i] += value.ToString();
      value_slices[i] = values[i];
-      batch.Put(keys[i], value_slices[i]);
+      if (FLAGS_use_merge_put) {
+        batch.Merge(keys[i], value_slices[i]);
+      } else {
+        batch.Put(keys[i], value_slices[i]);
+      }
    }

    s = db_->Write(writeoptions, &batch);
@@ -648,7 +700,6 @@ class StressTest {
    return s;
  }

-
  // Given a key K, this deletes ("0"+K), ("1"+K),... ("9"+K)
  // in DB atomically i.e in a single batch. Also refer MultiGet.
  Status MultiDelete(ThreadState* thread,
@@ -730,6 +781,82 @@ class StressTest {
    return s;
  }

+  // Given a prefix P, this does prefix scans for "0"+P, "1"+P,..."9"+P
+  // in the same snapshot.  Each of these 10 scans returns a series of
+  // values; each series should be the same length, and it is verified
+  // for each index i that all the i'th values are of the form "0"+V,
+  // "1"+V,..."9"+V.
+  // ASSUMES that MultiPut was used to put (K, V)
+  Status MultiPrefixScan(ThreadState* thread,
+                         const ReadOptions& readoptions,
+                         const Slice& prefix) {
+    std::string prefixes[10] = {"0", "1", "2", "3", "4",
+                                "5", "6", "7", "8", "9"};
+    Slice prefix_slices[10];
+    ReadOptions readoptionscopy[10];
+    const Snapshot* snapshot = db_->GetSnapshot();
+    Iterator* iters[10];
+    Status s = Status::OK();
+    for (int i = 0; i < 10; i++) {
+      prefixes[i] += prefix.ToString();
+      prefix_slices[i] = prefixes[i];
+      readoptionscopy[i] = readoptions;
+      readoptionscopy[i].prefix = &prefix_slices[i];
+      readoptionscopy[i].snapshot = snapshot;
+      iters[i] = db_->NewIterator(readoptionscopy[i]);
+      iters[i]->SeekToFirst();
+    }
+
+    int count = 0;
+    while (iters[0]->Valid()) {
+      count++;
+      std::string values[10];
+      // get list of all values for this iteration
+      for (int i = 0; i < 10; i++) {
+        // no iterator should finish before the first one
+        assert(iters[i]->Valid());
+        values[i] = iters[i]->value().ToString();
+
+        char expected_first = (prefixes[i])[0];
+        char actual_first = (values[i])[0];
+
+        if (actual_first != expected_first) {
+          fprintf(stderr, "error expected first = %c actual = %c\n",
+                  expected_first, actual_first);
+        }
+        (values[i])[0] = ' '; // blank out the differing character
+      }
+      // make sure all values are equivalent
+      for (int i = 0; i < 10; i++) {
+        if (values[i] != values[0]) {
+          fprintf(stderr, "error : inconsistent values for prefix %s: %s, %s\n",
+                  prefix.ToString().c_str(), values[0].c_str(),
+                  values[i].c_str());
+          // we continue after error rather than exiting so that we can
+          // find more errors if any
+        }
+        iters[i]->Next();
+      }
+    }
+
+    // cleanup iterators and snapshot
+    for (int i = 0; i < 10; i++) {
+      // if the first iterator finished, they should have all finished
+      assert(!iters[i]->Valid());
+      assert(iters[i]->status().ok());
+      delete iters[i];
+    }
+    db_->ReleaseSnapshot(snapshot);
+
+    if (s.ok()) {
+      thread->stats.AddPrefixes(1, count);
+    } else {
+      thread->stats.AddErrors(1);
+    }
+
+    return s;
+  }
+
  void OperateDb(ThreadState* thread) {
    ReadOptions read_opts(FLAGS_verify_checksum, true);
    WriteOptions write_opts;
@@ -761,11 +888,12 @@ class StressTest {
      }

      long rand_key = thread->rand.Next() % max_key;
-      Slice key((char*)&rand_key, sizeof(rand_key));
-      //Read:10%;Delete:30%;Write:60%
-      unsigned int probability_operation = thread->rand.Uniform(100);
-      if (probability_operation < FLAGS_readpercent) {
-        // read load
+      std::string keystr = Key(rand_key);
+      Slice key = keystr;
+      int prob_op = thread->rand.Uniform(100);
+
+      // OPERATION read?
+      if (prob_op >= 0 && prob_op < (int)FLAGS_readpercent) {
        if (!FLAGS_test_batches_snapshots) {
          Status s = db_->Get(read_opts, key, &from_db);
          if (s.ok()) {
@@ -781,19 +909,38 @@ class StressTest {
        } else {
          MultiGet(thread, read_opts, key, &from_db);
        }
-      } else if (probability_operation < FLAGS_delpercent + FLAGS_readpercent) {
-        //introduce delete load
-        if (!FLAGS_test_batches_snapshots) {
-          MutexLock l(thread->shared->GetMutexForKey(rand_key));
-          thread->shared->Delete(rand_key);
-          db_->Delete(write_opts, key);
-          thread->stats.AddDeletes(1);
-        } else {
-          MultiDelete(thread, write_opts, key);
-        }
+      }
+      prob_op -= FLAGS_readpercent;

-      } else {
-        // write load
+      // OPERATION prefix scan?
+      if (prob_op >= 0 && prob_op < (int)FLAGS_prefixpercent) {
+        // keys are longs (e.g., 8 bytes), so we let prefixes be
+        // everything except the last byte.  So there will be 2^8=256
+        // keys per prefix.
+        Slice prefix = Slice(key.data(), key.size() - 1);
+        if (!FLAGS_test_batches_snapshots) {
+          read_opts.prefix = &prefix;
+          Iterator* iter = db_->NewIterator(read_opts);
+          int count = 0;
+          for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+            assert(iter->key().starts_with(prefix));
+            count++;
+          }
+          assert(count <= 256);
+          if (iter->status().ok()) {
+            thread->stats.AddPrefixes(1, count);
+          } else {
+            thread->stats.AddErrors(1);
+          }
+          delete iter;
+        } else {
+          MultiPrefixScan(thread, read_opts, prefix);
+        }
+      }
+      prob_op -= FLAGS_prefixpercent;
+
+      // OPERATION write?
+      if (prob_op >= 0 && prob_op < (int)FLAGS_writepercent) {
        uint32_t value_base = thread->rand.Next();
        size_t sz = GenerateValue(value_base, value, sizeof(value));
        Slice v(value, sz);
@@ -803,14 +950,32 @@ class StressTest {
            VerifyValue(rand_key, read_opts, *(thread->shared), &from_db, true);
          }
          thread->shared->Put(rand_key, value_base);
-          db_->Put(write_opts, key, v);
+          if (FLAGS_use_merge_put) {
+            db_->Merge(write_opts, key, v);
+          } else {
+            db_->Put(write_opts, key, v);
+          }
          thread->stats.AddBytesForWrites(1, sz);
        } else {
          MultiPut(thread, write_opts, key, v, sz);
        }
-
        PrintKeyValue(rand_key, value, sz);
      }
+      prob_op -= FLAGS_writepercent;
+
+      // OPERATION delete?
+      if (prob_op >= 0 && prob_op < (int)FLAGS_delpercent) {
+        if (!FLAGS_test_batches_snapshots) {
+          MutexLock l(thread->shared->GetMutexForKey(rand_key));
+          thread->shared->Delete(rand_key);
+          db_->Delete(write_opts, key);
+          thread->stats.AddDeletes(1);
+        } else {
+          MultiDelete(thread, write_opts, key);
+        }
+      }
+      prob_op -= FLAGS_delpercent;
+
      thread->stats.FinishedSingleOp();
    }
    thread->stats.Stop();
@@ -837,7 +1002,8 @@ class StressTest {

  void VerifyValue(long key, const ReadOptions &opts, const SharedState &shared,
                   std::string *value_from_db, bool strict=false) const {
-    Slice k((char*)&key, sizeof(key));
+    std::string keystr = Key(key);
+    Slice k = keystr;
    char value[100];
    uint32_t value_base = shared.Get(key);
    if (value_base == SharedState::SENTINEL && !strict) {
@@ -893,15 +1059,18 @@ class StressTest {
    }
    fprintf(stdout, "Time to live(sec)   : %s\n", ttl_state.c_str());
    fprintf(stdout, "Read percentage     : %d\n", FLAGS_readpercent);
+    fprintf(stdout, "Prefix percentage   : %d\n", FLAGS_prefixpercent);
+    fprintf(stdout, "Write percentage    : %d\n", FLAGS_writepercent);
+    fprintf(stdout, "Delete percentage   : %d\n", FLAGS_delpercent);
    fprintf(stdout, "Write-buffer-size   : %d\n", FLAGS_write_buffer_size);
    fprintf(stdout, "Delete percentage   : %d\n", FLAGS_delpercent);
    fprintf(stdout, "Max key             : %ld\n", FLAGS_max_key);
-    fprintf(stdout, "Ratio #ops/#keys    : %ld\n",
-            (FLAGS_ops_per_thread * FLAGS_threads)/FLAGS_max_key);
+    fprintf(stdout, "Ratio #ops/#keys    : %f\n",
+            (1.0 * FLAGS_ops_per_thread * FLAGS_threads)/FLAGS_max_key);
    fprintf(stdout, "Num times DB reopens: %d\n", FLAGS_reopen);
    fprintf(stdout, "Batches/snapshots   : %d\n",
            FLAGS_test_batches_snapshots);
-    fprintf(stdout, "Purge redundant %%  : %d\n",
+    fprintf(stdout, "Purge redundant %%   : %d\n",
            FLAGS_purge_redundant_percent);
    fprintf(stdout, "Deletes use filter  : %d\n",
            FLAGS_filter_deletes);
@@ -935,8 +1104,10 @@ class StressTest {
    options.write_buffer_size = FLAGS_write_buffer_size;
    options.max_write_buffer_number = FLAGS_max_write_buffer_number;
    options.max_background_compactions = FLAGS_max_background_compactions;
+    options.compaction_style = FLAGS_compaction_style;
    options.block_size = FLAGS_block_size;
    options.filter_policy = filter_policy_;
+    options.prefix_extractor = prefix_extractor_;
    options.max_open_files = FLAGS_open_files;
    options.statistics = dbstats;
    options.env = FLAGS_env;
@@ -966,6 +1137,10 @@ class StressTest {
      options.purge_redundant_kvs_while_flush = false;
    }

+    if (FLAGS_use_merge_put) {
+      options.merge_operator = MergeOperators::CreatePutOperator();
+    }
+
    fprintf(stdout, "DB path: [%s]\n", FLAGS_db);

    Status s;
@@ -1008,6 +1183,7 @@ class StressTest {
 private:
  shared_ptr<Cache> cache_;
  const FilterPolicy* filter_policy_;
+  const SliceTransform* prefix_extractor_;
  DB* db_;
  StackableDB* sdb_;
  int num_times_reopened_;
@@ -1021,6 +1197,8 @@ int main(int argc, char** argv) {
  FLAGS_open_files = leveldb::Options().max_open_files;
  FLAGS_max_background_compactions =
    leveldb::Options().max_background_compactions;
+  FLAGS_compaction_style =
+    leveldb::Options().compaction_style;
  FLAGS_level0_file_num_compaction_trigger =
    leveldb::Options().level0_file_num_compaction_trigger;
  FLAGS_level0_slowdown_writes_trigger =
@@ -1073,6 +1251,8 @@ int main(int argc, char** argv) {
      FLAGS_max_write_buffer_number = n;
    } else if (sscanf(argv[i], "--max_background_compactions=%d%c", &n, &junk) == 1) {
      FLAGS_max_background_compactions = n;
+    } else if (sscanf(argv[i], "--compaction_style=%d%c", &n, &junk) == 1) {
+      FLAGS_compaction_style = (leveldb::CompactionStyle)n;
    } else if (sscanf(argv[i], "--cache_size=%ld%c", &l, &junk) == 1) {
      FLAGS_cache_size = l;
    } else if (sscanf(argv[i], "--block_size=%d%c", &n, &junk) == 1) {
@@ -1102,6 +1282,12 @@ int main(int argc, char** argv) {
    } else if (sscanf(argv[i], "--readpercent=%d%c", &n, &junk) == 1 &&
               (n >= 0 && n <= 100)) {
      FLAGS_readpercent = n;
+    } else if (sscanf(argv[i], "--prefixpercent=%d%c", &n, &junk) == 1 &&
+               (n >= 0 && n <= 100)) {
+      FLAGS_prefixpercent = n;
+    } else if (sscanf(argv[i], "--writepercent=%d%c", &n, &junk) == 1 &&
+               (n >= 0 && n <= 100)) {
+      FLAGS_writepercent = n;
    } else if (sscanf(argv[i], "--delpercent=%d%c", &n, &junk) == 1 &&
               (n >= 0 && n <= 100)) {
      FLAGS_delpercent = n;
@@ -1165,6 +1351,9 @@ int main(int argc, char** argv) {
    } else if (sscanf(argv[i], "--filter_deletes=%d%c", &n, &junk)
        == 1 && (n == 0 || n == 1)) {
      FLAGS_filter_deletes = n;
+    } else if (sscanf(argv[i], "--use_merge=%d%c", &n, &junk)
+        == 1 && (n == 0 || n == 1)) {
+      FLAGS_use_merge_put = n;
    } else {
      fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
      exit(1);
@@ -1175,8 +1364,9 @@ int main(int argc, char** argv) {
  // max number of concurrent compactions.
  FLAGS_env->SetBackgroundThreads(FLAGS_max_background_compactions);

-  if ((FLAGS_readpercent + FLAGS_delpercent) > 100) {
-      fprintf(stderr, "Error: Read + Delete percents > 100!\n");
+  if ((FLAGS_readpercent + FLAGS_prefixpercent +
+       FLAGS_writepercent + FLAGS_delpercent) != 100) {
+      fprintf(stderr, "Error: Read+Prefix+Write+Delete percents != 100!\n");
      exit(1);
  }
  if (FLAGS_disable_wal == 1 && FLAGS_reopen > 0) {
@@ -14,6 +14,7 @@
 #include "table/block.h"
 #include "table/block_builder.h"
 #include "table/format.h"
+#include "util/ldb_cmd.h"
 #include "util/random.h"
 #include "util/testharness.h"
 #include "util/testutil.h"
@@ -21,11 +22,17 @@
 namespace leveldb {

 class SstFileReader {
-public:
-  explicit SstFileReader(std::string file_name,
-                bool verify_checksum = false,
-                bool output_hex = false);
-  Status ReadSequential(bool print_kv, uint64_t read_num = -1);
+ public:
+  explicit SstFileReader(const std::string& file_name,
+                         bool verify_checksum,
+                         bool output_hex);
+
+  Status ReadSequential(bool print_kv,
+                        uint64_t read_num,
+                        bool has_from,
+                        const std::string& from_key,
+                        bool has_to,
+                        const std::string& to_key);

  uint64_t GetReadNumber() { return read_num_; }

@@ -37,17 +44,25 @@ private:
  EnvOptions soptions_;
 };

-SstFileReader::SstFileReader(std::string file_path,
+SstFileReader::SstFileReader(const std::string& file_path,
                             bool verify_checksum,
                             bool output_hex)
 :file_name_(file_path), read_num_(0), verify_checksum_(verify_checksum),
  output_hex_(output_hex) {
+  std::cout << "Process " << file_path << "\n";
 }

-Status SstFileReader::ReadSequential(bool print_kv, uint64_t read_num)
+Status SstFileReader::ReadSequential(bool print_kv,
+                                     uint64_t read_num,
+                                     bool has_from,
+                                     const std::string& from_key,
+                                     bool has_to,
+                                     const std::string& to_key)
 {
  unique_ptr<Table> table;
+  InternalKeyComparator internal_comparator_(BytewiseComparator());
  Options table_options;
+  table_options.comparator = &internal_comparator_;
  unique_ptr<RandomAccessFile> file;
  Status s = table_options.env->NewRandomAccessFile(file_name_, &file,
                                                    soptions_);
@@ -63,17 +78,38 @@ Status SstFileReader::ReadSequential(bool print_kv, uint64_t read_num)

  Iterator* iter = table->NewIterator(ReadOptions(verify_checksum_, false));
  uint64_t i = 0;
-  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+  if (has_from) {
+    InternalKey ikey(from_key, kMaxSequenceNumber, kValueTypeForSeek);
+    iter->Seek(ikey.Encode());
+  } else {
+    iter->SeekToFirst();
+  }
+  for (; iter->Valid(); iter->Next()) {
    Slice key = iter->key();
    Slice value = iter->value();
    ++i;
    if (read_num > 0 && i > read_num)
      break;
-    if (print_kv) {
-      fprintf(stdout, "%s ==> %s\n",
-              key.ToString(output_hex_).c_str(),
-              value.ToString(output_hex_).c_str());
+
+    ParsedInternalKey ikey;
+    if (!ParseInternalKey(key, &ikey)) {
+      std::cerr << "Internal Key ["
+                << key.ToString(true /* in hex*/)
+                << "] parse error!\n";
+      continue;
    }
+
+    // If end marker was specified, we stop before it
+    if (has_to && BytewiseComparator()->Compare(ikey.user_key, to_key) >= 0) {
+      break;
+    }
+
+    if (print_kv) {
+      std::cout << ikey.DebugString(output_hex_)
+                << " => "
+                << value.ToString(output_hex_) << "\n";
+    }
+
   }

   read_num_ += i;
@@ -90,9 +126,29 @@ static void print_help() {
      "sst_dump [--command=check|scan] [--verify_checksum] "
      "--file=data_dir_OR_sst_file"
      " [--output_hex]"
+      " [--input_key_hex]"
+      " [--from=<user_key>]"
+      " [--to=<user_key>]"
      " [--read_num=NUM]\n");
 }

+string HexToString(const string& str) {
+  string parsed;
+  if (str[0] != '0' || str[1] != 'x') {
+    fprintf(stderr, "Invalid hex input %s.  Must start with 0x\n",
+            str.c_str());
+    throw "Invalid hex input";
+  }
+
+  for (unsigned int i = 2; i < str.length();) {
+    int c;
+    sscanf(str.c_str() + i, "%2X", &c);
+    parsed.push_back(c);
+    i += 2;
+  }
+  return parsed;
+}
+
 int main(int argc, char** argv) {

  const char* dir_or_file = nullptr;
@@ -103,24 +159,47 @@ int main(int argc, char** argv) {
  uint64_t n;
  bool verify_checksum = false;
  bool output_hex = false;
+  bool input_key_hex = false;
+  bool has_from = false;
+  bool has_to = false;
+  std::string from_key;
+  std::string to_key;
  for (int i = 1; i < argc; i++)
  {
    if (strncmp(argv[i], "--file=", 7) == 0) {
      dir_or_file = argv[i] + 7;
    } else if (strcmp(argv[i], "--output_hex") == 0) {
      output_hex = true;
+    } else if (strcmp(argv[i], "--input_key_hex") == 0) {
+      input_key_hex = true;
    } else if (sscanf(argv[i], "--read_num=%ld%c", &n, &junk) == 1) {
      read_num = n;
    } else if (strcmp(argv[i], "--verify_checksum") == 0) {
      verify_checksum = true;
    } else if (strncmp(argv[i], "--command=", 10) == 0) {
      command = argv[i] + 10;
-    } else {
+    } else if (strncmp(argv[i], "--from=", 7) == 0) {
+      from_key = argv[i] + 7;
+      has_from = true;
+    } else if (strncmp(argv[i], "--to=", 5) == 0) {
+      to_key = argv[i] + 5;
+      has_to = true;
+    }else {
      print_help();
      exit(1);
    }
  }

+
+  if (input_key_hex) {
+    if (has_from) {
+      from_key = HexToString(from_key);
+    }
+    if (has_to) {
+      to_key = HexToString(to_key);
+    }
+  }
+
  if(dir_or_file == nullptr) {
    print_help();
    exit(1);
@@ -136,6 +215,9 @@ int main(int argc, char** argv) {
    dir = false;
  }

+  std::cout << "from [" << leveldb::Slice(from_key).ToString(true)
+            << "] to [" << leveldb::Slice(to_key).ToString(true) << "]\n";
+
  uint64_t total_read = 0;
  for (size_t i = 0; i < filenames.size(); i++) {
    std::string filename = filenames.at(i);
@@ -145,7 +227,7 @@ int main(int argc, char** argv) {
      continue;
    }
    if(dir) {
-      filename = std::string(dir_or_file) + "//" + filename;
+      filename = std::string(dir_or_file) + "/" + filename;
    }
    leveldb::SstFileReader reader(filename, verify_checksum,
                                  output_hex);
@@ -153,7 +235,9 @@ int main(int argc, char** argv) {
    // scan all files in give file path.
    if (command == "" || command == "scan" || command == "check") {
      st = reader.ReadSequential(command != "check",
-          read_num > 0 ? (read_num - total_read) : read_num);
+                                 read_num > 0 ? (read_num - total_read) :
+                                                read_num,
+                                 has_from, from_key, has_to, to_key);
      if (!st.ok()) {
        fprintf(stderr, "%s: %s\n", filename.c_str(),
            st.ToString().c_str());
@@ -0,0 +1,43 @@
+// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "leveldb/slice_transform.h"
+
+#include "leveldb/slice.h"
+
+namespace leveldb {
+
+namespace {
+
+class FixedPrefixTransform : public SliceTransform {
+ private:
+  size_t prefix_len_;
+
+ public:
+  explicit FixedPrefixTransform(size_t prefix_len) : prefix_len_(prefix_len) { }
+
+  virtual const char* Name() const {
+    return "rocksdb.FixedPrefix";
+  }
+
+  virtual Slice Transform(const Slice& src) const {
+    assert(InDomain(src));
+    return Slice(src.data(), prefix_len_);
+  }
+
+  virtual bool InDomain(const Slice& src) const {
+    return (src.size() >= prefix_len_);
+  }
+
+  virtual bool InRange(const Slice& dst) const {
+    return (dst.size() == prefix_len_);
+  }
+};
+}
+
+const SliceTransform* NewFixedPrefixTransform(size_t prefix_len) {
+  return new FixedPrefixTransform(prefix_len);
+}
+
+}  // namespace leveldb
@@ -578,6 +578,7 @@ void PrintBucketCounts(const vector<uint64_t>& bucket_counts, int ttl_start,

 const string InternalDumpCommand::ARG_COUNT_ONLY = "count_only";
 const string InternalDumpCommand::ARG_STATS = "stats";
+const string InternalDumpCommand::ARG_INPUT_KEY_HEX = "input_key_hex";

 InternalDumpCommand::InternalDumpCommand(const vector<string>& params,
                                         const map<string, string>& options,
@@ -585,12 +586,14 @@ InternalDumpCommand::InternalDumpCommand(const vector<string>& params,
    LDBCommand(options, flags, true,
               BuildCmdLineOptions({ ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
                                     ARG_FROM, ARG_TO, ARG_MAX_KEYS,
-                                     ARG_COUNT_ONLY, ARG_STATS})),
+                                     ARG_COUNT_ONLY, ARG_STATS,
+                                     ARG_INPUT_KEY_HEX})),
    has_from_(false),
    has_to_(false),
    max_keys_(-1),
    count_only_(false),
-    print_stats_(false) {
+    print_stats_(false),
+    is_input_key_hex_(false) {

  has_from_ = ParseStringOption(options, ARG_FROM, &from_);
  has_to_ = ParseStringOption(options, ARG_TO, &to_);
@@ -599,8 +602,9 @@ InternalDumpCommand::InternalDumpCommand(const vector<string>& params,

  print_stats_ = IsFlagPresent(flags, ARG_STATS);
  count_only_ = IsFlagPresent(flags, ARG_COUNT_ONLY);
+  is_input_key_hex_ = IsFlagPresent(flags, ARG_INPUT_KEY_HEX);

-  if (is_key_hex_) {
+  if (is_input_key_hex_) {
    if (has_from_) {
      from_ = HexToString(from_);
    }
@@ -614,6 +618,7 @@ void InternalDumpCommand::Help(string& ret) {
  ret.append("  ");
  ret.append(InternalDumpCommand::Name());
  ret.append(HelpRangeCmdArgs());
+  ret.append(" [--" + ARG_INPUT_KEY_HEX + "]");
  ret.append(" [--" + ARG_MAX_KEYS + "=<N>]");
  ret.append(" [--" + ARG_COUNT_ONLY + "]");
  ret.append(" [--" + ARG_STATS + "]");
@@ -674,7 +679,7 @@ void InternalDumpCommand::DoCommand() {
    if (!count_only_) {
      string key = ikey.DebugString(is_key_hex_);
      string value = iter->value().ToString(is_value_hex_);
-      fprintf(stdout, "%s => %s\n", key.data(), value.data());
+      std::cout << key << " => " << value << "\n";
    }

    // Terminate if maximum number of keys have been dumped
@@ -1197,7 +1202,7 @@ void ApproxSizeCommand::DoCommand() {
  uint64_t sizes[1];
  db_->GetApproximateSizes(ranges, 1, sizes);
  fprintf(stdout, "%ld\n", sizes[0]);
-  /* Wierd that GetApproximateSizes() returns void, although documentation
+  /* Weird that GetApproximateSizes() returns void, although documentation
   * says that it returns a Status object.
  if (!st.ok()) {
    exec_state_ = LDBCommandExecuteResult::FAILED(st.ToString());
@@ -424,9 +424,11 @@ private:
  int max_keys_;
  bool count_only_;
  bool print_stats_;
+  bool is_input_key_hex_;

  static const string ARG_COUNT_ONLY;
  static const string ARG_STATS;
+  static const string ARG_INPUT_KEY_HEX;
 };

 class DBLoaderCommand: public LDBCommand {
@@ -33,6 +33,8 @@ Options::Options()
      block_restart_interval(16),
      compression(kSnappyCompression),
      filter_policy(nullptr),
+      prefix_extractor(nullptr),
+      whole_key_filtering(true),
      num_levels(7),
      level0_file_num_compaction_trigger(4),
      level0_slowdown_writes_trigger(8),
@@ -78,8 +80,13 @@ Options::Options()
      access_hint_on_compaction_start(NORMAL),
      use_adaptive_mutex(false),
      bytes_per_sync(0),
+      compaction_style(kCompactionStyleLevel),
      filter_deletes(false),
-      memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)) {
+      memtable_factory(std::shared_ptr<SkipListFactory>(new SkipListFactory)),
+      compaction_filter_factory(
+          std::shared_ptr<CompactionFilterFactory>(
+            new DefaultCompactionFilterFactory())) {
+
  assert(memtable_factory.get() != nullptr);
 }

@@ -95,7 +102,10 @@ Options::Dump(Logger* log) const
        merge_operator? merge_operator->Name() : "None");
    Log(log,"       Options.compaction_filter: %s",
        compaction_filter? compaction_filter->Name() : "None");
+    Log(log,"       Options.compaction_filter_factory: %s",
+        compaction_filter_factory->Name());
    Log(log,"         Options.error_if_exists: %d", error_if_exists);
+    Log(log,"       Options.create_if_missing: %d", create_if_missing);
    Log(log,"         Options.paranoid_checks: %d", paranoid_checks);
    Log(log,"                     Options.env: %p", env);
    Log(log,"                Options.info_log: %p", info_log.get());
@@ -119,6 +129,9 @@ Options::Dump(Logger* log) const
    }
    Log(log,"         Options.filter_policy: %s",
        filter_policy == nullptr ? "nullptr" : filter_policy->Name());
+    Log(log,"      Options.prefix_extractor: %s",
+        prefix_extractor == nullptr ? "nullptr" : prefix_extractor->Name());
+    Log(log,"   Options.whole_key_filtering: %d", whole_key_filtering);
    Log(log,"            Options.num_levels: %d", num_levels);
    Log(log,"       Options.disableDataSync: %d", disableDataSync);
    Log(log,"             Options.use_fsync: %d", use_fsync);
@@ -182,9 +195,11 @@ Options::Dump(Logger* log) const
        delete_obsolete_files_period_micros);
    Log(log,"             Options.max_background_compactions: %d",
        max_background_compactions);
+    Log(log,"                      Options.soft_rate_limit: %.2f",
+        soft_rate_limit);
    Log(log,"                      Options.hard_rate_limit: %.2f",
        hard_rate_limit);
-    Log(log,"      Options.rate_limit_delay_max_milliseconds: %d",
+    Log(log,"      Options.rate_limit_delay_max_milliseconds: %u",
        rate_limit_delay_max_milliseconds);
    Log(log,"               Options.disable_auto_compactions: %d",
        disable_auto_compactions);
@@ -204,7 +219,7 @@ Options::Dump(Logger* log) const
        is_fd_close_on_exec);
    Log(log,"              Options.skip_log_error_on_recovery: %d",
        skip_log_error_on_recovery);
-    Log(log,"                   Options.stats_dump_period_sec: %d",
+    Log(log,"                   Options.stats_dump_period_sec: %u",
        stats_dump_period_sec);
    Log(log,"                    Options.block_size_deviation: %d",
        block_size_deviation);
@@ -218,6 +233,14 @@ Options::Dump(Logger* log) const
        bytes_per_sync);
    Log(log,"                          Options.filter_deletes: %d",
        filter_deletes);
+    Log(log,"                        Options.compaction_style: %d",
+        compaction_style);
+    Log(log,"        Options.compaction_options_universal.size_ratio: %u",
+        compaction_options_universal.size_ratio);
+    Log(log,"   Options.compaction_options_universal.min_merge_width: %u",
+        compaction_options_universal.min_merge_width);
+    Log(log,"   Options.compaction_options_universal.max_merge_width: %u",
+        compaction_options_universal.max_merge_width);
 }   // Options::Dump

 //
@@ -2,6 +2,7 @@
 #define MERGE_OPERATORS_H

 #include <memory>
+#include <stdio.h>

 #include "leveldb/merge_operator.h"

@@ -9,10 +10,31 @@ namespace leveldb {

 class MergeOperators {
 public:
-  static std::shared_ptr<leveldb::MergeOperator> CreatePutOperator();
-  static std::shared_ptr<leveldb::MergeOperator> CreateUInt64AddOperator();
+  static std::shared_ptr<MergeOperator> CreatePutOperator();
+  static std::shared_ptr<MergeOperator> CreateUInt64AddOperator();
+  static std::shared_ptr<MergeOperator> CreateStringAppendOperator();
+  static std::shared_ptr<MergeOperator> CreateStringAppendTESTOperator();
+
+  // Will return a different merge operator depending on the string.
+  // TODO: Hook the "name" up to the actual Name() of the MergeOperators?
+  static std::shared_ptr<MergeOperator> CreateFromStringId(
+      const std::string& name) {
+    if (name == "put") {
+      return CreatePutOperator();
+    } else if ( name == "uint64add") {
+      return CreateUInt64AddOperator();
+    } else if (name == "stringappend") {
+      return CreateStringAppendOperator();
+    } else if (name == "stringappendtest") {
+      return CreateStringAppendTESTOperator();
+    } else {
+      // Empty or unknown, just return nullptr
+      return nullptr;
+    }
+  }
+
 };

-}
+} // namespace leveldb

 #endif
@@ -17,11 +17,11 @@ namespace { // anonymous namespace
 // From the client-perspective, semantics are the same.
 class PutOperator : public MergeOperator {
 public:
-  virtual bool Merge(const Slice& key,
-                     const Slice* existing_value,
-                     const std::deque<std::string>& operand_sequence,
-                     std::string* new_value,
-                     Logger* logger) const override {
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_sequence,
+                         std::string* new_value,
+                         Logger* logger) const override {
    // Put basically only looks at the current/latest value
    assert(!operand_sequence.empty());
    assert(new_value != nullptr);
@@ -50,6 +50,10 @@ const char* StringAppendOperator::Name() const  {
  return "StringAppendOperator";
 }

+std::shared_ptr<MergeOperator> MergeOperators::CreateStringAppendOperator() {
+  return std::make_shared<StringAppendOperator>(',');
+}
+
 } // namespace leveldb


@@ -11,7 +11,6 @@ namespace leveldb {

 class StringAppendOperator : public AssociativeMergeOperator {
 public:
-
  StringAppendOperator(char delim_char);    /// Constructor: specify delimiter

  virtual bool Merge(const Slice& key,
@@ -20,11 +20,12 @@ StringAppendTESTOperator::StringAppendTESTOperator(char delim_char)
 }

 // Implementation for the merge operation (concatenates two strings)
-bool StringAppendTESTOperator::Merge(const Slice& key,
-                                     const Slice* existing_value,
-                                     const std::deque<std::string>& operands,
-                                     std::string* new_value,
-                                     Logger* logger) const {
+bool StringAppendTESTOperator::FullMerge(
+    const Slice& key,
+    const Slice* existing_value,
+    const std::deque<std::string>& operands,
+    std::string* new_value,
+    Logger* logger) const {

  // Clear the *new_value for writing.
  assert(new_value);
@@ -66,25 +67,38 @@ bool StringAppendTESTOperator::PartialMerge(const Slice& key,
                                            std::string* new_value,
                                            Logger* logger) const {
  return false;
+}

-//  // Clear the *new_value for writing.
-//  assert(new_value);
-//  new_value->clear();
-//
-//  // Generic append
-//  // Reserve correct size for *new_value, and apply concatenation.
-//  new_value->reserve(left_operand.size() + 1 + right_operand.size());
-//  new_value->assign(left_operand.data(), left_operand.size());
-//  new_value->append(1,delim_);
-//  new_value->append(right_operand.data(), right_operand.size());
-//
-//  return true;
+// A version of PartialMerge that actually performs "partial merging".
+// Use this to simulate the exact behaviour of the StringAppendOperator.
+bool StringAppendTESTOperator::_AssocPartialMerge(const Slice& key,
+                                            const Slice& left_operand,
+                                            const Slice& right_operand,
+                                            std::string* new_value,
+                                            Logger* logger) const {
+  // Clear the *new_value for writing.
+  assert(new_value);
+  new_value->clear();

+  // Generic append
+  // Reserve correct size for *new_value, and apply concatenation.
+  new_value->reserve(left_operand.size() + 1 + right_operand.size());
+  new_value->assign(left_operand.data(), left_operand.size());
+  new_value->append(1,delim_);
+  new_value->append(right_operand.data(), right_operand.size());
+
+  return true;
 }

 const char* StringAppendTESTOperator::Name() const  {
  return "StringAppendTESTOperator";
 }

+
+std::shared_ptr<MergeOperator>
+MergeOperators::CreateStringAppendTESTOperator() {
+  return std::make_shared<StringAppendTESTOperator>(',');
+}
+
 } // namespace leveldb

@@ -20,11 +20,11 @@ class StringAppendTESTOperator : public MergeOperator {

  StringAppendTESTOperator(char delim_char);    /// Constructor with delimiter

-  virtual bool Merge(const Slice& key,
-                     const Slice* existing_value,
-                     const std::deque<std::string>& operand_sequence,
-                     std::string* new_value,
-                     Logger* logger) const override;
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operand_sequence,
+                         std::string* new_value,
+                         Logger* logger) const override;

  virtual bool PartialMerge(const Slice& key,
                            const Slice& left_operand,
@@ -35,6 +35,14 @@ class StringAppendTESTOperator : public MergeOperator {
  virtual const char* Name() const override;

 private:
+  // A version of PartialMerge that actually performs "partial merging".
+  // Use this to simulate the exact behaviour of the StringAppendOperator.
+  bool _AssocPartialMerge(const Slice& key,
+                          const Slice& left_operand,
+                          const Slice& right_operand,
+                          std::string* new_value,
+                          Logger* logger) const;
+
  char delim_;         // The delimiter is inserted between elements

 };
@@ -13,6 +13,8 @@
 #include "leveldb/merge_operator.h"
 #include "utilities/merge_operators.h"
 #include "utilities/merge_operators/string_append/stringappend.h"
+#include "utilities/merge_operators/string_append/stringappend2.h"
+#include "utilities/ttl/db_ttl.h"
 #include "util/testharness.h"
 #include "util/random.h"

@@ -24,21 +26,29 @@ namespace leveldb {
 const std::string kDbName = "/tmp/mergetestdb";

 // OpenDb opens a (possibly new) rocksdb database with a StringAppendOperator
-std::shared_ptr<DB> OpenDb(StringAppendOperator* append_op) {
-    DB* db;
-    Options options;
-    options.create_if_missing = true;
-    options.merge_operator = append_op;
-    Status s = DB::Open(options, kDbName,  &db);
-    if (!s.ok()) {
-      std::cerr << s.ToString() << std::endl;
-      assert(false);
-    }
-    return std::shared_ptr<DB>(db);
+std::shared_ptr<DB> OpenNormalDb(char delim_char) {
+  DB* db;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator.reset(new StringAppendOperator(delim_char));
+  ASSERT_OK(DB::Open(options, kDbName,  &db));
+  return std::shared_ptr<DB>(db);
+}
+
+// Open a TtlDB with a non-associative StringAppendTESTOperator
+std::shared_ptr<DB> OpenTtlDb(char delim_char) {
+  StackableDB* db;
+  Options options;
+  options.create_if_missing = true;
+  options.merge_operator.reset(new StringAppendTESTOperator(delim_char));
+  Status s;
+  db = new DBWithTTL(123456, options, kDbName, s, false);
+  ASSERT_OK(s);
+  return std::shared_ptr<DB>(db);
 }

 /// StringLists represents a set of string-lists, each with a key-index.
-/// Supports Append(list,string) and Get(list)
+/// Supports Append(list, string) and Get(list)
 class StringLists {
 public:

@@ -52,8 +62,8 @@ class StringLists {

  // Append string val onto the list defined by key; return true on success
  bool Append(const std::string& key, const std::string& val){
-    Slice valSlice(val.data(),val.size());
-    auto s = db_->Merge(merge_option_,key,valSlice);
+    Slice valSlice(val.data(), val.size());
+    auto s = db_->Merge(merge_option_, key, valSlice);

    if (s.ok()) {
      return true;
@@ -66,7 +76,6 @@ class StringLists {
  // Returns the list of strings associated with key (or "" if does not exist)
  bool Get(const std::string& key, std::string* const result){
    assert(result != NULL); // we should have a place to store the result
-
    auto s = db_->Get(get_option_, key, result);

    if (s.ok()) {
@@ -86,6 +95,7 @@ class StringLists {
    return false;
  }

+
 private:
  std::shared_ptr<DB> db_;
  WriteOptions merge_option_;
@@ -93,105 +103,202 @@ class StringLists {

 };

+
+// The class for unit-testing
+class StringAppendOperatorTest {
+ public:
+  StringAppendOperatorTest() {
+    DestroyDB(kDbName, Options());    // Start each test with a fresh DB
+  }
+
+  typedef std::shared_ptr<DB> (* OpenFuncPtr)(char);
+
+  // Allows user to open databases with different configurations.
+  // e.g.: Can open a DB or a TtlDB, etc.
+  static void SetOpenDbFunction(OpenFuncPtr func) {
+    OpenDb = func;
+  }
+
+ protected:
+  static OpenFuncPtr OpenDb;
+};
+StringAppendOperatorTest::OpenFuncPtr StringAppendOperatorTest::OpenDb = nullptr;
+
 // THE TEST CASES BEGIN HERE

-class StringAppendOperatorTest { };
+TEST(StringAppendOperatorTest, IteratorTest) {
+  auto db_ = OpenDb(',');
+  StringLists slists(db_);

-TEST(StringAppendOperatorTest,SimpleTest) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");

-  StringAppendOperator append_op(',');
-  auto db = OpenDb(&append_op);
-  StringLists slists(db);
-
-  slists.Append("k1","v1");
-  slists.Append("k1","v2");
-  slists.Append("k1","v3");
+  slists.Append("k2", "a1");
+  slists.Append("k2", "a2");
+  slists.Append("k2", "a3");

  std::string res;
-  bool status = slists.Get("k1",&res);
+  std::unique_ptr<leveldb::Iterator> it(db_->NewIterator(ReadOptions()));
+  std::string k1("k1");
+  std::string k2("k2");
+  bool first = true;
+  for (it->Seek(k1); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3");
+    }
+  }
+  slists.Append("k2", "a4");
+  slists.Append("k1", "v4");
+
+  // Snapshot should still be the same. Should ignore a4 and v4.
+  first = true;
+  for (it->Seek(k1); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3");
+    }
+  }
+
+
+  // Should release the snapshot and be aware of the new stuff now
+  it.reset(db_->NewIterator(ReadOptions()));
+  first = true;
+  for (it->Seek(k1); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3,v4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+    }
+  }
+
+  // start from k2 this time.
+  for (it->Seek(k2); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "v1,v2,v3,v4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+    }
+  }
+
+  slists.Append("k3", "g1");
+
+  it.reset(db_->NewIterator(ReadOptions()));
+  first = true;
+  std::string k3("k3");
+  for(it->Seek(k2); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "g1");
+    }
+  }
+  for(it->Seek(k3); it->Valid(); it->Next()) {
+    res = it->value().ToString();
+    if (first) {
+      // should not be hit
+      ASSERT_EQ(res, "a1,a2,a3,a4");
+      first = false;
+    } else {
+      ASSERT_EQ(res, "g1");
+    }
+  }
+
+}
+
+TEST(StringAppendOperatorTest, SimpleTest) {
+  auto db = OpenDb(',');
+  StringLists slists(db);
+
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");
+
+  std::string res;
+  bool status = slists.Get("k1", &res);

  ASSERT_TRUE(status);
-  ASSERT_EQ(res,"v1,v2,v3");
+  ASSERT_EQ(res, "v1,v2,v3");
 }

-TEST(StringAppendOperatorTest,SimpleDelimiterTest) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op('|');
-  auto db = OpenDb(&append_op);
+TEST(StringAppendOperatorTest, SimpleDelimiterTest) {
+  auto db = OpenDb('|');
  StringLists slists(db);

-  slists.Append("k1","v1");
-  slists.Append("k1","v2");
-  slists.Append("k1","v3");
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");

  std::string res;
-  slists.Get("k1",&res);
-  ASSERT_EQ(res,"v1|v2|v3");
+  slists.Get("k1", &res);
+  ASSERT_EQ(res, "v1|v2|v3");
 }

-TEST(StringAppendOperatorTest,OneValueNoDelimiterTest) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op('!');
-  auto db = OpenDb(&append_op);
+TEST(StringAppendOperatorTest, OneValueNoDelimiterTest) {
+  auto db = OpenDb('!');
  StringLists slists(db);

-  slists.Append("random_key","single_val");
+  slists.Append("random_key", "single_val");

  std::string res;
-  slists.Get("random_key",&res);
-  ASSERT_EQ(res,"single_val");
+  slists.Get("random_key", &res);
+  ASSERT_EQ(res, "single_val");
 }

-TEST(StringAppendOperatorTest,VariousKeys) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op('\n');
-  auto db = OpenDb(&append_op);
+TEST(StringAppendOperatorTest, VariousKeys) {
+  auto db = OpenDb('\n');
  StringLists slists(db);

-  slists.Append("c","asdasd");
-  slists.Append("a","x");
-  slists.Append("b","y");
-  slists.Append("a","t");
-  slists.Append("a","r");
-  slists.Append("b","2");
-  slists.Append("c","asdasd");
+  slists.Append("c", "asdasd");
+  slists.Append("a", "x");
+  slists.Append("b", "y");
+  slists.Append("a", "t");
+  slists.Append("a", "r");
+  slists.Append("b", "2");
+  slists.Append("c", "asdasd");

-  std::string a,b,c;
-  bool sa,sb,sc;
-  sa = slists.Get("a",&a);
-  sb = slists.Get("b",&b);
-  sc = slists.Get("c",&c);
+  std::string a, b, c;
+  bool sa, sb, sc;
+  sa = slists.Get("a", &a);
+  sb = slists.Get("b", &b);
+  sc = slists.Get("c", &c);

  ASSERT_TRUE(sa && sb && sc); // All three keys should have been found

-  ASSERT_EQ(a,"x\nt\nr");
-  ASSERT_EQ(b,"y\n2");
-  ASSERT_EQ(c,"asdasd\nasdasd");
+  ASSERT_EQ(a, "x\nt\nr");
+  ASSERT_EQ(b, "y\n2");
+  ASSERT_EQ(c, "asdasd\nasdasd");
 }

 // Generate semi random keys/words from a small distribution.
-TEST(StringAppendOperatorTest,RandomMixGetAppend) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op(' ');
-  auto db = OpenDb(&append_op);
+TEST(StringAppendOperatorTest, RandomMixGetAppend) {
+  auto db = OpenDb(' ');
  StringLists slists(db);

  // Generate a list of random keys and values
  const int kWordCount = 15;
-  std::string words[] = {"sdasd","triejf","fnjsdfn","dfjisdfsf","342839",
-                         "dsuha","mabuais","sadajsid","jf9834hf","2d9j89",
-                         "dj9823jd","a","dk02ed2dh","$(jd4h984$(*", "mabz"};
+  std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
+                         "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
+                         "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
  const int kKeyCount = 6;
-  std::string keys[] = {"dhaiusdhu","denidw","daisda","keykey","muki",
+  std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
                        "shzassdianmd"};

  // Will store a local copy of all data in order to verify correctness
-  std::map<std::string,std::string> parallel_copy;
+  std::map<std::string, std::string> parallel_copy;

  // Generate a bunch of random queries (Append and Get)!
  enum query_t  { APPEND_OP, GET_OP, NUM_OPS };
@@ -204,14 +311,11 @@ TEST(StringAppendOperatorTest,RandomMixGetAppend) {
    std::string key = keys[randomGen.Uniform((int)kKeyCount)];
    std::string word = words[randomGen.Uniform((int)kWordCount)];

-    // Debug message.
-    //std::cout << (int)query << " " << key << " " << word << std::endl;
-
    // Apply the query and any checks.
    if (query == APPEND_OP) {

      // Apply the rocksdb test-harness Append defined above
-      slists.Append(key,word);  //apply the rocksdb append
+      slists.Append(key, word);  //apply the rocksdb append

      // Apply the similar "Append" to the parallel copy
      if (parallel_copy[key].size() > 0) {
@@ -223,32 +327,29 @@ TEST(StringAppendOperatorTest,RandomMixGetAppend) {
    } else if (query == GET_OP) {
      // Assumes that a non-existent key just returns <empty>
      std::string res;
-      slists.Get(key,&res);
-      ASSERT_EQ(res,parallel_copy[key]);
+      slists.Get(key, &res);
+      ASSERT_EQ(res, parallel_copy[key]);
    }

  }

 }

-TEST(StringAppendOperatorTest,BIGRandomMixGetAppend) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op(' ');
-  auto db = OpenDb(&append_op);
+TEST(StringAppendOperatorTest, BIGRandomMixGetAppend) {
+  auto db = OpenDb(' ');
  StringLists slists(db);

  // Generate a list of random keys and values
  const int kWordCount = 15;
-  std::string words[] = {"sdasd","triejf","fnjsdfn","dfjisdfsf","342839",
-                         "dsuha","mabuais","sadajsid","jf9834hf","2d9j89",
-                         "dj9823jd","a","dk02ed2dh","$(jd4h984$(*", "mabz"};
+  std::string words[] = {"sdasd", "triejf", "fnjsdfn", "dfjisdfsf", "342839",
+                         "dsuha", "mabuais", "sadajsid", "jf9834hf", "2d9j89",
+                         "dj9823jd", "a", "dk02ed2dh", "$(jd4h984$(*", "mabz"};
  const int kKeyCount = 6;
-  std::string keys[] = {"dhaiusdhu","denidw","daisda","keykey","muki",
+  std::string keys[] = {"dhaiusdhu", "denidw", "daisda", "keykey", "muki",
                        "shzassdianmd"};

  // Will store a local copy of all data in order to verify correctness
-  std::map<std::string,std::string> parallel_copy;
+  std::map<std::string, std::string> parallel_copy;

  // Generate a bunch of random queries (Append and Get)!
  enum query_t  { APPEND_OP, GET_OP, NUM_OPS };
@@ -261,14 +362,11 @@ TEST(StringAppendOperatorTest,BIGRandomMixGetAppend) {
    std::string key = keys[randomGen.Uniform((int)kKeyCount)];
    std::string word = words[randomGen.Uniform((int)kWordCount)];

-    // Debug message.
-    //std::cout << (int)query << " " << key << " " << word << std::endl;
-
    //Apply the query and any checks.
    if (query == APPEND_OP) {

      // Apply the rocksdb test-harness Append defined above
-      slists.Append(key,word);  //apply the rocksdb append
+      slists.Append(key, word);  //apply the rocksdb append

      // Apply the similar "Append" to the parallel copy
      if (parallel_copy[key].size() > 0) {
@@ -280,8 +378,8 @@ TEST(StringAppendOperatorTest,BIGRandomMixGetAppend) {
    } else if (query == GET_OP) {
      // Assumes that a non-existent key just returns <empty>
      std::string res;
-      slists.Get(key,&res);
-      ASSERT_EQ(res,parallel_copy[key]);
+      slists.Get(key, &res);
+      ASSERT_EQ(res, parallel_copy[key]);
    }

  }
@@ -289,191 +387,179 @@ TEST(StringAppendOperatorTest,BIGRandomMixGetAppend) {
 }


-TEST(StringAppendOperatorTest,PersistentVariousKeys) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
+TEST(StringAppendOperatorTest, PersistentVariousKeys) {
  // Perform the following operations in limited scope
  {
-    StringAppendOperator append_op('\n');
-    auto db = OpenDb(&append_op);
+    auto db = OpenDb('\n');
    StringLists slists(db);

-    slists.Append("c","asdasd");
-    slists.Append("a","x");
-    slists.Append("b","y");
-    slists.Append("a","t");
-    slists.Append("a","r");
-    slists.Append("b","2");
-    slists.Append("c","asdasd");
+    slists.Append("c", "asdasd");
+    slists.Append("a", "x");
+    slists.Append("b", "y");
+    slists.Append("a", "t");
+    slists.Append("a", "r");
+    slists.Append("b", "2");
+    slists.Append("c", "asdasd");

-    std::string a,b,c;
-    slists.Get("a",&a);
-    slists.Get("b",&b);
-    slists.Get("c",&c);
+    std::string a, b, c;
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);

-    ASSERT_EQ(a,"x\nt\nr");
-    ASSERT_EQ(b,"y\n2");
-    ASSERT_EQ(c,"asdasd\nasdasd");
+    ASSERT_EQ(a, "x\nt\nr");
+    ASSERT_EQ(b, "y\n2");
+    ASSERT_EQ(c, "asdasd\nasdasd");
  }

  // Reopen the database (the previous changes should persist / be remembered)
  {
-    StringAppendOperator append_op('\n');
-    auto db = OpenDb(&append_op);
+    auto db = OpenDb('\n');
    StringLists slists(db);

-    slists.Append("c","bbnagnagsx");
-    slists.Append("a","sa");
-    slists.Append("b","df");
-    slists.Append("a","gh");
-    slists.Append("a","jk");
-    slists.Append("b","l;");
-    slists.Append("c","rogosh");
+    slists.Append("c", "bbnagnagsx");
+    slists.Append("a", "sa");
+    slists.Append("b", "df");
+    slists.Append("a", "gh");
+    slists.Append("a", "jk");
+    slists.Append("b", "l;");
+    slists.Append("c", "rogosh");

    // The previous changes should be on disk (L0)
    // The most recent changes should be in memory (MemTable)
    // Hence, this will test both Get() paths.
-    std::string a,b,c;
-    slists.Get("a",&a);
-    slists.Get("b",&b);
-    slists.Get("c",&c);
+    std::string a, b, c;
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);

-    ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk");
-    ASSERT_EQ(b,"y\n2\ndf\nl;");
-    ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh");
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
  }

  // Reopen the database (the previous changes should persist / be remembered)
  {
-    StringAppendOperator append_op('\n');
-    auto db = OpenDb(&append_op);
+    auto db = OpenDb('\n');
    StringLists slists(db);

    // All changes should be on disk. This will test VersionSet Get()
-    std::string a,b,c;
-    slists.Get("a",&a);
-    slists.Get("b",&b);
-    slists.Get("c",&c);
+    std::string a, b, c;
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);

-    ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk");
-    ASSERT_EQ(b,"y\n2\ndf\nl;");
-    ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh");
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");
  }
 }

-TEST(StringAppendOperatorTest,PersistentFlushAndCompaction) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op('\n');
-
+TEST(StringAppendOperatorTest, PersistentFlushAndCompaction) {
  // Perform the following operations in limited scope
  {
-    auto db = OpenDb(&append_op);
+    auto db = OpenDb('\n');
    StringLists slists(db);
-    std::string a,b,c;
+    std::string a, b, c;
    bool success;

    // Append, Flush, Get
-    slists.Append("c","asdasd");
+    slists.Append("c", "asdasd");
    db->Flush(leveldb::FlushOptions());
-    success = slists.Get("c",&c);
+    success = slists.Get("c", &c);
    ASSERT_TRUE(success);
-    ASSERT_EQ(c,"asdasd");
+    ASSERT_EQ(c, "asdasd");

    // Append, Flush, Append, Get
-    slists.Append("a","x");
-    slists.Append("b","y");
+    slists.Append("a", "x");
+    slists.Append("b", "y");
    db->Flush(leveldb::FlushOptions());
-    slists.Append("a","t");
-    slists.Append("a","r");
-    slists.Append("b","2");
+    slists.Append("a", "t");
+    slists.Append("a", "r");
+    slists.Append("b", "2");

-    success = slists.Get("a",&a);
+    success = slists.Get("a", &a);
    assert(success == true);
-    ASSERT_EQ(a,"x\nt\nr");
+    ASSERT_EQ(a, "x\nt\nr");

-    success = slists.Get("b",&b);
+    success = slists.Get("b", &b);
    assert(success == true);
-    ASSERT_EQ(b,"y\n2");
+    ASSERT_EQ(b, "y\n2");

    // Append, Get
-    success = slists.Append("c","asdasd");
+    success = slists.Append("c", "asdasd");
    assert(success);
-    success = slists.Append("b","monkey");
+    success = slists.Append("b", "monkey");
    assert(success);

    // I omit the "assert(success)" checks here.
-    slists.Get("a",&a);
-    slists.Get("b",&b);
-    slists.Get("c",&c);
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);

-    ASSERT_EQ(a,"x\nt\nr");
-    ASSERT_EQ(b,"y\n2\nmonkey");
-    ASSERT_EQ(c,"asdasd\nasdasd");
+    ASSERT_EQ(a, "x\nt\nr");
+    ASSERT_EQ(b, "y\n2\nmonkey");
+    ASSERT_EQ(c, "asdasd\nasdasd");
  }

  // Reopen the database (the previous changes should persist / be remembered)
  {
-    auto db = OpenDb(&append_op);
+    auto db = OpenDb('\n');
    StringLists slists(db);
-    std::string a,b,c;
+    std::string a, b, c;

    // Get (Quick check for persistence of previous database)
-    slists.Get("a",&a);
-    ASSERT_EQ(a,"x\nt\nr");
+    slists.Get("a", &a);
+    ASSERT_EQ(a, "x\nt\nr");

    //Append, Compact, Get
-    slists.Append("c","bbnagnagsx");
-    slists.Append("a","sa");
-    slists.Append("b","df");
-    db->CompactRange(nullptr,nullptr);
-    slists.Get("a",&a);
-    slists.Get("b",&b);
-    slists.Get("c",&c);
-    ASSERT_EQ(a,"x\nt\nr\nsa");
-    ASSERT_EQ(b,"y\n2\nmonkey\ndf");
-    ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx");
+    slists.Append("c", "bbnagnagsx");
+    slists.Append("a", "sa");
+    slists.Append("b", "df");
+    db->CompactRange(nullptr, nullptr);
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+    ASSERT_EQ(a, "x\nt\nr\nsa");
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx");

    // Append, Get
-    slists.Append("a","gh");
-    slists.Append("a","jk");
-    slists.Append("b","l;");
-    slists.Append("c","rogosh");
-    slists.Get("a",&a);
-    slists.Get("b",&b);
-    slists.Get("c",&c);
-    ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk");
-    ASSERT_EQ(b,"y\n2\nmonkey\ndf\nl;");
-    ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh");
+    slists.Append("a", "gh");
+    slists.Append("a", "jk");
+    slists.Append("b", "l;");
+    slists.Append("c", "rogosh");
+    slists.Get("a", &a);
+    slists.Get("b", &b);
+    slists.Get("c", &c);
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");

    // Compact, Get
-    db->CompactRange(nullptr,nullptr);
-    ASSERT_EQ(a,"x\nt\nr\nsa\ngh\njk");
-    ASSERT_EQ(b,"y\n2\nmonkey\ndf\nl;");
-    ASSERT_EQ(c,"asdasd\nasdasd\nbbnagnagsx\nrogosh");
+    db->CompactRange(nullptr, nullptr);
+    ASSERT_EQ(a, "x\nt\nr\nsa\ngh\njk");
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;");
+    ASSERT_EQ(c, "asdasd\nasdasd\nbbnagnagsx\nrogosh");

    // Append, Flush, Compact, Get
-    slists.Append("b","afcg");
+    slists.Append("b", "afcg");
    db->Flush(leveldb::FlushOptions());
-    db->CompactRange(nullptr,nullptr);
-    slists.Get("b",&b);
-    ASSERT_EQ(b,"y\n2\nmonkey\ndf\nl;\nafcg");
+    db->CompactRange(nullptr, nullptr);
+    slists.Get("b", &b);
+    ASSERT_EQ(b, "y\n2\nmonkey\ndf\nl;\nafcg");
  }
 }

-TEST(StringAppendOperatorTest,SimpleTestNullDelimiter) {
-  DestroyDB(kDbName, Options());    // Start this test with a fresh DB
-
-  StringAppendOperator append_op('\0');
-  auto db = OpenDb(&append_op);
+TEST(StringAppendOperatorTest, SimpleTestNullDelimiter) {
+  auto db = OpenDb('\0');
  StringLists slists(db);

-  slists.Append("k1","v1");
-  slists.Append("k1","v2");
-  slists.Append("k1","v3");
+  slists.Append("k1", "v1");
+  slists.Append("k1", "v2");
+  slists.Append("k1", "v3");

  std::string res;
-  bool status = slists.Get("k1",&res);
+  bool status = slists.Get("k1", &res);
  ASSERT_TRUE(status);

  // Construct the desired string. Default constructor doesn't like '\0' chars.
@@ -484,13 +570,25 @@ TEST(StringAppendOperatorTest,SimpleTestNullDelimiter) {

  // Check that the leveldb result string matches the desired string
  assert(res.size() == checker.size());
-  ASSERT_EQ(res,checker);
+  ASSERT_EQ(res, checker);
 }

-
 } // namespace leveldb

 int main(int arc, char** argv) {
-  leveldb::test::RunAllTests();
+  // Run with regular database
+  {
+    fprintf(stderr, "Running tests with regular db and operator.\n");
+    StringAppendOperatorTest::SetOpenDbFunction(&OpenNormalDb);
+    leveldb::test::RunAllTests();
+  }
+
+  // Run with TTL
+  {
+    fprintf(stderr, "Running tests with ttl db and generic operator.\n");
+    StringAppendOperatorTest::SetOpenDbFunction(&OpenTtlDb);
+    leveldb::test::RunAllTests();
+  }
+
  return 0;
 }
@@ -20,13 +20,20 @@ DBWithTTL::DBWithTTL(const int32_t ttl,
      ttl_(ttl) {
  Options options_to_open = options;

-  ttl_comp_filter_.reset(new TtlCompactionFilter(ttl,
-                                                 options.compaction_filter));
-  options_to_open.compaction_filter = ttl_comp_filter_.get();
+  if (options.compaction_filter) {
+    ttl_comp_filter_.reset(
+        new TtlCompactionFilter(ttl, options.compaction_filter));
+    options_to_open.compaction_filter = ttl_comp_filter_.get();
+  } else {
+    options_to_open.compaction_filter_factory =
+      std::shared_ptr<CompactionFilterFactory>(
+          new TtlCompactionFilterFactory(
+            ttl, options.compaction_filter_factory));
+  }

  if (options.merge_operator) {
-    ttl_merge_op_.reset(new TtlMergeOperator(options.merge_operator));
-    options_to_open.merge_operator = ttl_merge_op_.get();
+    options_to_open.merge_operator.reset(
+      new TtlMergeOperator(options.merge_operator));
  }

  if (read_only) {
@@ -147,7 +154,7 @@ std::vector<Status> DBWithTTL::MultiGet(const ReadOptions& options,
                               supported with TTL"));
 }

-bool DBWithTTL::KeyMayExist(ReadOptions& options,
+bool DBWithTTL::KeyMayExist(const ReadOptions& options,
                            const Slice& key,
                            std::string* value,
                            bool* value_found) {
@@ -259,6 +266,14 @@ SequenceNumber DBWithTTL::GetLatestSequenceNumber() {
  return db_->GetLatestSequenceNumber();
 }

+Status DBWithTTL::GetSortedWalFiles(VectorLogPtr& files) {
+  return db_->GetSortedWalFiles(files);
+}
+
+Status DBWithTTL::DeleteWalFiles(const VectorLogPtr& files){
+  return db_->DeleteWalFiles(files);
+}
+
 Status DBWithTTL::GetUpdatesSince(
    SequenceNumber seq_number,
    unique_ptr<TransactionLogIterator>* iter) {
@@ -36,10 +36,10 @@ class DBWithTTL : public StackableDB {
                                       const std::vector<Slice>& keys,
                                       std::vector<std::string>* values);

-  virtual bool KeyMayExist(ReadOptions& options,
+  virtual bool KeyMayExist(const ReadOptions& options,
                           const Slice& key,
                           std::string* value,
-                           bool* value_found = nullptr);
+                           bool* value_found = nullptr) override;

  virtual Status Delete(const WriteOptions& wopts, const Slice& key);

@@ -77,6 +77,10 @@ class DBWithTTL : public StackableDB {

  virtual Status GetLiveFiles(std::vector<std::string>& vec, uint64_t* mfs);

+  virtual Status GetSortedWalFiles(VectorLogPtr& files);
+
+  virtual Status DeleteWalFiles(const VectorLogPtr& files);
+
  virtual SequenceNumber GetLatestSequenceNumber();

  virtual Status GetUpdatesSince(SequenceNumber seq_number,
@@ -108,7 +112,6 @@ class DBWithTTL : public StackableDB {
 private:
  DB* db_;
  int32_t ttl_;
-  unique_ptr<MergeOperator> ttl_merge_op_;
  unique_ptr<CompactionFilter> ttl_comp_filter_;
 };

@@ -176,11 +179,19 @@ class TtlIterator : public Iterator {
 class TtlCompactionFilter : public CompactionFilter {

 public:
-  TtlCompactionFilter(int32_t ttl, const CompactionFilter* comp_filter)
+  TtlCompactionFilter(
+      int32_t ttl,
+      const CompactionFilter* user_comp_filter,
+      std::unique_ptr<const CompactionFilter>
+      user_comp_filter_from_factory = nullptr)
    : ttl_(ttl),
-      user_comp_filter_(comp_filter) {
+      user_comp_filter_(user_comp_filter),
+      user_comp_filter_from_factory_(std::move(user_comp_filter_from_factory)) {
    // Unlike the merge operator, compaction filter is necessary for TTL, hence
    // this would be called even if user doesn't specify any compaction-filter
+    if (!user_comp_filter_) {
+      user_comp_filter_ = user_comp_filter_from_factory_.get();
+    }
  }

  virtual bool Filter(int level,
@@ -215,47 +226,78 @@ class TtlCompactionFilter : public CompactionFilter {
 private:
  int32_t ttl_;
  const CompactionFilter* user_comp_filter_;
+  std::unique_ptr<const CompactionFilter> user_comp_filter_from_factory_;
+};
+
+class TtlCompactionFilterFactory : public CompactionFilterFactory {
+  public:
+    TtlCompactionFilterFactory(
+        int32_t ttl,
+        std::shared_ptr<CompactionFilterFactory> comp_filter_factory)
+    : ttl_(ttl),
+      user_comp_filter_factory_(comp_filter_factory) { }
+
+    virtual std::unique_ptr<CompactionFilter> CreateCompactionFilter() {
+      return std::unique_ptr<TtlCompactionFilter>(
+          new TtlCompactionFilter(
+            ttl_,
+            nullptr,
+            std::move(user_comp_filter_factory_->CreateCompactionFilter())));
+    }
+
+    virtual const char* Name() const override {
+      return "TtlCompactionFilterFactory";
+    }
+
+  private:
+    int32_t ttl_;
+    std::shared_ptr<CompactionFilterFactory> user_comp_filter_factory_;
 };

 class TtlMergeOperator : public MergeOperator {

 public:
-  explicit TtlMergeOperator(const MergeOperator* merge_op)
+  explicit TtlMergeOperator(const std::shared_ptr<MergeOperator> merge_op)
    : user_merge_op_(merge_op) {
    assert(merge_op);
  }

-  virtual bool Merge(const Slice& key,
-                     const Slice* existing_value,
-                     const std::deque<std::string>& operands,
-                     std::string* new_value,
-                     Logger* logger) const override {
+  virtual bool FullMerge(const Slice& key,
+                         const Slice* existing_value,
+                         const std::deque<std::string>& operands,
+                         std::string* new_value,
+                         Logger* logger) const override {
    const uint32_t ts_len = DBWithTTL::kTSLength;
    if (existing_value && existing_value->size() < ts_len) {
      Log(logger, "Error: Could not remove timestamp from existing value.");
      return false;
-      // TODO: Change Merge semantics and add a counter here
    }

    // Extract time-stamp from each operand to be passed to user_merge_op_
    std::deque<std::string> operands_without_ts;
-    for (auto it = operands.begin(); it != operands.end(); ++it) {
-      if (it->size() < ts_len) {
+    for (const auto &operand : operands) {
+      if (operand.size() < ts_len) {
        Log(logger, "Error: Could not remove timestamp from operand value.");
        return false;
      }
-      operands_without_ts.push_back(it->substr(0, it->size() - ts_len));
+      operands_without_ts.push_back(operand.substr(0, operand.size() - ts_len));
    }

    // Apply the user merge operator (store result in *new_value)
+    bool good = true;
    if (existing_value) {
      Slice existing_value_without_ts(existing_value->data(),
                                      existing_value->size() - ts_len);
-      user_merge_op_->Merge(key, &existing_value_without_ts,
-                            operands_without_ts, new_value, logger);
+      good = user_merge_op_->FullMerge(key, &existing_value_without_ts,
+                                       operands_without_ts, new_value, logger);
    } else {
-      user_merge_op_->Merge(key, nullptr, operands_without_ts, new_value,
-                            logger);
+      good = user_merge_op_->FullMerge(key, nullptr, operands_without_ts,
+                                       new_value, logger);
+    }
+
+    // Return false if the user merge operator returned false
+    if (!good) {
+      return false;
    }

    // Augment the *new_value with the ttl time-stamp
@@ -282,15 +324,16 @@ class TtlMergeOperator : public MergeOperator {
    if (left_operand.size() < ts_len || right_operand.size() < ts_len) {
      Log(logger, "Error: Could not remove timestamp from value.");
      return false;
-      //TODO: Change Merge semantics and add a counter here
    }

    // Apply the user partial-merge operator (store result in *new_value)
    assert(new_value);
    Slice left_without_ts(left_operand.data(), left_operand.size() - ts_len);
    Slice right_without_ts(right_operand.data(), right_operand.size() - ts_len);
-    user_merge_op_->PartialMerge(key, left_without_ts, right_without_ts,
-                                 new_value, logger);
+    if (!user_merge_op_->PartialMerge(key, left_without_ts, right_without_ts,
+                                      new_value, logger)) {
+      return false;
+    }

    // Augment the *new_value with the ttl time-stamp
    int32_t curtime;
@@ -312,7 +355,7 @@ class TtlMergeOperator : public MergeOperator {
  }

 private:
-  const MergeOperator* user_merge_op_;
+  std::shared_ptr<MergeOperator> user_merge_op_;
 };

 }
@@ -2,6 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.

+#include <memory>
 #include "leveldb/compaction_filter.h"
 #include "utilities/utility_db.h"
 #include "util/testharness.h"
@@ -54,8 +55,9 @@ class TtlTest {

  // Open with TestFilter compaction filter
  void OpenTtlWithTestCompaction(int32_t ttl) {
-    test_comp_filter_.reset(new TestFilter(kSampleSize_, kNewValue_));
-    options_.compaction_filter = test_comp_filter_.get();
+    options_.compaction_filter_factory =
+      std::shared_ptr<CompactionFilterFactory>(
+          new TestFilterFactory(kSampleSize_, kNewValue_));
    OpenTtl(ttl);
  }

@@ -252,6 +254,29 @@ class TtlTest {
    const std::string kNewValue_;
  };

+  class TestFilterFactory : public CompactionFilterFactory {
+    public:
+      TestFilterFactory(const int64_t kSampleSize, const std::string kNewValue)
+        : kSampleSize_(kSampleSize),
+          kNewValue_(kNewValue) {
+      }
+
+      virtual std::unique_ptr<CompactionFilter>
+      CreateCompactionFilter() override {
+        return std::unique_ptr<CompactionFilter>(
+            new TestFilter(kSampleSize_, kNewValue_));
+      }
+
+      virtual const char* Name() const override {
+        return "TestFilterFactory";
+      }
+
+    private:
+      const int64_t kSampleSize_;
+      const std::string kNewValue_;
+  };
+
+
  // Choose carefully so that Put, Gets & Compaction complete in 1 second buffer
  const int64_t kSampleSize_ = 100;