Fix iOS compile

Summary: We need to lock accesses to some TestEnv variables. Otherwise we get failures like http://ci-builds.fb.com/job/rocksdb_asan_check/657/console

Test Plan: make check

Reviewers: dhruba, haobo, sdong

Reviewed By: haobo

CC: leveldb

Differential Revision: https://reviews.facebook.net/D18489

.gitignore

@@ -1,3 +1,4 @@
+TARGETS
 build_config.mk
 
 *.a
@@ -17,6 +18,7 @@ build_config.mk
 *.jar
 *.*jnilib*
 *.d-e
+*.o-*
 
 ldb
 manifest_dump

CONTRIBUTING.md

@@ -8,7 +8,7 @@ open source project, you're good to go. If you are submitting a pull
 request for the first time, just let us know that you have completed
 the CLA and we can cross-check with your GitHub username.
 
-Complete your CLA here: <https://developers.facebook.com/opensource/cla>
+Complete your CLA here: <https://code.facebook.com/cla>
 
 If you don't have a Facebook account, we can send you a PDF that you can
 sign offline. Send us an e-mail or create a new github issue to

HISTORY.md

@@ -1,8 +1,15 @@
 # Rocksdb Change Log
 
-## Unreleased
+## 3.0.0 (05/05/2014)
 
 ### Public API changes
+* Added _LEVEL to all InfoLogLevel enums
+* Deprecated ReadOptions.prefix and ReadOptions.prefix_seek. Seek() defaults to prefix-based seek when Options.prefix_extractor is supplied. More detail is documented in https://github.com/facebook/rocksdb/wiki/Prefix-Seek-API-Changes
+
+### New Features
+* Column family support
+* Added an option to use different checksum functions in BlockBasedTableOptions
+* Added ApplyToAllCacheEntries() function to Cache
 
 ## 2.8.0 (04/04/2014)
 
@@ -18,7 +25,7 @@
 * Added Env::GetThreadPoolQueueLen(), which returns the waiting queue length of thread pools
 * Added a command "checkconsistency" in ldb tool, which checks
   if file system state matches DB state (file existence and file sizes)
-* Separate options related to block based table to a new struct BlockBasedTableOptions
+* Separate options related to block based table to a new struct BlockBasedTableOptions.
 * WriteBatch has a new function Count() to return total size in the batch, and Data() now returns a reference instead of a copy
 * Add more counters to perf context.
 * Supports several more DB properties: compaction-pending, background-errors and cur-size-active-mem-table.

Makefile

@@ -15,7 +15,14 @@ endif
 
 ifeq ($(MAKECMDGOALS),shared_lib)
 PLATFORM_SHARED_LDFLAGS=-fPIC
+OPT += -DNDEBUG
 endif
+
+ifeq ($(MAKECMDGOALS),static_lib)
+PLATFORM_SHARED_LDFLAGS=-fPIC
+OPT += -DNDEBUG
+endif
+
 #-----------------------------------------------
 
 # detect what platform we're building on
@@ -44,7 +51,7 @@ else
 	PLATFORM_CCFLAGS += $(JEMALLOC_INCLUDE) -DHAVE_JEMALLOC
 endif
 
-WARNING_FLAGS = -Wall -Werror -Wno-sign-compare
+WARNING_FLAGS = -Wall -Werror
 CFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CCFLAGS) $(OPT)
 CXXFLAGS += $(WARNING_FLAGS) -I. -I./include $(PLATFORM_CXXFLAGS) $(OPT) -Woverloaded-virtual
 
@@ -56,6 +63,7 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
 
 TESTUTIL = ./util/testutil.o
 TESTHARNESS = ./util/testharness.o $(TESTUTIL)
+BENCHHARNESS = ./util/benchharness.o
 VALGRIND_ERROR = 2
 VALGRIND_DIR = build_tools/VALGRIND_LOGS
 VALGRIND_VER := $(join $(VALGRIND_VER),valgrind)
@@ -65,9 +73,11 @@ TESTS = \
 	db_test \
 	block_hash_index_test \
 	autovector_test \
+	column_family_test \
 	table_properties_collector_test \
 	arena_test \
 	auto_roll_logger_test \
+	benchharness_test \
 	block_test \
 	bloom_test \
 	dynamic_bloom_test \
@@ -98,6 +108,7 @@ TESTS = \
 	backupable_db_test \
 	version_edit_test \
 	version_set_test \
+	file_indexer_test \
 	write_batch_test\
 	deletefile_test \
 	table_test \
@@ -112,9 +123,7 @@ TOOLS = \
 	db_repl_stress \
 	blob_store_bench
 
-
-PROGRAMS = db_bench signal_test table_reader_bench $(TESTS) $(TOOLS)
-BENCHMARKS = db_bench_sqlite3 db_bench_tree_db table_reader_bench
+PROGRAMS = db_bench signal_test table_reader_bench log_and_apply_bench $(TOOLS)
 
 # The library name is configurable since we are maintaining libraries of both
 # debug/release mode.
@@ -138,7 +147,7 @@ SHARED3 = $(SHARED1)
 SHARED = $(SHARED1)
 else
 # Update db.h if you change these.
-SHARED_MAJOR = 2
+SHARED_MAJOR = 3
 SHARED_MINOR = 0
 SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT)
 SHARED2 = $(SHARED1).$(SHARED_MAJOR)
@@ -151,7 +160,7 @@ $(SHARED2): $(SHARED3)
 endif
 
 $(SHARED3):
-	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(LDFLAGS) $(SOURCES) -o $@
+	$(CXX) $(PLATFORM_SHARED_LDFLAGS)$(SHARED2) $(CXXFLAGS) $(PLATFORM_SHARED_CFLAGS) $(SOURCES) $(LDFLAGS) -o $@
 
 endif  # PLATFORM_SHARED_EXT
 
@@ -159,18 +168,22 @@ endif  # PLATFORM_SHARED_EXT
 	release tags valgrind_check whitebox_crash_test format static_lib shared_lib all \
 	dbg
 
-all: $(LIBRARY) $(PROGRAMS)
+all: $(LIBRARY) $(PROGRAMS) $(TESTS)
 
 static_lib: $(LIBRARY)
 
 shared_lib: $(SHARED)
 
-dbg: $(LIBRARY) $(PROGRAMS)
+dbg: $(LIBRARY) $(PROGRAMS) $(TESTS)
 
-# Will also generate shared libraries.
+# creates static library and programs
 release:
 	$(MAKE) clean
-	OPT="-DNDEBUG -O2" $(MAKE) all -j32
+	OPT="-DNDEBUG -O2" $(MAKE) static_lib $(PROGRAMS) -j32
+
+release_shared_lib:
+	$(MAKE) clean
+	OPT="-DNDEBUG -O2" $(MAKE) shared_lib -j32
 
 coverage:
 	$(MAKE) clean
@@ -183,10 +196,10 @@ check: $(PROGRAMS) $(TESTS) $(TOOLS)
 	for t in $(TESTS); do echo "***** Running $$t"; ./$$t || exit 1; done
 	python tools/ldb_test.py
 
-ldb_tests: all $(PROGRAMS) $(TOOLS)
+ldb_tests: all $(PROGRAMS) $(TESTS) $(TOOLS)
 	python tools/ldb_test.py
 
-crash_test: blackbox_crash_test whitebox_crash_test
+crash_test: whitebox_crash_test blackbox_crash_test
 
 blackbox_crash_test: db_stress
 	python -u tools/db_crashtest.py
@@ -201,7 +214,7 @@ asan_check:
 
 asan_crash_test:
 	$(MAKE) clean
-	COMPILE_WITH_ASAN=1 $(MAKE) crash_test -j32
+	COMPILE_WITH_ASAN=1 $(MAKE) crash_test
 	$(MAKE) clean
 
 valgrind_check: all $(PROGRAMS) $(TESTS)
@@ -219,7 +232,7 @@ valgrind_check: all $(PROGRAMS) $(TESTS)
 	done
 
 clean:
-	-rm -f $(PROGRAMS) $(BENCHMARKS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) build_config.mk
+	-rm -f $(PROGRAMS) $(TESTS) $(LIBRARY) $(SHARED) $(MEMENVLIBRARY) build_config.mk
 	-rm -rf ios-x86/* ios-arm/*
 	-find . -name "*.[od]" -exec rm {} \;
 	-find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" -exec rm {} \;
@@ -255,12 +268,6 @@ db_repl_stress: tools/db_repl_stress.o $(LIBOBJECTS) $(TESTUTIL)
 blob_store_bench: tools/blob_store_bench.o $(LIBOBJECTS) $(TESTUTIL)
 	$(CXX) tools/blob_store_bench.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@  $(LDFLAGS) $(COVERAGEFLAGS)
 
-db_bench_sqlite3: doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) doc/bench/db_bench_sqlite3.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lsqlite3 $(COVERAGEFLAGS)
-
-db_bench_tree_db: doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL)
-	$(CXX) doc/bench/db_bench_tree_db.o $(LIBOBJECTS) $(TESTUTIL) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) -lkyotocabinet $(COVERAGEFLAGS)
-
 signal_test: util/signal_test.o $(LIBOBJECTS)
 	$(CXX) util/signal_test.o $(LIBOBJECTS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -270,6 +277,9 @@ arena_test: util/arena_test.o $(LIBOBJECTS) $(TESTHARNESS)
 autovector_test: util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/autovector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
+column_family_test: db/column_family_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/column_family_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 table_properties_collector_test: db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/table_properties_collector_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -297,6 +307,9 @@ stringappend_test: utilities/merge_operators/string_append/stringappend_test.o $
 redis_test: utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) utilities/redis/redis_lists_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
+benchharness_test: util/benchharness_test.o $(LIBOBJECTS) $(TESTHARNESS) $(BENCHHARNESS)
+	$(CXX) util/benchharness_test.o $(LIBOBJECTS) $(TESTHARNESS) $(BENCHHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 histogram_test: util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) util/histogram_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o$@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -324,6 +337,9 @@ simple_table_db_test: db/simple_table_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 table_reader_bench: table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) table/table_reader_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -pg
 
+log_and_apply_bench: db/log_and_apply_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(BENCHHARNESS)
+	$(CXX) db/log_and_apply_bench.o $(LIBOBJECTS) $(TESTHARNESS) $(BENCHHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS) -pg
+
 perf_context_test: db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/perf_context_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
 
@@ -366,6 +382,9 @@ version_edit_test: db/version_edit_test.o $(LIBOBJECTS) $(TESTHARNESS)
 version_set_test: db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/version_set_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
+file_indexer_test : db/file_indexer_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/file_indexer_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
+
 reduce_levels_test: tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) tools/reduce_levels_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
@@ -412,9 +431,9 @@ ldb: tools/ldb.o $(LIBOBJECTS)
 # ---------------------------------------------------------------------------
 # Jni stuff
 # ---------------------------------------------------------------------------
-JNI_NATIVE_SOURCES = ./java/rocksjni/rocksjni.cc ./java/rocksjni/options.cc ./java/rocksjni/write_batch.cc
 
-JAVA_INCLUDE = -I/usr/lib/jvm/java-openjdk/include/ -I/usr/lib/jvm/java-openjdk/include/linux 
+JNI_NATIVE_SOURCES = ./java/rocksjni/*.cc
+JAVA_INCLUDE = -I$(JAVA_HOME)/include/ -I$(JAVA_HOME)/include/linux
 ROCKSDBJNILIB = ./java/librocksdbjni.so
 
 ifeq ($(PLATFORM), OS_MACOSX)
@@ -422,7 +441,7 @@ ROCKSDBJNILIB = ./java/librocksdbjni.jnilib
 JAVA_INCLUDE = -I/System/Library/Frameworks/JavaVM.framework/Headers/
 endif
 
-jni: clean
+rocksdbjava: clean
 	OPT="-fPIC -DNDEBUG -O2" $(MAKE) $(LIBRARY) -j32
 	cd java;$(MAKE) java;
 	rm -f $(ROCKSDBJNILIB)
@@ -435,6 +454,9 @@ jclean:
 jtest:
 	cd java;$(MAKE) sample;$(MAKE) test;
 
+jdb_bench:
+	cd java;$(MAKE) db_bench;
+
 # ---------------------------------------------------------------------------
 #  	Platform-specific compilation
 # ---------------------------------------------------------------------------

README.fb

@@ -1,3 +0,0 @@
-* Detailed instructions on how to compile using fbcode and jemalloc
-
-* Latest release is 2.7.fb

ROCKSDB_LITE.md

@@ -0,0 +1,20 @@
+# RocksDBLite
+
+RocksDBLite is a project focused on mobile use cases, which don't need a lot of fancy things we've built for server workloads and they are very sensitive to binary size. For that reason, we added a compile flag ROCKSDB_LITE that comments out a lot of the nonessential code and keeps the binary lean.
+
+Some examples of the features disabled by ROCKSDB_LITE:
+* compiled-in support for LDB tool
+* No backupable DB
+* No support for replication (which we provide in form of TrasactionalIterator)
+* No advanced monitoring tools
+* No special-purpose memtables that are highly optimized for specific use cases
+
+When adding a new big feature to RocksDB, please add ROCKSDB_LITE compile guard if:
+* Nobody from mobile really needs your feature,
+* Your feature is adding a lot of weight to the binary.
+
+Don't add ROCKSDB_LITE compile guard if:
+* It would introduce a lot of code complexity. Compile guards make code harder to read. It's a trade-off.
+* Your feature is not adding a lot of weight.
+
+If unsure, ask. :)

build_tools/build_detect_platform

@@ -49,7 +49,7 @@ if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
     if [ -z "$USE_CLANG" ]; then
         CENTOS_VERSION=`rpm -q --qf "%{VERSION}" \
           $(rpm -q --whatprovides redhat-release)`
-        if [ "$CENTOS_VERSION" -ge "6" ]; then
+        if [ "$CENTOS_VERSION" = "6" ]; then
           source $PWD/build_tools/fbcode.gcc481.sh
         else
           source $PWD/build_tools/fbcode.gcc471.sh
@@ -100,7 +100,7 @@ case "$TARGET_OS" in
         ;;
     IOS)
         PLATFORM=IOS
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE"
+        COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE -DROCKSDB_LITE"
         PLATFORM_SHARED_EXT=dylib
         PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
         CROSS_COMPILE=true
@@ -156,7 +156,9 @@ case "$TARGET_OS" in
         exit 1
 esac
 
-$PWD/build_tools/build_detect_version
+if test -z "$DO_NOT_RUN_BUILD_DETECT_VERSION"; then
+  $PWD/build_tools/build_detect_version
+fi
 
 # We want to make a list of all cc files within util, db, table, and helpers
 # except for the test and benchmark files. By default, find will output a list
@@ -166,7 +168,7 @@ DIRS="util db table utilities"
 
 set -f # temporarily disable globbing so that our patterns arent expanded
 PRUNE_TEST="-name *test*.cc -prune"
-PRUNE_BENCH="-name *_bench.cc -prune"
+PRUNE_BENCH="-name *bench*.cc -prune"
 PORTABLE_FILES=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cc' -print | sort | tr "\n" " "`
 PORTABLE_CPP=`cd $ROCKSDB_ROOT; find $DIRS $PRUNE_TEST -o $PRUNE_BENCH -o -name '*.cpp' -print | sort | tr "\n" " "`
 set +f # re-enable globbing

build_tools/fbcode.gcc481.sh

@@ -43,6 +43,10 @@ ZLIB_LIBS=" $TOOLCHAIN_LIB_BASE/zlib/zlib-1.2.5/c3f970a/lib/libz.a"
 BZIP_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/include"
 BZIP_LIBS=" $TOOLCHAIN_LIB_BASE/bzip2/bzip2-1.0.6/c3f970a/lib/libbz2.a"
 
+LZ4_REV=065ec7e38fe83329031f6668c43bef83eff5808b
+LZ4_INCLUDE=" -I /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/include"
+LZ4_LIBS=" /mnt/gvfs/third-party2/lz4/$LZ4_REV/r108/gcc-4.8.1-glibc-2.17/c3f970a/lib/liblz4.a"
+
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/include"
 GFLAGS_LIBS=" $TOOLCHAIN_LIB_BASE/gflags/gflags-1.6/c3f970a/lib/libgflags.a"
@@ -55,22 +59,22 @@ JEMALLOC_LIB=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-3.4.1/4
 export USE_SSE=" -msse -msse4.2 "
 
 CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/gcc"
-CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $GFLAGS_INCLUDE"
+CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.8.1/cc6c9dc/bin/g++ $JINCLUDE $SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $GFLAGS_INCLUDE"
 AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar
 RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib
 
 CFLAGS="-B$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/gold -m64 -mtune=generic"
 CFLAGS+=" $LIBGCC_INCLUDE $GLIBC_INCLUDE"
 CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_ATOMIC_PRESENT -DROCKSDB_FALLOCATE_PRESENT"
-CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2"
+CFLAGS+=" -DSNAPPY -DGFLAGS -DZLIB -DBZIP2 -DLZ4"
 
 EXEC_LDFLAGS="-Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
 EXEC_LDFLAGS+=" -Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-1.0.1/675d945/lib/libunwind.a"
-EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS+=" $HDFSLIB $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
 
 PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS "
 
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $GFLAGS_LIBS"
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $GFLAGS_LIBS"
 
 VALGRIND_VER="$TOOLCHAIN_LIB_BASE/valgrind/valgrind-3.8.1/c3f970a/bin/"
 

build_tools/make_new_version.sh

@@ -28,17 +28,19 @@ fi
 ROCKSDB_VERSION=$1
 
 GIT_BRANCH=`git rev-parse --abbrev-ref HEAD`
+echo $GIT_BRANCH
+
 if [ $GIT_BRANCH != "master" ]; then
   echo "Error: Current branch is '$GIT_BRANCH', Please switch to master branch."
+  exit 1
 fi
 
 title "Adding new tag for this release ..."
-TAG="$ROCKSDB_VERSION.fb"
-$GIT tag -a "$TAG" -m "RocksDB $ROCKSDB_VERSION"
+BRANCH="$ROCKSDB_VERSION.fb"
+$GIT co -b $BRANCH
 
 # Setting up the proxy for remote repo access
-title "Pushing new tag to remote repo ..."
-$GIT push origin --tags
+title "Pushing new branch to remote repo ..."
+git push origin --set-upstream $BRANCH
 
-title "Tag $TAG is pushed to github; if you want to delete it, please run"
-title "git tags -d $TAG && git push origin :refs/tags/$TAG"
+title "Branch $BRANCH is pushed to github;"

db/c.cc

@@ -7,6 +7,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef ROCKSDB_LITE
+
 #include "rocksdb/c.h"
 
 #include <stdlib.h>
@@ -328,6 +330,20 @@ rocksdb_t* rocksdb_open(
   return result;
 }
 
+rocksdb_t* rocksdb_open_for_read_only(
+    const rocksdb_options_t* options,
+    const char* name,
+    unsigned char error_if_log_file_exist,
+    char** errptr) {
+  DB* db;
+  if (SaveError(errptr, DB::OpenForReadOnly(options->rep, std::string(name), &db, error_if_log_file_exist))) {
+    return nullptr;
+  }
+  rocksdb_t* result = new rocksdb_t;
+  result->rep = db;
+  return result;
+}
+
 void rocksdb_close(rocksdb_t* db) {
   delete db->rep;
   delete db;
@@ -1228,23 +1244,12 @@ void rocksdb_readoptions_set_fill_cache(
   opt->rep.fill_cache = v;
 }
 
-void rocksdb_readoptions_set_prefix_seek(
-    rocksdb_readoptions_t* opt, unsigned char v) {
-  opt->rep.prefix_seek = v;
-}
-
 void rocksdb_readoptions_set_snapshot(
     rocksdb_readoptions_t* opt,
     const rocksdb_snapshot_t* snap) {
   opt->rep.snapshot = (snap ? snap->rep : nullptr);
 }
 
-void rocksdb_readoptions_set_prefix(
-    rocksdb_readoptions_t* opt, const char* key, size_t keylen) {
-  Slice prefix = Slice(key, keylen);
-  opt->rep.prefix = &prefix;
-}
-
 void rocksdb_readoptions_set_read_tier(
     rocksdb_readoptions_t* opt, int v) {
   opt->rep.read_tier = static_cast<rocksdb::ReadTier>(v);
@@ -1467,3 +1472,5 @@ extern void rocksdb_livefiles_destroy(
 }
 
 }  // end extern "C"
+
+#endif  // ROCKSDB_LITE

db/c_test.c

@@ -145,7 +145,7 @@ static char* FilterCreate(
   memcpy(result, "fake", 4);
   return result;
 }
-unsigned char FilterKeyMatch(
+static unsigned char FilterKeyMatch(
     void* arg,
     const char* key, size_t length,
     const char* filter, size_t filter_length) {
@@ -461,8 +461,6 @@ int main(int argc, char** argv) {
     rocksdb_put(db, woptions, "bar3", 4, "bar", 3, &err);
     CheckNoError(err);
 
-    rocksdb_readoptions_set_prefix_seek(roptions, 1);
-
     rocksdb_iterator_t* iter = rocksdb_create_iterator(db, roptions);
     CheckCondition(!rocksdb_iter_valid(iter));
 

db/column_family.cc

@@ -0,0 +1,583 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/column_family.h"
+
+#include <vector>
+#include <string>
+#include <algorithm>
+
+#include "db/db_impl.h"
+#include "db/version_set.h"
+#include "db/internal_stats.h"
+#include "db/compaction_picker.h"
+#include "db/table_properties_collector.h"
+#include "util/autovector.h"
+#include "util/hash_skiplist_rep.h"
+
+namespace rocksdb {
+
+ColumnFamilyHandleImpl::ColumnFamilyHandleImpl(ColumnFamilyData* cfd,
+                                               DBImpl* db, port::Mutex* mutex)
+    : cfd_(cfd), db_(db), mutex_(mutex) {
+  if (cfd_ != nullptr) {
+    cfd_->Ref();
+  }
+}
+
+ColumnFamilyHandleImpl::~ColumnFamilyHandleImpl() {
+  if (cfd_ != nullptr) {
+    DBImpl::DeletionState deletion_state;
+    mutex_->Lock();
+    if (cfd_->Unref()) {
+      delete cfd_;
+    }
+    db_->FindObsoleteFiles(deletion_state, false, true);
+    mutex_->Unlock();
+    if (deletion_state.HaveSomethingToDelete()) {
+      db_->PurgeObsoleteFiles(deletion_state);
+    }
+  }
+}
+
+uint32_t ColumnFamilyHandleImpl::GetID() const { return cfd()->GetID(); }
+
+namespace {
+// Fix user-supplied options to be reasonable
+template <class T, class V>
+static void ClipToRange(T* ptr, V minvalue, V maxvalue) {
+  if (static_cast<V>(*ptr) > maxvalue) *ptr = maxvalue;
+  if (static_cast<V>(*ptr) < minvalue) *ptr = minvalue;
+}
+}  // anonymous namespace
+
+ColumnFamilyOptions SanitizeOptions(const InternalKeyComparator* icmp,
+                                    const InternalFilterPolicy* ipolicy,
+                                    const ColumnFamilyOptions& src) {
+  ColumnFamilyOptions result = src;
+  result.comparator = icmp;
+  result.filter_policy = (src.filter_policy != nullptr) ? ipolicy : nullptr;
+#ifdef OS_MACOSX
+  // TODO(icanadi) make write_buffer_size uint64_t instead of size_t
+  ClipToRange(&result.write_buffer_size, ((size_t)64) << 10, ((size_t)1) << 30);
+#else
+  ClipToRange(&result.write_buffer_size,
+              ((size_t)64) << 10, ((size_t)64) << 30);
+#endif
+  // if user sets arena_block_size, we trust user to use this value. Otherwise,
+  // calculate a proper value from writer_buffer_size;
+  if (result.arena_block_size <= 0) {
+    result.arena_block_size = result.write_buffer_size / 10;
+  }
+  result.min_write_buffer_number_to_merge =
+      std::min(result.min_write_buffer_number_to_merge,
+               result.max_write_buffer_number - 1);
+  if (result.block_cache == nullptr && !result.no_block_cache) {
+    result.block_cache = NewLRUCache(8 << 20);
+  }
+  result.compression_per_level = src.compression_per_level;
+  if (result.block_size_deviation < 0 || result.block_size_deviation > 100) {
+    result.block_size_deviation = 0;
+  }
+  if (result.max_mem_compaction_level >= result.num_levels) {
+    result.max_mem_compaction_level = result.num_levels - 1;
+  }
+  if (result.soft_rate_limit > result.hard_rate_limit) {
+    result.soft_rate_limit = result.hard_rate_limit;
+  }
+  if (!result.prefix_extractor) {
+    assert(result.memtable_factory);
+    Slice name = result.memtable_factory->Name();
+    if (name.compare("HashSkipListRepFactory") == 0 ||
+        name.compare("HashLinkListRepFactory") == 0) {
+      result.memtable_factory = std::make_shared<SkipListFactory>();
+    }
+  }
+
+  // -- Sanitize the table properties collector
+  // All user defined properties collectors will be wrapped by
+  // UserKeyTablePropertiesCollector since for them they only have the
+  // knowledge of the user keys; internal keys are invisible to them.
+  auto& collectors = result.table_properties_collectors;
+  for (size_t i = 0; i < result.table_properties_collectors.size(); ++i) {
+    assert(collectors[i]);
+    collectors[i] =
+        std::make_shared<UserKeyTablePropertiesCollector>(collectors[i]);
+  }
+  // Add collector to collect internal key statistics
+  collectors.push_back(std::make_shared<InternalKeyPropertiesCollector>());
+
+  return result;
+}
+
+int SuperVersion::dummy = 0;
+void* const SuperVersion::kSVInUse = &SuperVersion::dummy;
+void* const SuperVersion::kSVObsolete = nullptr;
+
+SuperVersion::~SuperVersion() {
+  for (auto td : to_delete) {
+    delete td;
+  }
+}
+
+SuperVersion* SuperVersion::Ref() {
+  refs.fetch_add(1, std::memory_order_relaxed);
+  return this;
+}
+
+bool SuperVersion::Unref() {
+  // fetch_sub returns the previous value of ref
+  uint32_t previous_refs = refs.fetch_sub(1, std::memory_order_relaxed);
+  assert(previous_refs > 0);
+  return previous_refs == 1;
+}
+
+void SuperVersion::Cleanup() {
+  assert(refs.load(std::memory_order_relaxed) == 0);
+  imm->Unref(&to_delete);
+  MemTable* m = mem->Unref();
+  if (m != nullptr) {
+    to_delete.push_back(m);
+  }
+  current->Unref();
+}
+
+void SuperVersion::Init(MemTable* new_mem, MemTableListVersion* new_imm,
+                        Version* new_current) {
+  mem = new_mem;
+  imm = new_imm;
+  current = new_current;
+  mem->Ref();
+  imm->Ref();
+  current->Ref();
+  refs.store(1, std::memory_order_relaxed);
+}
+
+namespace {
+void SuperVersionUnrefHandle(void* ptr) {
+  // UnrefHandle is called when a thread exists or a ThreadLocalPtr gets
+  // destroyed. When former happens, the thread shouldn't see kSVInUse.
+  // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
+  // well.
+  SuperVersion* sv = static_cast<SuperVersion*>(ptr);
+  if (sv->Unref()) {
+    sv->db_mutex->Lock();
+    sv->Cleanup();
+    sv->db_mutex->Unlock();
+    delete sv;
+  }
+}
+}  // anonymous namespace
+
+ColumnFamilyData::ColumnFamilyData(const std::string& dbname, uint32_t id,
+                                   const std::string& name,
+                                   Version* dummy_versions, Cache* table_cache,
+                                   const ColumnFamilyOptions& options,
+                                   const DBOptions* db_options,
+                                   const EnvOptions& storage_options,
+                                   ColumnFamilySet* column_family_set)
+    : id_(id),
+      name_(name),
+      dummy_versions_(dummy_versions),
+      current_(nullptr),
+      refs_(0),
+      dropped_(false),
+      internal_comparator_(options.comparator),
+      internal_filter_policy_(options.filter_policy),
+      options_(*db_options, SanitizeOptions(&internal_comparator_,
+                                            &internal_filter_policy_, options)),
+      mem_(nullptr),
+      imm_(options.min_write_buffer_number_to_merge),
+      super_version_(nullptr),
+      super_version_number_(0),
+      local_sv_(new ThreadLocalPtr(&SuperVersionUnrefHandle)),
+      next_(nullptr),
+      prev_(nullptr),
+      log_number_(0),
+      need_slowdown_for_num_level0_files_(false),
+      column_family_set_(column_family_set) {
+  Ref();
+
+  // if dummy_versions is nullptr, then this is a dummy column family.
+  if (dummy_versions != nullptr) {
+    internal_stats_.reset(new InternalStats(options.num_levels, db_options->env,
+                                            db_options->statistics.get()));
+    table_cache_.reset(
+        new TableCache(dbname, &options_, storage_options, table_cache));
+    if (options_.compaction_style == kCompactionStyleUniversal) {
+      compaction_picker_.reset(
+          new UniversalCompactionPicker(&options_, &internal_comparator_));
+    } else {
+      compaction_picker_.reset(
+          new LevelCompactionPicker(&options_, &internal_comparator_));
+    }
+
+    Log(options_.info_log, "Options for column family \"%s\":\n",
+        name.c_str());
+    const ColumnFamilyOptions* cf_options = &options_;
+    cf_options->Dump(options_.info_log.get());
+  }
+}
+
+// DB mutex held
+ColumnFamilyData::~ColumnFamilyData() {
+  assert(refs_ == 0);
+  // remove from linked list
+  auto prev = prev_;
+  auto next = next_;
+  prev->next_ = next;
+  next->prev_ = prev;
+
+  // it's nullptr for dummy CFD
+  if (column_family_set_ != nullptr) {
+    // remove from column_family_set
+    column_family_set_->RemoveColumnFamily(this);
+  }
+
+  if (current_ != nullptr) {
+    current_->Unref();
+  }
+
+  if (super_version_ != nullptr) {
+    // Release SuperVersion reference kept in ThreadLocalPtr.
+    // This must be done outside of mutex_ since unref handler can lock mutex.
+    super_version_->db_mutex->Unlock();
+    local_sv_.reset();
+    super_version_->db_mutex->Lock();
+
+    bool is_last_reference __attribute__((unused));
+    is_last_reference = super_version_->Unref();
+    assert(is_last_reference);
+    super_version_->Cleanup();
+    delete super_version_;
+    super_version_ = nullptr;
+  }
+
+  if (dummy_versions_ != nullptr) {
+    // List must be empty
+    assert(dummy_versions_->next_ == dummy_versions_);
+    delete dummy_versions_;
+  }
+
+  if (mem_ != nullptr) {
+    delete mem_->Unref();
+  }
+  autovector<MemTable*> to_delete;
+  imm_.current()->Unref(&to_delete);
+  for (MemTable* m : to_delete) {
+    delete m;
+  }
+}
+
+const EnvOptions* ColumnFamilyData::soptions() const {
+  return &(column_family_set_->storage_options_);
+}
+
+void ColumnFamilyData::SetCurrent(Version* current) {
+  current_ = current;
+  need_slowdown_for_num_level0_files_ =
+      (options_.level0_slowdown_writes_trigger >= 0 &&
+       current_->NumLevelFiles(0) >= options_.level0_slowdown_writes_trigger);
+}
+
+void ColumnFamilyData::CreateNewMemtable() {
+  assert(current_ != nullptr);
+  if (mem_ != nullptr) {
+    delete mem_->Unref();
+  }
+  mem_ = new MemTable(internal_comparator_, options_);
+  mem_->Ref();
+}
+
+Compaction* ColumnFamilyData::PickCompaction(LogBuffer* log_buffer) {
+  return compaction_picker_->PickCompaction(current_, log_buffer);
+}
+
+Compaction* ColumnFamilyData::CompactRange(int input_level, int output_level,
+                                           const InternalKey* begin,
+                                           const InternalKey* end,
+                                           InternalKey** compaction_end) {
+  return compaction_picker_->CompactRange(current_, input_level, output_level,
+                                          begin, end, compaction_end);
+}
+
+SuperVersion* ColumnFamilyData::GetReferencedSuperVersion(
+    port::Mutex* db_mutex) {
+  SuperVersion* sv = nullptr;
+  if (LIKELY(column_family_set_->db_options_->allow_thread_local)) {
+    sv = GetThreadLocalSuperVersion(db_mutex);
+    sv->Ref();
+    if (!ReturnThreadLocalSuperVersion(sv)) {
+      sv->Unref();
+    }
+  } else {
+    db_mutex->Lock();
+    sv = super_version_->Ref();
+    db_mutex->Unlock();
+  }
+  return sv;
+}
+
+SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(
+    port::Mutex* db_mutex) {
+  SuperVersion* sv = nullptr;
+  // The SuperVersion is cached in thread local storage to avoid acquiring
+  // mutex when SuperVersion does not change since the last use. When a new
+  // SuperVersion is installed, the compaction or flush thread cleans up
+  // cached SuperVersion in all existing thread local storage. To avoid
+  // acquiring mutex for this operation, we use atomic Swap() on the thread
+  // local pointer to guarantee exclusive access. If the thread local pointer
+  // is being used while a new SuperVersion is installed, the cached
+  // SuperVersion can become stale. In that case, the background thread would
+  // have swapped in kSVObsolete. We re-check the value at when returning
+  // SuperVersion back to thread local, with an atomic compare and swap.
+  // The superversion will need to be released if detected to be stale.
+  void* ptr = local_sv_->Swap(SuperVersion::kSVInUse);
+  // Invariant:
+  // (1) Scrape (always) installs kSVObsolete in ThreadLocal storage
+  // (2) the Swap above (always) installs kSVInUse, ThreadLocal storage
+  // should only keep kSVInUse before ReturnThreadLocalSuperVersion call
+  // (if no Scrape happens).
+  assert(ptr != SuperVersion::kSVInUse);
+  sv = static_cast<SuperVersion*>(ptr);
+  if (sv == SuperVersion::kSVObsolete ||
+      sv->version_number != super_version_number_.load()) {
+    RecordTick(options_.statistics.get(), NUMBER_SUPERVERSION_ACQUIRES);
+    SuperVersion* sv_to_delete = nullptr;
+
+    if (sv && sv->Unref()) {
+      RecordTick(options_.statistics.get(), NUMBER_SUPERVERSION_CLEANUPS);
+      db_mutex->Lock();
+      // NOTE: underlying resources held by superversion (sst files) might
+      // not be released until the next background job.
+      sv->Cleanup();
+      sv_to_delete = sv;
+    } else {
+      db_mutex->Lock();
+    }
+    sv = super_version_->Ref();
+    db_mutex->Unlock();
+
+    delete sv_to_delete;
+  }
+  assert(sv != nullptr);
+  return sv;
+}
+
+bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
+  assert(sv != nullptr);
+  // Put the SuperVersion back
+  void* expected = SuperVersion::kSVInUse;
+  if (local_sv_->CompareAndSwap(static_cast<void*>(sv), expected)) {
+    // When we see kSVInUse in the ThreadLocal, we are sure ThreadLocal
+    // storage has not been altered and no Scrape has happend. The
+    // SuperVersion is still current.
+    return true;
+  } else {
+    // ThreadLocal scrape happened in the process of this GetImpl call (after
+    // thread local Swap() at the beginning and before CompareAndSwap()).
+    // This means the SuperVersion it holds is obsolete.
+    assert(expected == SuperVersion::kSVObsolete);
+  }
+  return false;
+}
+
+SuperVersion* ColumnFamilyData::InstallSuperVersion(
+    SuperVersion* new_superversion, port::Mutex* db_mutex) {
+  new_superversion->db_mutex = db_mutex;
+  new_superversion->Init(mem_, imm_.current(), current_);
+  SuperVersion* old_superversion = super_version_;
+  super_version_ = new_superversion;
+  ++super_version_number_;
+  super_version_->version_number = super_version_number_;
+  // Reset SuperVersions cached in thread local storage
+  if (column_family_set_->db_options_->allow_thread_local) {
+    ResetThreadLocalSuperVersions();
+  }
+  if (old_superversion != nullptr && old_superversion->Unref()) {
+    old_superversion->Cleanup();
+    return old_superversion;  // will let caller delete outside of mutex
+  }
+  return nullptr;
+}
+
+void ColumnFamilyData::ResetThreadLocalSuperVersions() {
+  autovector<void*> sv_ptrs;
+  local_sv_->Scrape(&sv_ptrs, SuperVersion::kSVObsolete);
+  for (auto ptr : sv_ptrs) {
+    assert(ptr);
+    if (ptr == SuperVersion::kSVInUse) {
+      continue;
+    }
+    auto sv = static_cast<SuperVersion*>(ptr);
+    if (sv->Unref()) {
+      sv->Cleanup();
+      delete sv;
+    }
+  }
+}
+
+ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
+                                 const DBOptions* db_options,
+                                 const EnvOptions& storage_options,
+                                 Cache* table_cache)
+    : max_column_family_(0),
+      dummy_cfd_(new ColumnFamilyData(dbname, 0, "", nullptr, nullptr,
+                                      ColumnFamilyOptions(), db_options,
+                                      storage_options_, nullptr)),
+      default_cfd_cache_(nullptr),
+      db_name_(dbname),
+      db_options_(db_options),
+      storage_options_(storage_options),
+      table_cache_(table_cache),
+      spin_lock_(ATOMIC_FLAG_INIT) {
+  // initialize linked list
+  dummy_cfd_->prev_ = dummy_cfd_;
+  dummy_cfd_->next_ = dummy_cfd_;
+}
+
+ColumnFamilySet::~ColumnFamilySet() {
+  while (column_family_data_.size() > 0) {
+    // cfd destructor will delete itself from column_family_data_
+    auto cfd = column_family_data_.begin()->second;
+    cfd->Unref();
+    delete cfd;
+  }
+  dummy_cfd_->Unref();
+  delete dummy_cfd_;
+}
+
+ColumnFamilyData* ColumnFamilySet::GetDefault() const {
+  assert(default_cfd_cache_ != nullptr);
+  return default_cfd_cache_;
+}
+
+ColumnFamilyData* ColumnFamilySet::GetColumnFamily(uint32_t id) const {
+  auto cfd_iter = column_family_data_.find(id);
+  if (cfd_iter != column_family_data_.end()) {
+    return cfd_iter->second;
+  } else {
+    return nullptr;
+  }
+}
+
+ColumnFamilyData* ColumnFamilySet::GetColumnFamily(const std::string& name)
+    const {
+  auto cfd_iter = column_families_.find(name);
+  if (cfd_iter != column_families_.end()) {
+    auto cfd = GetColumnFamily(cfd_iter->second);
+    assert(cfd != nullptr);
+    return cfd;
+  } else {
+    return nullptr;
+  }
+}
+
+uint32_t ColumnFamilySet::GetNextColumnFamilyID() {
+  return ++max_column_family_;
+}
+
+uint32_t ColumnFamilySet::GetMaxColumnFamily() { return max_column_family_; }
+
+void ColumnFamilySet::UpdateMaxColumnFamily(uint32_t new_max_column_family) {
+  max_column_family_ = std::max(new_max_column_family, max_column_family_);
+}
+
+// under a DB mutex
+ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
+    const std::string& name, uint32_t id, Version* dummy_versions,
+    const ColumnFamilyOptions& options) {
+  assert(column_families_.find(name) == column_families_.end());
+  ColumnFamilyData* new_cfd =
+      new ColumnFamilyData(db_name_, id, name, dummy_versions, table_cache_,
+                           options, db_options_, storage_options_, this);
+  Lock();
+  column_families_.insert({name, id});
+  column_family_data_.insert({id, new_cfd});
+  Unlock();
+  max_column_family_ = std::max(max_column_family_, id);
+  // add to linked list
+  new_cfd->next_ = dummy_cfd_;
+  auto prev = dummy_cfd_->prev_;
+  new_cfd->prev_ = prev;
+  prev->next_ = new_cfd;
+  dummy_cfd_->prev_ = new_cfd;
+  if (id == 0) {
+    default_cfd_cache_ = new_cfd;
+  }
+  return new_cfd;
+}
+
+void ColumnFamilySet::Lock() {
+  // spin lock
+  while (spin_lock_.test_and_set(std::memory_order_acquire)) {
+  }
+}
+
+void ColumnFamilySet::Unlock() { spin_lock_.clear(std::memory_order_release); }
+
+// REQUIRES: DB mutex held
+void ColumnFamilySet::FreeDeadColumnFamilies() {
+  autovector<ColumnFamilyData*> to_delete;
+  for (auto cfd = dummy_cfd_->next_; cfd != dummy_cfd_; cfd = cfd->next_) {
+    if (cfd->refs_ == 0) {
+      to_delete.push_back(cfd);
+    }
+  }
+  for (auto cfd : to_delete) {
+    // this is very rare, so it's not a problem that we do it under a mutex
+    delete cfd;
+  }
+}
+
+// under a DB mutex
+void ColumnFamilySet::RemoveColumnFamily(ColumnFamilyData* cfd) {
+  auto cfd_iter = column_family_data_.find(cfd->GetID());
+  assert(cfd_iter != column_family_data_.end());
+  Lock();
+  column_family_data_.erase(cfd_iter);
+  column_families_.erase(cfd->GetName());
+  Unlock();
+}
+
+bool ColumnFamilyMemTablesImpl::Seek(uint32_t column_family_id) {
+  if (column_family_id == 0) {
+    // optimization for common case
+    current_ = column_family_set_->GetDefault();
+  } else {
+    // maybe outside of db mutex, should lock
+    column_family_set_->Lock();
+    current_ = column_family_set_->GetColumnFamily(column_family_id);
+    column_family_set_->Unlock();
+  }
+  handle_.SetCFD(current_);
+  return current_ != nullptr;
+}
+
+uint64_t ColumnFamilyMemTablesImpl::GetLogNumber() const {
+  assert(current_ != nullptr);
+  return current_->GetLogNumber();
+}
+
+MemTable* ColumnFamilyMemTablesImpl::GetMemTable() const {
+  assert(current_ != nullptr);
+  return current_->mem();
+}
+
+const Options* ColumnFamilyMemTablesImpl::GetOptions() const {
+  assert(current_ != nullptr);
+  return current_->options();
+}
+
+ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() {
+  assert(current_ != nullptr);
+  return &handle_;
+}
+
+}  // namespace rocksdb

db/column_family.h

@@ -0,0 +1,418 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+
+#include <unordered_map>
+#include <string>
+#include <vector>
+#include <atomic>
+
+#include "rocksdb/options.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "db/memtable_list.h"
+#include "db/write_batch_internal.h"
+#include "db/table_cache.h"
+#include "util/thread_local.h"
+
+namespace rocksdb {
+
+class Version;
+class VersionSet;
+class MemTable;
+class MemTableListVersion;
+class CompactionPicker;
+class Compaction;
+class InternalKey;
+class InternalStats;
+class ColumnFamilyData;
+class DBImpl;
+class LogBuffer;
+
+// ColumnFamilyHandleImpl is the class that clients use to access different
+// column families. It has non-trivial destructor, which gets called when client
+// is done using the column family
+class ColumnFamilyHandleImpl : public ColumnFamilyHandle {
+ public:
+  // create while holding the mutex
+  ColumnFamilyHandleImpl(ColumnFamilyData* cfd, DBImpl* db, port::Mutex* mutex);
+  // destroy without mutex
+  virtual ~ColumnFamilyHandleImpl();
+  virtual ColumnFamilyData* cfd() const { return cfd_; }
+
+  virtual uint32_t GetID() const;
+
+ private:
+  ColumnFamilyData* cfd_;
+  DBImpl* db_;
+  port::Mutex* mutex_;
+};
+
+// Does not ref-count ColumnFamilyData
+// We use this dummy ColumnFamilyHandleImpl because sometimes MemTableInserter
+// calls DBImpl methods. When this happens, MemTableInserter need access to
+// ColumnFamilyHandle (same as the client would need). In that case, we feed
+// MemTableInserter dummy ColumnFamilyHandle and enable it to call DBImpl
+// methods
+class ColumnFamilyHandleInternal : public ColumnFamilyHandleImpl {
+ public:
+  ColumnFamilyHandleInternal()
+      : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr) {}
+
+  void SetCFD(ColumnFamilyData* cfd) { internal_cfd_ = cfd; }
+  virtual ColumnFamilyData* cfd() const override { return internal_cfd_; }
+
+ private:
+  ColumnFamilyData* internal_cfd_;
+};
+
+// holds references to memtable, all immutable memtables and version
+struct SuperVersion {
+  MemTable* mem;
+  MemTableListVersion* imm;
+  Version* current;
+  std::atomic<uint32_t> refs;
+  // We need to_delete because during Cleanup(), imm->Unref() returns
+  // all memtables that we need to free through this vector. We then
+  // delete all those memtables outside of mutex, during destruction
+  autovector<MemTable*> to_delete;
+  // Version number of the current SuperVersion
+  uint64_t version_number;
+  port::Mutex* db_mutex;
+
+  // should be called outside the mutex
+  SuperVersion() = default;
+  ~SuperVersion();
+  SuperVersion* Ref();
+
+  bool Unref();
+
+  // call these two methods with db mutex held
+  // Cleanup unrefs mem, imm and current. Also, it stores all memtables
+  // that needs to be deleted in to_delete vector. Unrefing those
+  // objects needs to be done in the mutex
+  void Cleanup();
+  void Init(MemTable* new_mem, MemTableListVersion* new_imm,
+            Version* new_current);
+
+  // The value of dummy is not actually used. kSVInUse takes its address as a
+  // mark in the thread local storage to indicate the SuperVersion is in use
+  // by thread. This way, the value of kSVInUse is guaranteed to have no
+  // conflict with SuperVersion object address and portable on different
+  // platform.
+  static int dummy;
+  static void* const kSVInUse;
+  static void* const kSVObsolete;
+};
+
+extern ColumnFamilyOptions SanitizeOptions(const InternalKeyComparator* icmp,
+                                           const InternalFilterPolicy* ipolicy,
+                                           const ColumnFamilyOptions& src);
+
+class ColumnFamilySet;
+
+// This class keeps all the data that a column family needs. It's mosly dumb and
+// used just to provide access to metadata.
+// Most methods require DB mutex held, unless otherwise noted
+class ColumnFamilyData {
+ public:
+  ~ColumnFamilyData();
+
+  // thread-safe
+  uint32_t GetID() const { return id_; }
+  // thread-safe
+  const std::string& GetName() const { return name_; }
+
+  void Ref() { ++refs_; }
+  // will just decrease reference count to 0, but will not delete it. returns
+  // true if the ref count was decreased to zero. in that case, it can be
+  // deleted by the caller immediatelly, or later, by calling
+  // FreeDeadColumnFamilies()
+  bool Unref() {
+    assert(refs_ > 0);
+    return --refs_ == 0;
+  }
+
+  // This can only be called from single-threaded VersionSet::LogAndApply()
+  // After dropping column family no other operation on that column family
+  // will be executed. All the files and memory will be, however, kept around
+  // until client drops the column family handle. That way, client can still
+  // access data from dropped column family.
+  // Column family can be dropped and still alive. In that state:
+  // *) Column family is not included in the iteration.
+  // *) Compaction and flush is not executed on the dropped column family.
+  // *) Client can continue writing and reading from column family. However, all
+  // writes stay in the current memtable.
+  // When the dropped column family is unreferenced, then we:
+  // *) delete all memory associated with that column family
+  // *) delete all the files associated with that column family
+  void SetDropped() {
+    // can't drop default CF
+    assert(id_ != 0);
+    dropped_ = true;
+  }
+  bool IsDropped() const { return dropped_; }
+
+  // thread-safe
+  int NumberLevels() const { return options_.num_levels; }
+
+  void SetLogNumber(uint64_t log_number) { log_number_ = log_number; }
+  uint64_t GetLogNumber() const { return log_number_; }
+
+  // thread-safe
+  const Options* options() const { return &options_; }
+  const EnvOptions* soptions() const;
+
+  InternalStats* internal_stats() { return internal_stats_.get(); }
+
+  MemTableList* imm() { return &imm_; }
+  MemTable* mem() { return mem_; }
+  Version* current() { return current_; }
+  Version* dummy_versions() { return dummy_versions_; }
+  void SetMemtable(MemTable* new_mem) { mem_ = new_mem; }
+  void SetCurrent(Version* current);
+  void CreateNewMemtable();
+
+  TableCache* table_cache() { return table_cache_.get(); }
+
+  // See documentation in compaction_picker.h
+  Compaction* PickCompaction(LogBuffer* log_buffer);
+  Compaction* CompactRange(int input_level, int output_level,
+                           const InternalKey* begin, const InternalKey* end,
+                           InternalKey** compaction_end);
+
+  CompactionPicker* compaction_picker() { return compaction_picker_.get(); }
+  // thread-safe
+  const Comparator* user_comparator() const {
+    return internal_comparator_.user_comparator();
+  }
+  // thread-safe
+  const InternalKeyComparator& internal_comparator() const {
+    return internal_comparator_;
+  }
+
+  SuperVersion* GetSuperVersion() { return super_version_; }
+  // thread-safe
+  // Return a already referenced SuperVersion to be used safely.
+  SuperVersion* GetReferencedSuperVersion(port::Mutex* db_mutex);
+  // thread-safe
+  // Get SuperVersion stored in thread local storage. If it does not exist,
+  // get a reference from a current SuperVersion.
+  SuperVersion* GetThreadLocalSuperVersion(port::Mutex* db_mutex);
+  // Try to return SuperVersion back to thread local storage. Retrun true on
+  // success and false on failure. It fails when the thread local storage
+  // contains anything other than SuperVersion::kSVInUse flag.
+  bool ReturnThreadLocalSuperVersion(SuperVersion* sv);
+  // thread-safe
+  uint64_t GetSuperVersionNumber() const {
+    return super_version_number_.load();
+  }
+  // will return a pointer to SuperVersion* if previous SuperVersion
+  // if its reference count is zero and needs deletion or nullptr if not
+  // As argument takes a pointer to allocated SuperVersion to enable
+  // the clients to allocate SuperVersion outside of mutex.
+  SuperVersion* InstallSuperVersion(SuperVersion* new_superversion,
+                                    port::Mutex* db_mutex);
+
+  void ResetThreadLocalSuperVersions();
+
+  // A Flag indicating whether write needs to slowdown because of there are
+  // too many number of level0 files.
+  bool NeedSlowdownForNumLevel0Files() const {
+    return need_slowdown_for_num_level0_files_;
+  }
+
+ private:
+  friend class ColumnFamilySet;
+  ColumnFamilyData(const std::string& dbname, uint32_t id,
+                   const std::string& name, Version* dummy_versions,
+                   Cache* table_cache, const ColumnFamilyOptions& options,
+                   const DBOptions* db_options,
+                   const EnvOptions& storage_options,
+                   ColumnFamilySet* column_family_set);
+
+  uint32_t id_;
+  const std::string name_;
+  Version* dummy_versions_;  // Head of circular doubly-linked list of versions.
+  Version* current_;         // == dummy_versions->prev_
+
+  int refs_;                   // outstanding references to ColumnFamilyData
+  bool dropped_;               // true if client dropped it
+
+  const InternalKeyComparator internal_comparator_;
+  const InternalFilterPolicy internal_filter_policy_;
+
+  Options const options_;
+
+  std::unique_ptr<TableCache> table_cache_;
+
+  std::unique_ptr<InternalStats> internal_stats_;
+
+  MemTable* mem_;
+  MemTableList imm_;
+  SuperVersion* super_version_;
+
+  // An ordinal representing the current SuperVersion. Updated by
+  // InstallSuperVersion(), i.e. incremented every time super_version_
+  // changes.
+  std::atomic<uint64_t> super_version_number_;
+
+  // Thread's local copy of SuperVersion pointer
+  // This needs to be destructed before mutex_
+  std::unique_ptr<ThreadLocalPtr> local_sv_;
+
+  // pointers for a circular linked list. we use it to support iterations
+  // that can be concurrent with writes
+  ColumnFamilyData* next_;
+  ColumnFamilyData* prev_;
+
+  // This is the earliest log file number that contains data from this
+  // Column Family. All earlier log files must be ignored and not
+  // recovered from
+  uint64_t log_number_;
+
+  // A flag indicating whether we should delay writes because
+  // we have too many level 0 files
+  bool need_slowdown_for_num_level0_files_;
+
+  // An object that keeps all the compaction stats
+  // and picks the next compaction
+  std::unique_ptr<CompactionPicker> compaction_picker_;
+
+  ColumnFamilySet* column_family_set_;
+};
+
+// ColumnFamilySet has interesting thread-safety requirements
+// * CreateColumnFamily() or RemoveColumnFamily() -- need to protect by DB
+// mutex. Inside, column_family_data_ and column_families_ will be protected
+// by Lock() and Unlock(). CreateColumnFamily() should ONLY be called from
+// VersionSet::LogAndApply() in the normal runtime. It is also called
+// during Recovery and in DumpManifest(). RemoveColumnFamily() is called
+// from ColumnFamilyData destructor
+// * Iteration -- hold DB mutex, but you can release it in the body of
+// iteration. If you release DB mutex in body, reference the column
+// family before the mutex and unreference after you unlock, since the column
+// family might get dropped when the DB mutex is released
+// * GetDefault() -- thread safe
+// * GetColumnFamily() -- either inside of DB mutex or call Lock() <-> Unlock()
+// * GetNextColumnFamilyID(), GetMaxColumnFamily(), UpdateMaxColumnFamily() --
+// inside of DB mutex
+class ColumnFamilySet {
+ public:
+  // ColumnFamilySet supports iteration
+  class iterator {
+   public:
+    explicit iterator(ColumnFamilyData* cfd)
+        : current_(cfd) {}
+    iterator& operator++() {
+      // dummy is never dead or dropped, so this will never be infinite
+      do {
+        current_ = current_->next_;
+      } while (current_->refs_ == 0 || current_->IsDropped());
+      return *this;
+    }
+    bool operator!=(const iterator& other) {
+      return this->current_ != other.current_;
+    }
+    ColumnFamilyData* operator*() { return current_; }
+
+   private:
+    ColumnFamilyData* current_;
+  };
+
+  ColumnFamilySet(const std::string& dbname, const DBOptions* db_options,
+                  const EnvOptions& storage_options, Cache* table_cache);
+  ~ColumnFamilySet();
+
+  ColumnFamilyData* GetDefault() const;
+  // GetColumnFamily() calls return nullptr if column family is not found
+  ColumnFamilyData* GetColumnFamily(uint32_t id) const;
+  ColumnFamilyData* GetColumnFamily(const std::string& name) const;
+  // this call will return the next available column family ID. it guarantees
+  // that there is no column family with id greater than or equal to the
+  // returned value in the current running instance or anytime in RocksDB
+  // instance history.
+  uint32_t GetNextColumnFamilyID();
+  uint32_t GetMaxColumnFamily();
+  void UpdateMaxColumnFamily(uint32_t new_max_column_family);
+
+  ColumnFamilyData* CreateColumnFamily(const std::string& name, uint32_t id,
+                                       Version* dummy_version,
+                                       const ColumnFamilyOptions& options);
+
+  iterator begin() { return iterator(dummy_cfd_->next_); }
+  iterator end() { return iterator(dummy_cfd_); }
+
+  void Lock();
+  void Unlock();
+
+  // REQUIRES: DB mutex held
+  // Don't call while iterating over ColumnFamilySet
+  void FreeDeadColumnFamilies();
+
+ private:
+  friend class ColumnFamilyData;
+  // helper function that gets called from cfd destructor
+  // REQUIRES: DB mutex held
+  void RemoveColumnFamily(ColumnFamilyData* cfd);
+
+  // column_families_ and column_family_data_ need to be protected:
+  // * when mutating: 1. DB mutex locked first, 2. spinlock locked second
+  // * when reading, either: 1. lock DB mutex, or 2. lock spinlock
+  //  (if both, respect the ordering to avoid deadlock!)
+  std::unordered_map<std::string, uint32_t> column_families_;
+  std::unordered_map<uint32_t, ColumnFamilyData*> column_family_data_;
+
+  uint32_t max_column_family_;
+  ColumnFamilyData* dummy_cfd_;
+  // We don't hold the refcount here, since default column family always exists
+  // We are also not responsible for cleaning up default_cfd_cache_. This is
+  // just a cache that makes common case (accessing default column family)
+  // faster
+  ColumnFamilyData* default_cfd_cache_;
+
+  const std::string db_name_;
+  const DBOptions* const db_options_;
+  const EnvOptions storage_options_;
+  Cache* table_cache_;
+  std::atomic_flag spin_lock_;
+};
+
+// We use ColumnFamilyMemTablesImpl to provide WriteBatch a way to access
+// memtables of different column families (specified by ID in the write batch)
+class ColumnFamilyMemTablesImpl : public ColumnFamilyMemTables {
+ public:
+  explicit ColumnFamilyMemTablesImpl(ColumnFamilySet* column_family_set)
+      : column_family_set_(column_family_set), current_(nullptr) {}
+
+  // sets current_ to ColumnFamilyData with column_family_id
+  // returns false if column family doesn't exist
+  bool Seek(uint32_t column_family_id) override;
+
+  // Returns log number of the selected column family
+  uint64_t GetLogNumber() const override;
+
+  // REQUIRES: Seek() called first
+  virtual MemTable* GetMemTable() const override;
+
+  // Returns options for selected column family
+  // REQUIRES: Seek() called first
+  virtual const Options* GetOptions() const override;
+
+  // Returns column family handle for the selected column family
+  virtual ColumnFamilyHandle* GetColumnFamilyHandle() override;
+
+ private:
+  ColumnFamilySet* column_family_set_;
+  ColumnFamilyData* current_;
+  ColumnFamilyHandleInternal handle_;
+};
+
+}  // namespace rocksdb

db/column_family_test.cc

@@ -0,0 +1,977 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <algorithm>
+#include <vector>
+#include <string>
+
+#include "db/db_impl.h"
+#include "rocksdb/env.h"
+#include "rocksdb/db.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+#include "util/coding.h"
+#include "utilities/merge_operators.h"
+
+namespace rocksdb {
+
+namespace {
+std::string RandomString(Random* rnd, int len) {
+  std::string r;
+  test::RandomString(rnd, len, &r);
+  return r;
+}
+}  // anonymous namespace
+
+// counts how many operations were performed
+class EnvCounter : public EnvWrapper {
+ public:
+  explicit EnvCounter(Env* base)
+      : EnvWrapper(base), num_new_writable_file_(0) {}
+  int GetNumberOfNewWritableFileCalls() {
+    return num_new_writable_file_;
+  }
+  Status NewWritableFile(const std::string& f, unique_ptr<WritableFile>* r,
+                         const EnvOptions& soptions) {
+    ++num_new_writable_file_;
+    return EnvWrapper::NewWritableFile(f, r, soptions);
+  }
+
+ private:
+  int num_new_writable_file_;
+};
+
+class ColumnFamilyTest {
+ public:
+  ColumnFamilyTest() : rnd_(139) {
+    env_ = new EnvCounter(Env::Default());
+    dbname_ = test::TmpDir() + "/column_family_test";
+    db_options_.create_if_missing = true;
+    db_options_.env = env_;
+    DestroyDB(dbname_, Options(db_options_, column_family_options_));
+  }
+
+  ~ColumnFamilyTest() {
+    delete env_;
+  }
+
+  void Close() {
+    for (auto h : handles_) {
+      delete h;
+    }
+    handles_.clear();
+    names_.clear();
+    delete db_;
+    db_ = nullptr;
+  }
+
+  Status TryOpen(std::vector<std::string> cf,
+                 std::vector<ColumnFamilyOptions> options = {}) {
+    std::vector<ColumnFamilyDescriptor> column_families;
+    names_.clear();
+    for (size_t i = 0; i < cf.size(); ++i) {
+      column_families.push_back(ColumnFamilyDescriptor(
+          cf[i], options.size() == 0 ? column_family_options_ : options[i]));
+      names_.push_back(cf[i]);
+    }
+    return DB::Open(db_options_, dbname_, column_families, &handles_, &db_);
+  }
+
+  Status OpenReadOnly(std::vector<std::string> cf,
+                         std::vector<ColumnFamilyOptions> options = {}) {
+    std::vector<ColumnFamilyDescriptor> column_families;
+    names_.clear();
+    for (size_t i = 0; i < cf.size(); ++i) {
+      column_families.push_back(ColumnFamilyDescriptor(
+          cf[i], options.size() == 0 ? column_family_options_ : options[i]));
+      names_.push_back(cf[i]);
+    }
+    return DB::OpenForReadOnly(db_options_, dbname_, column_families, &handles_,
+                               &db_);
+  }
+
+  void AssertOpenReadOnly(std::vector<std::string> cf,
+                    std::vector<ColumnFamilyOptions> options = {}) {
+    ASSERT_OK(OpenReadOnly(cf, options));
+  }
+
+
+  void Open(std::vector<std::string> cf,
+            std::vector<ColumnFamilyOptions> options = {}) {
+    ASSERT_OK(TryOpen(cf, options));
+  }
+
+  void Open() {
+    Open({"default"});
+  }
+
+  DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); }
+
+  int GetProperty(int cf, std::string property) {
+    std::string value;
+    ASSERT_TRUE(dbfull()->GetProperty(handles_[cf], property, &value));
+    return std::stoi(value);
+  }
+
+  void Destroy() {
+    for (auto h : handles_) {
+      delete h;
+    }
+    handles_.clear();
+    names_.clear();
+    delete db_;
+    db_ = nullptr;
+    ASSERT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
+  }
+
+  void CreateColumnFamilies(
+      const std::vector<std::string>& cfs,
+      const std::vector<ColumnFamilyOptions> options = {}) {
+    int cfi = handles_.size();
+    handles_.resize(cfi + cfs.size());
+    names_.resize(cfi + cfs.size());
+    for (size_t i = 0; i < cfs.size(); ++i) {
+      ASSERT_OK(db_->CreateColumnFamily(
+          options.size() == 0 ? column_family_options_ : options[i], cfs[i],
+          &handles_[cfi]));
+      names_[cfi] = cfs[i];
+      cfi++;
+    }
+  }
+
+  void Reopen(const std::vector<ColumnFamilyOptions> options = {}) {
+    std::vector<std::string> names;
+    for (auto name : names_) {
+      if (name != "") {
+        names.push_back(name);
+      }
+    }
+    Close();
+    assert(options.size() == 0 || names.size() == options.size());
+    Open(names, options);
+  }
+
+  void CreateColumnFamiliesAndReopen(const std::vector<std::string>& cfs) {
+    CreateColumnFamilies(cfs);
+    Reopen();
+  }
+
+  void DropColumnFamilies(const std::vector<int>& cfs) {
+    for (auto cf : cfs) {
+      ASSERT_OK(db_->DropColumnFamily(handles_[cf]));
+      delete handles_[cf];
+      handles_[cf] = nullptr;
+      names_[cf] = "";
+    }
+  }
+
+  void PutRandomData(int cf, int num, int key_value_size) {
+    for (int i = 0; i < num; ++i) {
+      // 10 bytes for key, rest is value
+      ASSERT_OK(Put(cf, test::RandomKey(&rnd_, 10),
+                    RandomString(&rnd_, key_value_size - 10)));
+    }
+  }
+
+  void WaitForFlush(int cf) {
+    ASSERT_OK(dbfull()->TEST_WaitForFlushMemTable(handles_[cf]));
+  }
+
+  void WaitForCompaction() { ASSERT_OK(dbfull()->TEST_WaitForCompact()); }
+
+  Status Put(int cf, const std::string& key, const std::string& value) {
+    return db_->Put(WriteOptions(), handles_[cf], Slice(key), Slice(value));
+  }
+  Status Merge(int cf, const std::string& key, const std::string& value) {
+    return db_->Merge(WriteOptions(), handles_[cf], Slice(key), Slice(value));
+  }
+  Status Flush(int cf) {
+    return db_->Flush(FlushOptions(), handles_[cf]);
+  }
+
+  std::string Get(int cf, const std::string& key) {
+    ReadOptions options;
+    options.verify_checksums = true;
+    std::string result;
+    Status s = db_->Get(options, handles_[cf], Slice(key), &result);
+    if (s.IsNotFound()) {
+      result = "NOT_FOUND";
+    } else if (!s.ok()) {
+      result = s.ToString();
+    }
+    return result;
+  }
+
+  void CompactAll(int cf) {
+    ASSERT_OK(db_->CompactRange(handles_[cf], nullptr, nullptr));
+  }
+
+  void Compact(int cf, const Slice& start, const Slice& limit) {
+    ASSERT_OK(db_->CompactRange(handles_[cf], &start, &limit));
+  }
+
+  int NumTableFilesAtLevel(int level, int cf) {
+    return GetProperty(cf,
+                       "rocksdb.num-files-at-level" + std::to_string(level));
+  }
+
+  // Return spread of files per level
+  std::string FilesPerLevel(int cf) {
+    std::string result;
+    int last_non_zero_offset = 0;
+    for (int level = 0; level < dbfull()->NumberLevels(handles_[cf]); level++) {
+      int f = NumTableFilesAtLevel(level, cf);
+      char buf[100];
+      snprintf(buf, sizeof(buf), "%s%d", (level ? "," : ""), f);
+      result += buf;
+      if (f > 0) {
+        last_non_zero_offset = result.size();
+      }
+    }
+    result.resize(last_non_zero_offset);
+    return result;
+  }
+
+  int CountLiveFiles() {
+    std::vector<LiveFileMetaData> metadata;
+    db_->GetLiveFilesMetaData(&metadata);
+    return static_cast<int>(metadata.size());
+  }
+
+  // Do n memtable flushes, each of which produces an sstable
+  // covering the range [small,large].
+  void MakeTables(int cf, int n, const std::string& small,
+                  const std::string& large) {
+    for (int i = 0; i < n; i++) {
+      ASSERT_OK(Put(cf, small, "begin"));
+      ASSERT_OK(Put(cf, large, "end"));
+      ASSERT_OK(db_->Flush(FlushOptions(), handles_[cf]));
+    }
+  }
+
+  int CountLiveLogFiles() {
+    int micros_wait_for_log_deletion = 20000;
+    env_->SleepForMicroseconds(micros_wait_for_log_deletion);
+    int ret = 0;
+    VectorLogPtr wal_files;
+    Status s;
+    // GetSortedWalFiles is a flakey function -- it gets all the wal_dir
+    // children files and then later checks for their existance. if some of the
+    // log files doesn't exist anymore, it reports an error. it does all of this
+    // without DB mutex held, so if a background process deletes the log file
+    // while the function is being executed, it returns an error. We retry the
+    // function 10 times to avoid the error failing the test
+    for (int retries = 0; retries < 10; ++retries) {
+      wal_files.clear();
+      s = db_->GetSortedWalFiles(wal_files);
+      if (s.ok()) {
+        break;
+      }
+    }
+    ASSERT_OK(s);
+    for (const auto& wal : wal_files) {
+      if (wal->Type() == kAliveLogFile) {
+        ++ret;
+      }
+    }
+    return ret;
+  }
+
+  void AssertNumberOfImmutableMemtables(std::vector<int> num_per_cf) {
+    assert(num_per_cf.size() == handles_.size());
+
+    for (size_t i = 0; i < num_per_cf.size(); ++i) {
+      ASSERT_EQ(num_per_cf[i],
+                GetProperty(i, "rocksdb.num-immutable-mem-table"));
+    }
+  }
+
+  void CopyFile(const std::string& source, const std::string& destination,
+                uint64_t size = 0) {
+    const EnvOptions soptions;
+    unique_ptr<SequentialFile> srcfile;
+    ASSERT_OK(env_->NewSequentialFile(source, &srcfile, soptions));
+    unique_ptr<WritableFile> destfile;
+    ASSERT_OK(env_->NewWritableFile(destination, &destfile, soptions));
+
+    if (size == 0) {
+      // default argument means copy everything
+      ASSERT_OK(env_->GetFileSize(source, &size));
+    }
+
+    char buffer[4096];
+    Slice slice;
+    while (size > 0) {
+      uint64_t one = std::min(uint64_t(sizeof(buffer)), size);
+      ASSERT_OK(srcfile->Read(one, &slice, buffer));
+      ASSERT_OK(destfile->Append(slice));
+      size -= slice.size();
+    }
+    ASSERT_OK(destfile->Close());
+  }
+
+  std::vector<ColumnFamilyHandle*> handles_;
+  std::vector<std::string> names_;
+  ColumnFamilyOptions column_family_options_;
+  DBOptions db_options_;
+  std::string dbname_;
+  DB* db_ = nullptr;
+  EnvCounter* env_;
+  Random rnd_;
+};
+
+TEST(ColumnFamilyTest, DontReuseColumnFamilyID) {
+  for (int iter = 0; iter < 3; ++iter) {
+    Open();
+    CreateColumnFamilies({"one", "two", "three"});
+    for (size_t i = 0; i < handles_.size(); ++i) {
+      auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[i]);
+      ASSERT_EQ(i, cfh->GetID());
+    }
+    if (iter == 1) {
+      Reopen();
+    }
+    DropColumnFamilies({3});
+    Reopen();
+    if (iter == 2) {
+      // this tests if max_column_family is correctly persisted with
+      // WriteSnapshot()
+      Reopen();
+    }
+    CreateColumnFamilies({"three2"});
+    // ID 3 that was used for dropped column family "three" should not be reused
+    auto cfh3 = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[3]);
+    ASSERT_EQ(4U, cfh3->GetID());
+    Close();
+    Destroy();
+  }
+}
+
+
+TEST(ColumnFamilyTest, AddDrop) {
+  Open();
+  CreateColumnFamilies({"one", "two", "three"});
+  ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
+  ASSERT_EQ("NOT_FOUND", Get(2, "fodor"));
+  DropColumnFamilies({2});
+  ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
+  CreateColumnFamilies({"four"});
+  ASSERT_EQ("NOT_FOUND", Get(3, "fodor"));
+  ASSERT_OK(Put(1, "fodor", "mirko"));
+  ASSERT_EQ("mirko", Get(1, "fodor"));
+  ASSERT_EQ("NOT_FOUND", Get(3, "fodor"));
+  Close();
+  ASSERT_TRUE(TryOpen({"default"}).IsInvalidArgument());
+  Open({"default", "one", "three", "four"});
+  DropColumnFamilies({1});
+  Reopen();
+  Close();
+
+  std::vector<std::string> families;
+  ASSERT_OK(DB::ListColumnFamilies(db_options_, dbname_, &families));
+  sort(families.begin(), families.end());
+  ASSERT_TRUE(families ==
+              std::vector<std::string>({"default", "four", "three"}));
+}
+
+TEST(ColumnFamilyTest, DropTest) {
+  // first iteration - dont reopen DB before dropping
+  // second iteration - reopen DB before dropping
+  for (int iter = 0; iter < 2; ++iter) {
+    Open({"default"});
+    CreateColumnFamiliesAndReopen({"pikachu"});
+    for (int i = 0; i < 100; ++i) {
+      ASSERT_OK(Put(1, std::to_string(i), "bar" + std::to_string(i)));
+    }
+    ASSERT_OK(Flush(1));
+
+    if (iter == 1) {
+      Reopen();
+    }
+    ASSERT_EQ("bar1", Get(1, "1"));
+
+    ASSERT_EQ(CountLiveFiles(), 1);
+    DropColumnFamilies({1});
+    // make sure that all files are deleted when we drop the column family
+    ASSERT_EQ(CountLiveFiles(), 0);
+    Destroy();
+  }
+}
+
+TEST(ColumnFamilyTest, WriteBatchFailure) {
+  Open();
+  CreateColumnFamiliesAndReopen({"one", "two"});
+  WriteBatch batch;
+  batch.Put(handles_[1], Slice("non-existing"), Slice("column-family"));
+  ASSERT_OK(db_->Write(WriteOptions(), &batch));
+  DropColumnFamilies({1});
+  Status s = db_->Write(WriteOptions(), &batch);
+  ASSERT_TRUE(s.IsInvalidArgument());
+  Close();
+}
+
+TEST(ColumnFamilyTest, ReadWrite) {
+  Open();
+  CreateColumnFamiliesAndReopen({"one", "two"});
+  ASSERT_OK(Put(0, "foo", "v1"));
+  ASSERT_OK(Put(0, "bar", "v2"));
+  ASSERT_OK(Put(1, "mirko", "v3"));
+  ASSERT_OK(Put(0, "foo", "v2"));
+  ASSERT_OK(Put(2, "fodor", "v5"));
+
+  for (int iter = 0; iter <= 3; ++iter) {
+    ASSERT_EQ("v2", Get(0, "foo"));
+    ASSERT_EQ("v2", Get(0, "bar"));
+    ASSERT_EQ("v3", Get(1, "mirko"));
+    ASSERT_EQ("v5", Get(2, "fodor"));
+    ASSERT_EQ("NOT_FOUND", Get(0, "fodor"));
+    ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
+    ASSERT_EQ("NOT_FOUND", Get(2, "foo"));
+    if (iter <= 1) {
+      Reopen();
+    }
+  }
+  Close();
+}
+
+TEST(ColumnFamilyTest, IgnoreRecoveredLog) {
+  std::string backup_logs = dbname_ + "/backup_logs";
+
+  // delete old files in backup_logs directory
+  ASSERT_OK(env_->CreateDirIfMissing(dbname_));
+  ASSERT_OK(env_->CreateDirIfMissing(backup_logs));
+  std::vector<std::string> old_files;
+  env_->GetChildren(backup_logs, &old_files);
+  for (auto& file : old_files) {
+    if (file != "." && file != "..") {
+      env_->DeleteFile(backup_logs + "/" + file);
+    }
+  }
+
+  column_family_options_.merge_operator =
+      MergeOperators::CreateUInt64AddOperator();
+  db_options_.wal_dir = dbname_ + "/logs";
+  Destroy();
+  Open();
+  CreateColumnFamilies({"cf1", "cf2"});
+
+  // fill up the DB
+  std::string one, two, three;
+  PutFixed64(&one, 1);
+  PutFixed64(&two, 2);
+  PutFixed64(&three, 3);
+  ASSERT_OK(Merge(0, "foo", one));
+  ASSERT_OK(Merge(1, "mirko", one));
+  ASSERT_OK(Merge(0, "foo", one));
+  ASSERT_OK(Merge(2, "bla", one));
+  ASSERT_OK(Merge(2, "fodor", one));
+  ASSERT_OK(Merge(0, "bar", one));
+  ASSERT_OK(Merge(2, "bla", one));
+  ASSERT_OK(Merge(1, "mirko", two));
+  ASSERT_OK(Merge(1, "franjo", one));
+
+  // copy the logs to backup
+  std::vector<std::string> logs;
+  env_->GetChildren(db_options_.wal_dir, &logs);
+  for (auto& log : logs) {
+    if (log != ".." && log != ".") {
+      CopyFile(db_options_.wal_dir + "/" + log, backup_logs + "/" + log);
+    }
+  }
+
+  // recover the DB
+  Close();
+
+  // 1. check consistency
+  // 2. copy the logs from backup back to WAL dir. if the recovery happens
+  // again on the same log files, this should lead to incorrect results
+  // due to applying merge operator twice
+  // 3. check consistency
+  for (int iter = 0; iter < 2; ++iter) {
+    // assert consistency
+    Open({"default", "cf1", "cf2"});
+    ASSERT_EQ(two, Get(0, "foo"));
+    ASSERT_EQ(one, Get(0, "bar"));
+    ASSERT_EQ(three, Get(1, "mirko"));
+    ASSERT_EQ(one, Get(1, "franjo"));
+    ASSERT_EQ(one, Get(2, "fodor"));
+    ASSERT_EQ(two, Get(2, "bla"));
+    Close();
+
+    if (iter == 0) {
+      // copy the logs from backup back to wal dir
+      for (auto& log : logs) {
+        if (log != ".." && log != ".") {
+          CopyFile(backup_logs + "/" + log, db_options_.wal_dir + "/" + log);
+        }
+      }
+    }
+  }
+}
+
+TEST(ColumnFamilyTest, FlushTest) {
+  Open();
+  CreateColumnFamiliesAndReopen({"one", "two"});
+  ASSERT_OK(Put(0, "foo", "v1"));
+  ASSERT_OK(Put(0, "bar", "v2"));
+  ASSERT_OK(Put(1, "mirko", "v3"));
+  ASSERT_OK(Put(0, "foo", "v2"));
+  ASSERT_OK(Put(2, "fodor", "v5"));
+  for (int i = 0; i < 3; ++i) {
+    Flush(i);
+  }
+  Reopen();
+
+  for (int iter = 0; iter <= 2; ++iter) {
+    ASSERT_EQ("v2", Get(0, "foo"));
+    ASSERT_EQ("v2", Get(0, "bar"));
+    ASSERT_EQ("v3", Get(1, "mirko"));
+    ASSERT_EQ("v5", Get(2, "fodor"));
+    ASSERT_EQ("NOT_FOUND", Get(0, "fodor"));
+    ASSERT_EQ("NOT_FOUND", Get(1, "fodor"));
+    ASSERT_EQ("NOT_FOUND", Get(2, "foo"));
+    if (iter <= 1) {
+      Reopen();
+    }
+  }
+  Close();
+}
+
+// Makes sure that obsolete log files get deleted
+TEST(ColumnFamilyTest, LogDeletionTest) {
+  db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
+  column_family_options_.write_buffer_size = 100000;  // 100KB
+  Open();
+  CreateColumnFamilies({"one", "two", "three", "four"});
+  // Each bracket is one log file. if number is in (), it means
+  // we don't need it anymore (it's been flushed)
+  // []
+  ASSERT_EQ(CountLiveLogFiles(), 0);
+  PutRandomData(0, 1, 100);
+  // [0]
+  PutRandomData(1, 1, 100);
+  // [0, 1]
+  PutRandomData(1, 1000, 100);
+  WaitForFlush(1);
+  // [0, (1)] [1]
+  ASSERT_EQ(CountLiveLogFiles(), 2);
+  PutRandomData(0, 1, 100);
+  // [0, (1)] [0, 1]
+  ASSERT_EQ(CountLiveLogFiles(), 2);
+  PutRandomData(2, 1, 100);
+  // [0, (1)] [0, 1, 2]
+  PutRandomData(2, 1000, 100);
+  WaitForFlush(2);
+  // [0, (1)] [0, 1, (2)] [2]
+  ASSERT_EQ(CountLiveLogFiles(), 3);
+  PutRandomData(2, 1000, 100);
+  WaitForFlush(2);
+  // [0, (1)] [0, 1, (2)] [(2)] [2]
+  ASSERT_EQ(CountLiveLogFiles(), 4);
+  PutRandomData(3, 1, 100);
+  // [0, (1)] [0, 1, (2)] [(2)] [2, 3]
+  PutRandomData(1, 1, 100);
+  // [0, (1)] [0, 1, (2)] [(2)] [1, 2, 3]
+  ASSERT_EQ(CountLiveLogFiles(), 4);
+  PutRandomData(1, 1000, 100);
+  WaitForFlush(1);
+  // [0, (1)] [0, (1), (2)] [(2)] [(1), 2, 3] [1]
+  ASSERT_EQ(CountLiveLogFiles(), 5);
+  PutRandomData(0, 1000, 100);
+  WaitForFlush(0);
+  // [(0), (1)] [(0), (1), (2)] [(2)] [(1), 2, 3] [1, (0)] [0]
+  // delete obsolete logs -->
+  // [(1), 2, 3] [1, (0)] [0]
+  ASSERT_EQ(CountLiveLogFiles(), 3);
+  PutRandomData(0, 1000, 100);
+  WaitForFlush(0);
+  // [(1), 2, 3] [1, (0)], [(0)] [0]
+  ASSERT_EQ(CountLiveLogFiles(), 4);
+  PutRandomData(1, 1000, 100);
+  WaitForFlush(1);
+  // [(1), 2, 3] [(1), (0)] [(0)] [0, (1)] [1]
+  ASSERT_EQ(CountLiveLogFiles(), 5);
+  PutRandomData(2, 1000, 100);
+  WaitForFlush(2);
+  // [(1), (2), 3] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2]
+  ASSERT_EQ(CountLiveLogFiles(), 6);
+  PutRandomData(3, 1000, 100);
+  WaitForFlush(3);
+  // [(1), (2), (3)] [(1), (0)] [(0)] [0, (1)] [1, (2)], [2, (3)] [3]
+  // delete obsolete logs -->
+  // [0, (1)] [1, (2)], [2, (3)] [3]
+  ASSERT_EQ(CountLiveLogFiles(), 4);
+  Close();
+}
+
+// Makes sure that obsolete log files get deleted
+TEST(ColumnFamilyTest, DifferentWriteBufferSizes) {
+  // disable flushing stale column families
+  db_options_.max_total_wal_size = std::numeric_limits<uint64_t>::max();
+  Open();
+  CreateColumnFamilies({"one", "two", "three"});
+  ColumnFamilyOptions default_cf, one, two, three;
+  // setup options. all column families have max_write_buffer_number setup to 10
+  // "default" -> 100KB memtable, start flushing immediatelly
+  // "one" -> 200KB memtable, start flushing with two immutable memtables
+  // "two" -> 1MB memtable, start flushing with three immutable memtables
+  // "three" -> 90KB memtable, start flushing with four immutable memtables
+  default_cf.write_buffer_size = 100000;
+  default_cf.max_write_buffer_number = 10;
+  default_cf.min_write_buffer_number_to_merge = 1;
+  one.write_buffer_size = 200000;
+  one.max_write_buffer_number = 10;
+  one.min_write_buffer_number_to_merge = 2;
+  two.write_buffer_size = 1000000;
+  two.max_write_buffer_number = 10;
+  two.min_write_buffer_number_to_merge = 3;
+  three.write_buffer_size = 90000;
+  three.max_write_buffer_number = 10;
+  three.min_write_buffer_number_to_merge = 4;
+
+  Reopen({default_cf, one, two, three});
+
+  int micros_wait_for_flush = 10000;
+  PutRandomData(0, 100, 1000);
+  WaitForFlush(0);
+  AssertNumberOfImmutableMemtables({0, 0, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 1);
+  PutRandomData(1, 200, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 2);
+  PutRandomData(2, 1000, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 1, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 3);
+  PutRandomData(2, 1000, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 2, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 4);
+  PutRandomData(3, 90, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 2, 1});
+  ASSERT_EQ(CountLiveLogFiles(), 5);
+  PutRandomData(3, 90, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 2, 2});
+  ASSERT_EQ(CountLiveLogFiles(), 6);
+  PutRandomData(3, 90, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 2, 3});
+  ASSERT_EQ(CountLiveLogFiles(), 7);
+  PutRandomData(0, 100, 1000);
+  WaitForFlush(0);
+  AssertNumberOfImmutableMemtables({0, 1, 2, 3});
+  ASSERT_EQ(CountLiveLogFiles(), 8);
+  PutRandomData(2, 100, 10000);
+  WaitForFlush(2);
+  AssertNumberOfImmutableMemtables({0, 1, 0, 3});
+  ASSERT_EQ(CountLiveLogFiles(), 9);
+  PutRandomData(3, 90, 1000);
+  WaitForFlush(3);
+  AssertNumberOfImmutableMemtables({0, 1, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 10);
+  PutRandomData(3, 90, 1000);
+  env_->SleepForMicroseconds(micros_wait_for_flush);
+  AssertNumberOfImmutableMemtables({0, 1, 0, 1});
+  ASSERT_EQ(CountLiveLogFiles(), 11);
+  PutRandomData(1, 200, 1000);
+  WaitForFlush(1);
+  AssertNumberOfImmutableMemtables({0, 0, 0, 1});
+  ASSERT_EQ(CountLiveLogFiles(), 5);
+  PutRandomData(3, 90*6, 1000);
+  WaitForFlush(3);
+  AssertNumberOfImmutableMemtables({0, 0, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 12);
+  PutRandomData(0, 100, 1000);
+  WaitForFlush(0);
+  AssertNumberOfImmutableMemtables({0, 0, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 12);
+  PutRandomData(2, 3*100, 10000);
+  WaitForFlush(2);
+  AssertNumberOfImmutableMemtables({0, 0, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 12);
+  PutRandomData(1, 2*200, 1000);
+  WaitForFlush(1);
+  AssertNumberOfImmutableMemtables({0, 0, 0, 0});
+  ASSERT_EQ(CountLiveLogFiles(), 7);
+  Close();
+}
+
+TEST(ColumnFamilyTest, DifferentMergeOperators) {
+  Open();
+  CreateColumnFamilies({"first", "second"});
+  ColumnFamilyOptions default_cf, first, second;
+  first.merge_operator = MergeOperators::CreateUInt64AddOperator();
+  second.merge_operator = MergeOperators::CreateStringAppendOperator();
+  Reopen({default_cf, first, second});
+
+  std::string one, two, three;
+  PutFixed64(&one, 1);
+  PutFixed64(&two, 2);
+  PutFixed64(&three, 3);
+
+  ASSERT_OK(Put(0, "foo", two));
+  ASSERT_OK(Put(0, "foo", one));
+  ASSERT_TRUE(Merge(0, "foo", two).IsNotSupported());
+  ASSERT_EQ(Get(0, "foo"), one);
+
+  ASSERT_OK(Put(1, "foo", two));
+  ASSERT_OK(Put(1, "foo", one));
+  ASSERT_OK(Merge(1, "foo", two));
+  ASSERT_EQ(Get(1, "foo"), three);
+
+  ASSERT_OK(Put(2, "foo", two));
+  ASSERT_OK(Put(2, "foo", one));
+  ASSERT_OK(Merge(2, "foo", two));
+  ASSERT_EQ(Get(2, "foo"), one + "," + two);
+  Close();
+}
+
+TEST(ColumnFamilyTest, DifferentCompactionStyles) {
+  Open();
+  CreateColumnFamilies({"one", "two"});
+  ColumnFamilyOptions default_cf, one, two;
+  db_options_.max_open_files = 20;  // only 10 files in file cache
+  db_options_.disableDataSync = true;
+
+  default_cf.compaction_style = kCompactionStyleLevel;
+  default_cf.num_levels = 3;
+  default_cf.write_buffer_size = 64 << 10;  // 64KB
+  default_cf.target_file_size_base = 30 << 10;
+  default_cf.filter_policy = nullptr;
+  default_cf.no_block_cache = true;
+  default_cf.source_compaction_factor = 100;
+  default_cf.disable_seek_compaction = false;
+
+  one.compaction_style = kCompactionStyleUniversal;
+  // trigger compaction if there are >= 4 files
+  one.level0_file_num_compaction_trigger = 4;
+  one.write_buffer_size = 100000;
+
+  two.compaction_style = kCompactionStyleLevel;
+  two.num_levels = 4;
+  two.max_mem_compaction_level = 0;
+  two.level0_file_num_compaction_trigger = 3;
+  two.write_buffer_size = 100000;
+
+  Reopen({default_cf, one, two});
+
+  // SETUP column family "default" - test read compaction
+  ASSERT_EQ("", FilesPerLevel(0));
+  PutRandomData(0, 1, 4096);
+  ASSERT_OK(Flush(0));
+  ASSERT_EQ("0,0,1", FilesPerLevel(0));
+  // write 8MB
+  PutRandomData(0, 2000, 4096);
+  ASSERT_OK(Flush(0));
+  // clear levels 0 and 1
+  dbfull()->TEST_CompactRange(0, nullptr, nullptr, handles_[0]);
+  dbfull()->TEST_CompactRange(1, nullptr, nullptr, handles_[0]);
+  ASSERT_EQ(NumTableFilesAtLevel(0, 0), 0);
+  ASSERT_EQ(NumTableFilesAtLevel(1, 0), 0);
+  // write some new keys into level 0 and 1
+  PutRandomData(0, 1024, 512);
+  ASSERT_OK(Flush(0));
+  WaitForCompaction();
+  PutRandomData(0, 10, 512);
+  ASSERT_OK(Flush(0));
+  // remember number of files in each level
+  int l1 = NumTableFilesAtLevel(0, 0);
+  int l2 = NumTableFilesAtLevel(1, 0);
+  int l3 = NumTableFilesAtLevel(2, 0);
+  ASSERT_NE(l1, 0);
+  ASSERT_NE(l2, 0);
+  ASSERT_NE(l3, 0);
+
+  // SETUP column family "one" -- universal style
+  for (int i = 0; i < one.level0_file_num_compaction_trigger - 1; ++i) {
+    PutRandomData(1, 11, 10000);
+    WaitForFlush(1);
+    ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(1));
+  }
+
+  // SETUP column family "two" -- level style with 4 levels
+  for (int i = 0; i < two.level0_file_num_compaction_trigger - 1; ++i) {
+    PutRandomData(2, 15, 10000);
+    WaitForFlush(2);
+    ASSERT_EQ(std::to_string(i + 1), FilesPerLevel(2));
+  }
+
+  // TRIGGER compaction "default"
+  // read a bunch of times, trigger read compaction
+  for (int i = 0; i < 200000; ++i) {
+    Get(0, std::to_string(i));
+  }
+
+  // TRIGGER compaction "one"
+  PutRandomData(1, 12, 10000);
+
+  // TRIGGER compaction "two"
+  PutRandomData(2, 10, 10000);
+
+  // WAIT for compactions
+  WaitForCompaction();
+
+  // VERIFY compaction "default"
+  // verify that the number of files have decreased
+  // in some level, indicating that there was a compaction
+  ASSERT_TRUE(NumTableFilesAtLevel(0, 0) < l1 ||
+              NumTableFilesAtLevel(1, 0) < l2 ||
+              NumTableFilesAtLevel(2, 0) < l3);
+
+  // VERIFY compaction "one"
+  ASSERT_EQ("1", FilesPerLevel(1));
+
+  // VERIFY compaction "two"
+  ASSERT_EQ("0,1", FilesPerLevel(2));
+  CompactAll(2);
+  ASSERT_EQ("0,1", FilesPerLevel(2));
+
+  Close();
+}
+
+namespace {
+std::string IterStatus(Iterator* iter) {
+  std::string result;
+  if (iter->Valid()) {
+    result = iter->key().ToString() + "->" + iter->value().ToString();
+  } else {
+    result = "(invalid)";
+  }
+  return result;
+}
+}  // anonymous namespace
+
+TEST(ColumnFamilyTest, NewIteratorsTest) {
+  // iter == 0 -- no tailing
+  // iter == 2 -- tailing
+  for (int iter = 0; iter < 2; ++iter) {
+    Open();
+    CreateColumnFamiliesAndReopen({"one", "two"});
+    ASSERT_OK(Put(0, "a", "b"));
+    ASSERT_OK(Put(1, "b", "a"));
+    ASSERT_OK(Put(2, "c", "m"));
+    ASSERT_OK(Put(2, "v", "t"));
+    std::vector<Iterator*> iterators;
+    ReadOptions options;
+    options.tailing = (iter == 1);
+    ASSERT_OK(db_->NewIterators(options, handles_, &iterators));
+
+    for (auto it : iterators) {
+      it->SeekToFirst();
+    }
+    ASSERT_EQ(IterStatus(iterators[0]), "a->b");
+    ASSERT_EQ(IterStatus(iterators[1]), "b->a");
+    ASSERT_EQ(IterStatus(iterators[2]), "c->m");
+
+    ASSERT_OK(Put(1, "x", "x"));
+
+    for (auto it : iterators) {
+      it->Next();
+    }
+
+    ASSERT_EQ(IterStatus(iterators[0]), "(invalid)");
+    if (iter == 0) {
+      // no tailing
+      ASSERT_EQ(IterStatus(iterators[1]), "(invalid)");
+    } else {
+      // tailing
+      ASSERT_EQ(IterStatus(iterators[1]), "x->x");
+    }
+    ASSERT_EQ(IterStatus(iterators[2]), "v->t");
+
+    for (auto it : iterators) {
+      delete it;
+    }
+    Destroy();
+  }
+}
+
+TEST(ColumnFamilyTest, ReadOnlyDBTest) {
+  Open();
+  CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
+  ASSERT_OK(Put(1, "foo", "bla"));
+  ASSERT_OK(Put(2, "foo", "blabla"));
+  ASSERT_OK(Put(3, "foo", "blablabla"));
+  ASSERT_OK(Put(4, "foo", "blablablabla"));
+
+  DropColumnFamilies({2});
+  Close();
+  // open only a subset of column families
+  AssertOpenReadOnly({"default", "one", "four"});
+  ASSERT_EQ("NOT_FOUND", Get(0, "foo"));
+  ASSERT_EQ("bla", Get(1, "foo"));
+  ASSERT_EQ("blablablabla", Get(2, "foo"));
+
+  Close();
+  // can't open dropped column family
+  Status s = OpenReadOnly({"default", "one", "two"});
+  ASSERT_TRUE(!s.ok());
+
+  // Can't open without specifying default column family
+  s = OpenReadOnly({"one", "four"});
+  ASSERT_TRUE(!s.ok());
+}
+
+TEST(ColumnFamilyTest, DontRollEmptyLogs) {
+  Open();
+  CreateColumnFamiliesAndReopen({"one", "two", "three", "four"});
+
+  for (size_t i = 0; i < handles_.size(); ++i) {
+    PutRandomData(i, 10, 100);
+  }
+  int num_writable_file_start = env_->GetNumberOfNewWritableFileCalls();
+  // this will trigger the flushes
+  ASSERT_OK(db_->Write(WriteOptions(), nullptr));
+
+  for (int i = 0; i < 4; ++i) {
+    dbfull()->TEST_WaitForFlushMemTable(handles_[i]);
+  }
+  int total_new_writable_files =
+      env_->GetNumberOfNewWritableFileCalls() - num_writable_file_start;
+  ASSERT_EQ(static_cast<size_t>(total_new_writable_files), handles_.size() + 1);
+  Close();
+}
+
+TEST(ColumnFamilyTest, FlushStaleColumnFamilies) {
+  Open();
+  CreateColumnFamilies({"one", "two"});
+  ColumnFamilyOptions default_cf, one, two;
+  default_cf.write_buffer_size = 100000;  // small write buffer size
+  default_cf.disable_auto_compactions = true;
+  one.disable_auto_compactions = true;
+  two.disable_auto_compactions = true;
+  db_options_.max_total_wal_size = 210000;
+
+  Reopen({default_cf, one, two});
+
+  PutRandomData(2, 1, 10);  // 10 bytes
+  for (int i = 0; i < 2; ++i) {
+    PutRandomData(0, 100, 1000);  // flush
+    WaitForFlush(0);
+    ASSERT_EQ(i + 1, CountLiveFiles());
+  }
+  // third flush. now, CF [two] should be detected as stale and flushed
+  // column family 1 should not be flushed since it's empty
+  PutRandomData(0, 100, 1000);  // flush
+  WaitForFlush(0);
+  WaitForFlush(2);
+  // 3 files for default column families, 1 file for column family [two], zero
+  // files for column family [one], because it's empty
+  ASSERT_EQ(4, CountLiveFiles());
+  Close();
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}

db/compaction.cc

@@ -8,6 +8,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/compaction.h"
+#include "db/column_family.h"
 
 namespace rocksdb {
 
@@ -29,6 +30,7 @@ Compaction::Compaction(Version* input_version, int level, int out_level,
       max_grandparent_overlap_bytes_(max_grandparent_overlap_bytes),
       input_version_(input_version),
       number_levels_(input_version_->NumberLevels()),
+      cfd_(input_version_->cfd_),
       seek_compaction_(seek_compaction),
       enable_compression_(enable_compression),
       grandparent_index_(0),
@@ -42,8 +44,10 @@ Compaction::Compaction(Version* input_version, int level, int out_level,
       is_manual_compaction_(false),
       level_ptrs_(std::vector<size_t>(number_levels_)) {
 
+  cfd_->Ref();
   input_version_->Ref();
   edit_ = new VersionEdit();
+  edit_->SetColumnFamily(cfd_->GetID());
   for (int i = 0; i < number_levels_; i++) {
     level_ptrs_[i] = 0;
   }
@@ -54,6 +58,11 @@ Compaction::~Compaction() {
   if (input_version_ != nullptr) {
     input_version_->Unref();
   }
+  if (cfd_ != nullptr) {
+    if (cfd_->Unref()) {
+      delete cfd_;
+    }
+  }
 }
 
 bool Compaction::IsTrivialMove() const {
@@ -77,12 +86,11 @@ void Compaction::AddInputDeletions(VersionEdit* edit) {
 }
 
 bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
-  if (input_version_->vset_->options_->compaction_style ==
-      kCompactionStyleUniversal) {
+  if (cfd_->options()->compaction_style == kCompactionStyleUniversal) {
     return bottommost_level_;
   }
   // Maybe use binary search to find right entry instead of linear search?
-  const Comparator* user_cmp = input_version_->vset_->icmp_.user_comparator();
+  const Comparator* user_cmp = cfd_->user_comparator();
   for (int lvl = level_ + 2; lvl < number_levels_; lvl++) {
     const std::vector<FileMetaData*>& files = input_version_->files_[lvl];
     for (; level_ptrs_[lvl] < files.size(); ) {
@@ -103,7 +111,7 @@ bool Compaction::IsBaseLevelForKey(const Slice& user_key) {
 
 bool Compaction::ShouldStopBefore(const Slice& internal_key) {
   // Scan to find earliest grandparent file that contains key.
-  const InternalKeyComparator* icmp = &input_version_->vset_->icmp_;
+  const InternalKeyComparator* icmp = &cfd_->internal_comparator();
   while (grandparent_index_ < grandparents_.size() &&
       icmp->Compare(internal_key,
                     grandparents_[grandparent_index_]->largest.Encode()) > 0) {
@@ -141,8 +149,7 @@ void Compaction::MarkFilesBeingCompacted(bool value) {
 
 // Is this compaction producing files at the bottommost level?
 void Compaction::SetupBottomMostLevel(bool isManual) {
-  if (input_version_->vset_->options_->compaction_style  ==
-         kCompactionStyleUniversal) {
+  if (cfd_->options()->compaction_style == kCompactionStyleUniversal) {
     // If universal compaction style is used and manual
     // compaction is occuring, then we are guaranteed that
     // all files will be picked in a single compaction
@@ -155,8 +162,7 @@ void Compaction::SetupBottomMostLevel(bool isManual) {
     return;
   }
   bottommost_level_ = true;
-  int num_levels = input_version_->vset_->NumberLevels();
-  for (int i = output_level() + 1; i < num_levels; i++) {
+  for (int i = output_level() + 1; i < number_levels_; i++) {
     if (input_version_->NumLevelFiles(i) > 0) {
       bottommost_level_ = false;
       break;
@@ -169,6 +175,16 @@ void Compaction::ReleaseInputs() {
     input_version_->Unref();
     input_version_ = nullptr;
   }
+  if (cfd_ != nullptr) {
+    if (cfd_->Unref()) {
+      delete cfd_;
+    }
+    cfd_ = nullptr;
+  }
+}
+
+void Compaction::ReleaseCompactionFiles(Status status) {
+  cfd_->compaction_picker()->ReleaseCompactionFiles(this, status);
 }
 
 void Compaction::ResetNextCompactionIndex() {

db/compaction.h

@@ -13,6 +13,7 @@
 namespace rocksdb {
 
 class Version;
+class ColumnFamilyData;
 
 // A Compaction encapsulates information about a compaction.
 class Compaction {
@@ -36,6 +37,8 @@ class Compaction {
   // Returns input version of the compaction
   Version* input_version() const { return input_version_; }
 
+  ColumnFamilyData* column_family_data() const { return cfd_; }
+
   // Return the ith input file at "level()+which" ("which" must be 0 or 1).
   FileMetaData* input(int which, int i) const { return inputs_[which][i]; }
 
@@ -67,6 +70,10 @@ class Compaction {
   // is successful.
   void ReleaseInputs();
 
+  // Clear all files to indicate that they are not being compacted
+  // Delete this compaction from the list of running compactions.
+  void ReleaseCompactionFiles(Status status);
+
   void Summary(char* output, int len);
 
   // Return the score that was used to pick this compaction run.
@@ -97,6 +104,7 @@ class Compaction {
   Version* input_version_;
   VersionEdit* edit_;
   int number_levels_;
+  ColumnFamilyData* cfd_;
 
   bool seek_compaction_;
   bool enable_compression_;

db/compaction_picker.cc

@@ -180,7 +180,8 @@ bool CompactionPicker::ExpandWhileOverlapping(Compaction* c) {
   int parent_index = -1;
   if (c->inputs_[0].empty()) {
     Log(options_->info_log,
-        "ExpandWhileOverlapping() failure because zero input files");
+        "[%s] ExpandWhileOverlapping() failure because zero input files",
+        c->column_family_data()->GetName().c_str());
   }
   if (c->inputs_[0].empty() || FilesInCompaction(c->inputs_[0]) ||
       (c->level() != c->output_level() &&
@@ -275,16 +276,13 @@ void CompactionPicker::SetupOtherInputs(Compaction* c) {
       if (expanded1.size() == c->inputs_[1].size() &&
           !FilesInCompaction(expanded1)) {
         Log(options_->info_log,
-            "Expanding@%lu %lu+%lu (%lu+%lu bytes) to %lu+%lu (%lu+%lu bytes)"
-            "\n",
-            (unsigned long)level,
+            "[%s] Expanding@%lu %lu+%lu (%lu+%lu bytes) to %lu+%lu (%lu+%lu "
+            "bytes)\n",
+            c->column_family_data()->GetName().c_str(), (unsigned long)level,
             (unsigned long)(c->inputs_[0].size()),
-            (unsigned long)(c->inputs_[1].size()),
-            (unsigned long)inputs0_size,
-            (unsigned long)inputs1_size,
-            (unsigned long)(expanded0.size()),
-            (unsigned long)(expanded1.size()),
-            (unsigned long)expanded0_size,
+            (unsigned long)(c->inputs_[1].size()), (unsigned long)inputs0_size,
+            (unsigned long)inputs1_size, (unsigned long)(expanded0.size()),
+            (unsigned long)(expanded1.size()), (unsigned long)expanded0_size,
             (unsigned long)inputs1_size);
         smallest = new_start;
         largest = new_limit;
@@ -349,7 +347,9 @@ Compaction* CompactionPicker::CompactRange(Version* version, int input_level,
   c->inputs_[0] = inputs;
   if (ExpandWhileOverlapping(c) == false) {
     delete c;
-    Log(options_->info_log, "Could not compact due to expansion failure.\n");
+    Log(options_->info_log,
+        "[%s] Could not compact due to expansion failure.\n",
+        version->cfd_->GetName().c_str());
     return nullptr;
   }
 
@@ -519,10 +519,6 @@ Compaction* LevelCompactionPicker::PickCompactionBySize(Version* version,
       nextIndex = i;
     }
 
-    //if (i > Version::number_of_files_to_sort_) {
-    //  Log(options_->info_log, "XXX Looking at index %d", i);
-    //}
-
     // Do not pick this file if its parents at level+1 are being compacted.
     // Maybe we can avoid redoing this work in SetupOtherInputs
     int parent_index = -1;
@@ -557,19 +553,21 @@ Compaction* UniversalCompactionPicker::PickCompaction(Version* version,
 
   if ((version->files_[level].size() <
        (unsigned int)options_->level0_file_num_compaction_trigger)) {
-    LogToBuffer(log_buffer, "Universal: nothing to do\n");
+    LogToBuffer(log_buffer, "[%s] Universal: nothing to do\n",
+                version->cfd_->GetName().c_str());
     return nullptr;
   }
   Version::FileSummaryStorage tmp;
-  LogToBuffer(log_buffer, "Universal: candidate files(%zu): %s\n",
-              version->files_[level].size(),
+  LogToBuffer(log_buffer, "[%s] Universal: candidate files(%zu): %s\n",
+              version->cfd_->GetName().c_str(), version->files_[level].size(),
               version->LevelFileSummary(&tmp, 0));
 
   // Check for size amplification first.
   Compaction* c;
   if ((c = PickCompactionUniversalSizeAmp(version, score, log_buffer)) !=
       nullptr) {
-    LogToBuffer(log_buffer, "Universal: compacting for size amp\n");
+    LogToBuffer(log_buffer, "[%s] Universal: compacting for size amp\n",
+                version->cfd_->GetName().c_str());
   } else {
     // Size amplification is within limits. Try reducing read
     // amplification while maintaining file size ratios.
@@ -577,7 +575,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(Version* version,
 
     if ((c = PickCompactionUniversalReadAmp(version, score, ratio, UINT_MAX,
                                             log_buffer)) != nullptr) {
-      LogToBuffer(log_buffer, "Universal: compacting for size ratio\n");
+      LogToBuffer(log_buffer, "[%s] Universal: compacting for size ratio\n",
+                  version->cfd_->GetName().c_str());
     } else {
       // Size amplification and file size ratios are within configured limits.
       // If max read amplification is exceeding configured limits, then force
@@ -587,7 +586,8 @@ Compaction* UniversalCompactionPicker::PickCompaction(Version* version,
                                options_->level0_file_num_compaction_trigger;
       if ((c = PickCompactionUniversalReadAmp(
                version, score, UINT_MAX, num_files, log_buffer)) != nullptr) {
-        LogToBuffer(log_buffer, "Universal: compacting for file num\n");
+        LogToBuffer(log_buffer, "[%s] Universal: compacting for file num\n",
+                    version->cfd_->GetName().c_str());
       }
     }
   }
@@ -675,9 +675,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
         candidate_count = 1;
         break;
       }
-      LogToBuffer(log_buffer,
-                  "Universal: file %lu[%d] being compacted, skipping",
-                  (unsigned long)f->number, loop);
+      LogToBuffer(
+          log_buffer, "[%s] Universal: file %lu[%d] being compacted, skipping",
+          version->cfd_->GetName().c_str(), (unsigned long)f->number, loop);
       f = nullptr;
     }
 
@@ -685,8 +685,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
     // first candidate to be compacted.
     uint64_t candidate_size =  f != nullptr? f->file_size : 0;
     if (f != nullptr) {
-      LogToBuffer(log_buffer, "Universal: Possible candidate file %lu[%d].",
-                  (unsigned long)f->number, loop);
+      LogToBuffer(
+          log_buffer, "[%s] Universal: Possible candidate file %lu[%d].",
+          version->cfd_->GetName().c_str(), (unsigned long)f->number, loop);
     }
 
     // Check if the suceeding files need compaction.
@@ -737,9 +738,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
        int index = file_by_time[i];
        FileMetaData* f = version->files_[level][index];
        LogToBuffer(log_buffer,
-                   "Universal: Skipping file %lu[%d] with size %lu %d\n",
-                   (unsigned long)f->number, i, (unsigned long)f->file_size,
-                   f->being_compacted);
+                   "[%s] Universal: Skipping file %lu[%d] with size %lu %d\n",
+                   version->cfd_->GetName().c_str(), (unsigned long)f->number,
+                   i, (unsigned long)f->file_size, f->being_compacted);
       }
     }
   }
@@ -773,8 +774,10 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalReadAmp(
     int index = file_by_time[i];
     FileMetaData* f = c->input_version_->files_[level][index];
     c->inputs_[0].push_back(f);
-    LogToBuffer(log_buffer, "Universal: Picking file %lu[%d] with size %lu\n",
-                (unsigned long)f->number, i, (unsigned long)f->file_size);
+    LogToBuffer(log_buffer,
+                "[%s] Universal: Picking file %lu[%d] with size %lu\n",
+                version->cfd_->GetName().c_str(), (unsigned long)f->number, i,
+                (unsigned long)f->file_size);
   }
   return c;
 }
@@ -810,17 +813,19 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
       start_index = loop;         // Consider this as the first candidate.
       break;
     }
-    LogToBuffer(log_buffer, "Universal: skipping file %lu[%d] compacted %s",
-                (unsigned long)f->number, loop,
-                " cannot be a candidate to reduce size amp.\n");
+    LogToBuffer(log_buffer,
+                "[%s] Universal: skipping file %lu[%d] compacted %s",
+                version->cfd_->GetName().c_str(), (unsigned long)f->number,
+                loop, " cannot be a candidate to reduce size amp.\n");
     f = nullptr;
   }
   if (f == nullptr) {
     return nullptr;             // no candidate files
   }
 
-  LogToBuffer(log_buffer, "Universal: First candidate file %lu[%d] %s",
-              (unsigned long)f->number, start_index, " to reduce size amp.\n");
+  LogToBuffer(log_buffer, "[%s] Universal: First candidate file %lu[%d] %s",
+              version->cfd_->GetName().c_str(), (unsigned long)f->number,
+              start_index, " to reduce size amp.\n");
 
   // keep adding up all the remaining files
   for (unsigned int loop = start_index; loop < file_by_time.size() - 1;
@@ -829,8 +834,8 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
     f = version->files_[level][index];
     if (f->being_compacted) {
       LogToBuffer(
-          log_buffer, "Universal: Possible candidate file %lu[%d] %s.",
-          (unsigned long)f->number, loop,
+          log_buffer, "[%s] Universal: Possible candidate file %lu[%d] %s.",
+          version->cfd_->GetName().c_str(), (unsigned long)f->number, loop,
           " is already being compacted. No size amp reduction possible.\n");
       return nullptr;
     }
@@ -847,17 +852,18 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
 
   // size amplification = percentage of additional size
   if (candidate_size * 100 < ratio * earliest_file_size) {
-    LogToBuffer(log_buffer,
-                "Universal: size amp not needed. newer-files-total-size %lu "
-                "earliest-file-size %lu",
-                (unsigned long)candidate_size,
-                (unsigned long)earliest_file_size);
+    LogToBuffer(
+        log_buffer,
+        "[%s] Universal: size amp not needed. newer-files-total-size %lu "
+        "earliest-file-size %lu",
+        version->cfd_->GetName().c_str(), (unsigned long)candidate_size,
+        (unsigned long)earliest_file_size);
     return nullptr;
   } else {
     LogToBuffer(log_buffer,
-                "Universal: size amp needed. newer-files-total-size %lu "
+                "[%s] Universal: size amp needed. newer-files-total-size %lu "
                 "earliest-file-size %lu",
-                (unsigned long)candidate_size,
+                version->cfd_->GetName().c_str(), (unsigned long)candidate_size,
                 (unsigned long)earliest_file_size);
   }
   assert(start_index >= 0 && start_index < file_by_time.size() - 1);
@@ -873,8 +879,9 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp(
     f = c->input_version_->files_[level][index];
     c->inputs_[0].push_back(f);
     LogToBuffer(log_buffer,
-                "Universal: size amp picking file %lu[%d] with size %lu",
-                (unsigned long)f->number, index, (unsigned long)f->file_size);
+                "[%s] Universal: size amp picking file %lu[%d] with size %lu",
+                version->cfd_->GetName().c_str(), (unsigned long)f->number,
+                index, (unsigned long)f->file_size);
   }
   return c;
 }

db/compaction_picker.h

@@ -12,6 +12,7 @@
 #include "db/compaction.h"
 #include "rocksdb/status.h"
 #include "rocksdb/options.h"
+#include "rocksdb/env.h"
 
 #include <vector>
 #include <memory>
@@ -118,6 +119,7 @@ class CompactionPicker {
   std::unique_ptr<uint64_t[]> level_max_bytes_;
 
   const Options* const options_;
+
  private:
   int num_levels_;
 

db/corruption_test.cc

@@ -40,7 +40,7 @@ class CorruptionTest {
   CorruptionTest() {
     tiny_cache_ = NewLRUCache(100);
     options_.env = &env_;
-    dbname_ = test::TmpDir() + "/db_test";
+    dbname_ = test::TmpDir() + "/corruption_test";
     DestroyDB(dbname_, options_);
 
     db_ = nullptr;
@@ -127,24 +127,7 @@ class CorruptionTest {
     ASSERT_GE(max_expected, correct);
   }
 
-  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
-    // Pick file to corrupt
-    std::vector<std::string> filenames;
-    ASSERT_OK(env_.GetChildren(dbname_, &filenames));
-    uint64_t number;
-    FileType type;
-    std::string fname;
-    int picked_number = -1;
-    for (unsigned int i = 0; i < filenames.size(); i++) {
-      if (ParseFileName(filenames[i], &number, &type) &&
-          type == filetype &&
-          int(number) > picked_number) {  // Pick latest file
-        fname = dbname_ + "/" + filenames[i];
-        picked_number = number;
-      }
-    }
-    ASSERT_TRUE(!fname.empty()) << filetype;
-
+  void CorruptFile(const std::string fname, int offset, int bytes_to_corrupt) {
     struct stat sbuf;
     if (stat(fname.c_str(), &sbuf) != 0) {
       const char* msg = strerror(errno);
@@ -177,6 +160,42 @@ class CorruptionTest {
     ASSERT_TRUE(s.ok()) << s.ToString();
   }
 
+  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
+    // Pick file to corrupt
+    std::vector<std::string> filenames;
+    ASSERT_OK(env_.GetChildren(dbname_, &filenames));
+    uint64_t number;
+    FileType type;
+    std::string fname;
+    int picked_number = -1;
+    for (unsigned int i = 0; i < filenames.size(); i++) {
+      if (ParseFileName(filenames[i], &number, &type) &&
+          type == filetype &&
+          static_cast<int>(number) > picked_number) {  // Pick latest file
+        fname = dbname_ + "/" + filenames[i];
+        picked_number = number;
+      }
+    }
+    ASSERT_TRUE(!fname.empty()) << filetype;
+
+    CorruptFile(fname, offset, bytes_to_corrupt);
+  }
+
+  // corrupts exactly one file at level `level`. if no file found at level,
+  // asserts
+  void CorruptTableFileAtLevel(int level, int offset, int bytes_to_corrupt) {
+    std::vector<LiveFileMetaData> metadata;
+    db_->GetLiveFilesMetaData(&metadata);
+    for (const auto& m : metadata) {
+      if (m.level == level) {
+        CorruptFile(dbname_ + "/" + m.name, offset, bytes_to_corrupt);
+        return;
+      }
+    }
+    ASSERT_TRUE(false) << "no file found at level";
+  }
+
+
   int Property(const std::string& name) {
     std::string property;
     int result;
@@ -326,30 +345,35 @@ TEST(CorruptionTest, CompactionInputError) {
 TEST(CorruptionTest, CompactionInputErrorParanoid) {
   Options options;
   options.paranoid_checks = true;
-  options.write_buffer_size = 1048576;
+  options.write_buffer_size = 131072;
+  options.max_write_buffer_number = 2;
   Reopen(&options);
   DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
 
-  // Fill levels >= 1 so memtable compaction outputs to level 1
+  // Fill levels >= 1 so memtable flush outputs to level 0
   for (int level = 1; level < dbi->NumberLevels(); level++) {
     dbi->Put(WriteOptions(), "", "begin");
     dbi->Put(WriteOptions(), "~", "end");
     dbi->TEST_FlushMemTable();
   }
 
+  options.max_mem_compaction_level = 0;
+  Reopen(&options);
+
+  dbi = reinterpret_cast<DBImpl*>(db_);
   Build(10);
   dbi->TEST_FlushMemTable();
   dbi->TEST_WaitForCompact();
   ASSERT_EQ(1, Property("rocksdb.num-files-at-level0"));
 
-  Corrupt(kTableFile, 100, 1);
+  CorruptTableFileAtLevel(0, 100, 1);
   Check(9, 9);
 
   // Write must eventually fail because of corrupted table
   Status s;
   std::string tmp1, tmp2;
   bool failed = false;
-  for (int i = 0; i < 10000 && s.ok(); i++) {
+  for (int i = 0; i < 10000; i++) {
     s = db_->Put(WriteOptions(), Key(i, &tmp1), Value(i, &tmp2));
     if (!s.ok()) {
       failed = true;

db/db_filesnapshot.cc

@@ -7,6 +7,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+#ifndef ROCKSDB_LITE
+
 #define __STDC_FORMAT_MACROS
 #include <inttypes.h>
 #include <algorithm>
@@ -63,21 +65,36 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
 
   *manifest_file_size = 0;
 
+  mutex_.Lock();
+
   if (flush_memtable) {
     // flush all dirty data to disk.
-    Status status =  Flush(FlushOptions());
+    Status status;
+    for (auto cfd : *versions_->GetColumnFamilySet()) {
+      cfd->Ref();
+      mutex_.Unlock();
+      status = FlushMemTable(cfd, FlushOptions());
+      mutex_.Lock();
+      cfd->Unref();
+      if (!status.ok()) {
+        break;
+      }
+    }
+    versions_->GetColumnFamilySet()->FreeDeadColumnFamilies();
+
     if (!status.ok()) {
+      mutex_.Unlock();
       Log(options_.info_log, "Cannot Flush data %s\n",
           status.ToString().c_str());
       return status;
     }
   }
 
-  MutexLock l(&mutex_);
-
   // Make a set of all of the live *.sst files
   std::set<uint64_t> live;
-  versions_->current()->AddLiveFiles(&live);
+  for (auto cfd : *versions_->GetColumnFamilySet()) {
+    cfd->current()->AddLiveFiles(&live);
+  }
 
   ret.clear();
   ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
@@ -94,6 +111,7 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
   // find length of manifest file while holding the mutex lock
   *manifest_file_size = versions_->ManifestFileSize();
 
+  mutex_.Unlock();
   return Status::OK();
 }
 
@@ -150,3 +168,5 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
 }
 
 }
+
+#endif  // ROCKSDB_LITE

db/db_impl.h

@@ -13,10 +13,12 @@
 #include <set>
 #include <utility>
 #include <vector>
+#include <string>
 
 #include "db/dbformat.h"
 #include "db/log_writer.h"
 #include "db/snapshot.h"
+#include "db/column_family.h"
 #include "db/version_edit.h"
 #include "memtable_list.h"
 #include "port/port.h"
@@ -40,44 +42,83 @@ class CompactionFilterV2;
 
 class DBImpl : public DB {
  public:
-  DBImpl(const Options& options, const std::string& dbname);
+  DBImpl(const DBOptions& options, const std::string& dbname);
   virtual ~DBImpl();
 
   // Implementations of the DB interface
-  virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value);
-  virtual Status Merge(const WriteOptions&, const Slice& key,
+  using DB::Put;
+  virtual Status Put(const WriteOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     const Slice& value);
+  using DB::Merge;
+  virtual Status Merge(const WriteOptions& options,
+                       ColumnFamilyHandle* column_family, const Slice& key,
                        const Slice& value);
-  virtual Status Delete(const WriteOptions&, const Slice& key);
+  using DB::Delete;
+  virtual Status Delete(const WriteOptions& options,
+                        ColumnFamilyHandle* column_family, const Slice& key);
+  using DB::Write;
   virtual Status Write(const WriteOptions& options, WriteBatch* updates);
+  using DB::Get;
   virtual Status Get(const ReadOptions& options,
-                     const Slice& key,
+                     ColumnFamilyHandle* column_family, const Slice& key,
                      std::string* value);
-  virtual std::vector<Status> MultiGet(const ReadOptions& options,
-                                       const std::vector<Slice>& keys,
-                                       std::vector<std::string>* values);
+  using DB::MultiGet;
+  virtual std::vector<Status> MultiGet(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_family,
+      const std::vector<Slice>& keys, std::vector<std::string>* values);
+
+  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
+                                    const std::string& column_family,
+                                    ColumnFamilyHandle** handle);
+  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family);
 
   // Returns false if key doesn't exist in the database and true if it may.
   // If value_found is not passed in as null, then return the value if found in
   // memory. On return, if value was found, then value_found will be set to true
   // , otherwise false.
+  using DB::KeyMayExist;
   virtual bool KeyMayExist(const ReadOptions& options,
-                           const Slice& key,
-                           std::string* value,
-                           bool* value_found = nullptr);
-  virtual Iterator* NewIterator(const ReadOptions&);
+                           ColumnFamilyHandle* column_family, const Slice& key,
+                           std::string* value, bool* value_found = nullptr);
+  using DB::NewIterator;
+  virtual Iterator* NewIterator(const ReadOptions& options,
+                                ColumnFamilyHandle* column_family);
+  virtual Status NewIterators(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_families,
+      std::vector<Iterator*>* iterators);
   virtual const Snapshot* GetSnapshot();
   virtual void ReleaseSnapshot(const Snapshot* snapshot);
-  virtual bool GetProperty(const Slice& property, std::string* value);
-  virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
-  virtual Status CompactRange(const Slice* begin, const Slice* end,
+  using DB::GetProperty;
+  virtual bool GetProperty(ColumnFamilyHandle* column_family,
+                           const Slice& property, std::string* value);
+  using DB::GetApproximateSizes;
+  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
+                                   const Range* range, int n, uint64_t* sizes);
+  using DB::CompactRange;
+  virtual Status CompactRange(ColumnFamilyHandle* column_family,
+                              const Slice* begin, const Slice* end,
                               bool reduce_level = false, int target_level = -1);
-  virtual int NumberLevels();
-  virtual int MaxMemCompactionLevel();
-  virtual int Level0StopWriteTrigger();
+
+  using DB::NumberLevels;
+  virtual int NumberLevels(ColumnFamilyHandle* column_family);
+  using DB::MaxMemCompactionLevel;
+  virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family);
+  using DB::Level0StopWriteTrigger;
+  virtual int Level0StopWriteTrigger(ColumnFamilyHandle* column_family);
   virtual const std::string& GetName() const;
   virtual Env* GetEnv() const;
-  virtual const Options& GetOptions() const;
-  virtual Status Flush(const FlushOptions& options);
+  using DB::GetOptions;
+  virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const;
+  using DB::Flush;
+  virtual Status Flush(const FlushOptions& options,
+                       ColumnFamilyHandle* column_family);
+
+  virtual SequenceNumber GetLatestSequenceNumber() const;
+
+#ifndef ROCKSDB_LITE
   virtual Status DisableFileDeletions();
   virtual Status EnableFileDeletions(bool force);
   // All the returned filenames start with "/"
@@ -85,15 +126,15 @@ class DBImpl : public DB {
                               uint64_t* manifest_file_size,
                               bool flush_memtable = true);
   virtual Status GetSortedWalFiles(VectorLogPtr& files);
-  virtual SequenceNumber GetLatestSequenceNumber() const;
+
   virtual Status GetUpdatesSince(
       SequenceNumber seq_number, unique_ptr<TransactionLogIterator>* iter,
       const TransactionLogIterator::ReadOptions&
           read_options = TransactionLogIterator::ReadOptions());
   virtual Status DeleteFile(std::string name);
 
-  virtual void GetLiveFilesMetaData(
-    std::vector<LiveFileMetaData> *metadata);
+  virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata);
+#endif  // ROCKSDB_LITE
 
   // checks if all live files exist on file system and that their file sizes
   // match to our in-memory records
@@ -101,23 +142,23 @@ class DBImpl : public DB {
 
   virtual Status GetDbIdentity(std::string& identity);
 
-  Status RunManualCompaction(int input_level,
-                             int output_level,
-                             const Slice* begin,
+  Status RunManualCompaction(ColumnFamilyData* cfd, int input_level,
+                             int output_level, const Slice* begin,
                              const Slice* end);
 
+#ifndef ROCKSDB_LITE
   // Extra methods (for testing) that are not in the public DB interface
+  // Implemented in db_impl_debug.cc
 
   // Compact any files in the named level that overlap [*begin, *end]
-  Status TEST_CompactRange(int level,
-                           const Slice* begin,
-                           const Slice* end);
+  Status TEST_CompactRange(int level, const Slice* begin, const Slice* end,
+                           ColumnFamilyHandle* column_family = nullptr);
 
   // Force current memtable contents to be flushed.
   Status TEST_FlushMemTable(bool wait = true);
 
   // Wait for memtable compaction
-  Status TEST_WaitForFlushMemTable();
+  Status TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family = nullptr);
 
   // Wait for any compaction
   Status TEST_WaitForCompact();
@@ -125,14 +166,13 @@ class DBImpl : public DB {
   // Return an internal iterator over the current state of the database.
   // The keys of this iterator are internal keys (see format.h).
   // The returned iterator should be deleted when no longer needed.
-  Iterator* TEST_NewInternalIterator();
+  Iterator* TEST_NewInternalIterator(ColumnFamilyHandle* column_family =
+                                         nullptr);
 
   // Return the maximum overlapping data (in bytes) at next level for any
   // file at a level >= 1.
-  int64_t TEST_MaxNextLevelOverlappingBytes();
-
-  // Simulate a db crash, no elegant closing of database.
-  void TEST_Destroy_DBImpl();
+  int64_t TEST_MaxNextLevelOverlappingBytes(ColumnFamilyHandle* column_family =
+                                                nullptr);
 
   // Return the current manifest file no.
   uint64_t TEST_Current_Manifest_FileNo();
@@ -148,61 +188,14 @@ class DBImpl : public DB {
     default_interval_to_delete_obsolete_WAL_ = default_interval_to_delete_obsolete_WAL;
   }
 
-  void TEST_GetFilesMetaData(std::vector<std::vector<FileMetaData>>* metadata);
+  void TEST_GetFilesMetaData(ColumnFamilyHandle* column_family,
+                             std::vector<std::vector<FileMetaData>>* metadata);
 
-  // holds references to memtable, all immutable memtables and version
-  struct SuperVersion {
-    MemTable* mem;
-    MemTableListVersion* imm;
-    Version* current;
-    std::atomic<uint32_t> refs;
-    // We need to_delete because during Cleanup(), imm->Unref() returns
-    // all memtables that we need to free through this vector. We then
-    // delete all those memtables outside of mutex, during destruction
-    autovector<MemTable*> to_delete;
-    // Version number of the current SuperVersion
-    uint64_t version_number;
-    DBImpl* db;
+  Status TEST_ReadFirstRecord(const WalFileType type, const uint64_t number,
+                              SequenceNumber* sequence);
 
-    // should be called outside the mutex
-    SuperVersion() = default;
-    ~SuperVersion();
-    SuperVersion* Ref();
-    // Returns true if this was the last reference and caller should
-    // call Clenaup() and delete the object
-    bool Unref();
-
-    // call these two methods with db mutex held
-    // Cleanup unrefs mem, imm and current. Also, it stores all memtables
-    // that needs to be deleted in to_delete vector. Unrefing those
-    // objects needs to be done in the mutex
-    void Cleanup();
-    void Init(MemTable* new_mem, MemTableListVersion* new_imm,
-              Version* new_current);
-
-    // The value of dummy is not actually used. kSVInUse takes its address as a
-    // mark in the thread local storage to indicate the SuperVersion is in use
-    // by thread. This way, the value of kSVInUse is guaranteed to have no
-    // conflict with SuperVersion object address and portable on different
-    // platform.
-    static int dummy;
-    static void* const kSVInUse;
-    static void* const kSVObsolete;
-  };
-
-  static void SuperVersionUnrefHandle(void* ptr) {
-    // UnrefHandle is called when a thread exists or a ThreadLocalPtr gets
-    // destroyed. When former happens, the thread shouldn't see kSVInUse.
-    // When latter happens, we are in ~DBImpl(), no get should happen as well.
-    assert(ptr != SuperVersion::kSVInUse);
-    DBImpl::SuperVersion* sv = static_cast<DBImpl::SuperVersion*>(ptr);
-    if (sv->Unref()) {
-      sv->db->mutex_.Lock();
-      sv->Cleanup();
-      sv->db->mutex_.Unlock();
-      delete sv;
-    }
-  }
+  Status TEST_ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
+#endif  // NDEBUG
 
   // needed for CleanupIteratorState
   struct DeletionState {
@@ -231,7 +224,7 @@ class DBImpl : public DB {
 
     autovector<SuperVersion*> superversions_to_free;
 
-    SuperVersion* new_superversion; // if nullptr no new superversion
+    SuperVersion* new_superversion;  // if nullptr no new superversion
 
     // the current manifest_file_number, log_number and prev_log_number
     // that corresponds to the set of files in 'live'.
@@ -243,8 +236,7 @@ class DBImpl : public DB {
       pending_manifest_file_number = 0;
       log_number = 0;
       prev_log_number = 0;
-      new_superversion =
-          create_superversion ? new SuperVersion() : nullptr;
+      new_superversion = create_superversion ? new SuperVersion() : nullptr;
     }
 
     ~DeletionState() {
@@ -277,28 +269,23 @@ class DBImpl : public DB {
   // It is not necessary to hold the mutex when invoking this method.
   void PurgeObsoleteFiles(DeletionState& deletion_state);
 
+  ColumnFamilyHandle* DefaultColumnFamily() const;
+
  protected:
   Env* const env_;
   const std::string dbname_;
   unique_ptr<VersionSet> versions_;
-  const InternalKeyComparator internal_comparator_;
-  const Options options_;  // options_.comparator == &internal_comparator_
+  const DBOptions options_;
 
-  const Comparator* user_comparator() const {
-    return internal_comparator_.user_comparator();
-  }
-
-  SuperVersion* GetSuperVersion() {
-    return super_version_;
-  }
-
-  Iterator* NewInternalIterator(const ReadOptions&,
-                                SequenceNumber* latest_snapshot);
+  Iterator* NewInternalIterator(const ReadOptions&, ColumnFamilyData* cfd,
+                                SuperVersion* super_version);
 
  private:
   friend class DB;
   friend class InternalStats;
+#ifndef ROCKSDB_LITE
   friend class TailingIterator;
+#endif
   friend struct SuperVersion;
   struct CompactionState;
   struct Writer;
@@ -306,8 +293,10 @@ class DBImpl : public DB {
   Status NewDB();
 
   // Recover the descriptor from persistent storage.  May do a significant
-  // amount of work to recover recently logged updates.
-  Status Recover(bool read_only = false, bool error_if_log_file_exist = false);
+  // amount of work to recover recently logged updates.  Any changes to
+  // be made to the descriptor are added to *edit.
+  Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
+                 bool read_only = false, bool error_if_log_file_exist = false);
 
   void MaybeIgnoreError(Status* s) const;
 
@@ -318,7 +307,7 @@ class DBImpl : public DB {
 
   // Flush the in-memory write buffer to storage.  Switches to a new
   // log-file/memtable and writes a new descriptor iff successful.
-  Status FlushMemTableToOutputFile(bool* madeProgress,
+  Status FlushMemTableToOutputFile(ColumnFamilyData* cfd, bool* madeProgress,
                                    DeletionState& deletion_state,
                                    LogBuffer* log_buffer);
 
@@ -330,32 +319,35 @@ class DBImpl : public DB {
   // database is opened) and is heavyweight because it holds the mutex
   // for the entire period. The second method WriteLevel0Table supports
   // concurrent flush memtables to storage.
-  Status WriteLevel0TableForRecovery(MemTable* mem, VersionEdit* edit);
-  Status WriteLevel0Table(autovector<MemTable*>& mems, VersionEdit* edit,
-                          uint64_t* filenumber,
+  Status WriteLevel0TableForRecovery(ColumnFamilyData* cfd, MemTable* mem,
+                                     VersionEdit* edit);
+  Status WriteLevel0Table(ColumnFamilyData* cfd, autovector<MemTable*>& mems,
+                          VersionEdit* edit, uint64_t* filenumber,
                           LogBuffer* log_buffer);
 
   uint64_t SlowdownAmount(int n, double bottom, double top);
-  // MakeRoomForWrite will return superversion_to_free through an arugment,
-  // which the caller needs to delete. We do it because caller can delete
-  // the superversion outside of mutex.
-  // old_log if not nullptr is the old log writer that should be safely
-  // closed whenever DB mutex is released.
-  Status MakeRoomForWrite(bool force /* compact even if there is room? */,
-                          SuperVersion** superversion_to_free,
-                          log::Writer** old_log);
+
+  // TODO(icanadi) free superversion_to_free and old_log outside of mutex
+  Status MakeRoomForWrite(ColumnFamilyData* cfd,
+                          bool force /* flush even if there is room? */,
+                          autovector<SuperVersion*>* superversions_to_free,
+                          autovector<log::Writer*>* logs_to_free);
+
   void BuildBatchGroup(Writer** last_writer,
                        autovector<WriteBatch*>* write_batch_group);
 
   // Force current memtable contents to be flushed.
-  Status FlushMemTable(const FlushOptions& options);
+  Status FlushMemTable(ColumnFamilyData* cfd, const FlushOptions& options);
 
   // Wait for memtable flushed
-  Status WaitForFlushMemTable();
+  Status WaitForFlushMemTable(ColumnFamilyData* cfd);
 
   void MaybeScheduleLogDBDeployStats();
+
+#ifndef ROCKSDB_LITE
   static void BGLogDBDeployStats(void* db);
   void LogDBDeployStats();
+#endif  // ROCKSDB_LITE
 
   void MaybeScheduleFlushOrCompaction();
   static void BGWorkCompaction(void* db);
@@ -371,6 +363,13 @@ class DBImpl : public DB {
                           DeletionState& deletion_state,
                           LogBuffer* log_buffer);
 
+  // This function is called as part of compaction. It enables Flush process to
+  // preempt compaction, since it's higher prioirty
+  // Returns: micros spent executing
+  uint64_t CallFlushDuringCompaction(ColumnFamilyData* cfd,
+                                     DeletionState& deletion_state,
+                                     LogBuffer* log_buffer);
+
   // Call compaction filter if is_compaction_v2 is not true. Then iterate
   // through input and compact the kv-pairs
   Status ProcessKeyValueCompaction(
@@ -396,6 +395,12 @@ class DBImpl : public DB {
   void AllocateCompactionOutputFileNumbers(CompactionState* compact);
   void ReleaseCompactionUnusedFileNumbers(CompactionState* compact);
 
+#ifdef ROCKSDB_LITE
+  void PurgeObsoleteWALFiles() {
+    // this function is used for archiving WAL files. we don't need this in
+    // ROCKSDB_LITE
+  }
+#else
   void PurgeObsoleteWALFiles();
 
   Status GetSortedWalsOfType(const std::string& path,
@@ -407,46 +412,36 @@ class DBImpl : public DB {
   // Greater Than or Equal to the requested SequenceNumber.
   Status RetainProbableWalFiles(VectorLogPtr& all_logs,
                                 const SequenceNumber target);
-  //  return true if
-  bool CheckWalFileExistsAndEmpty(const WalFileType type,
-                                  const uint64_t number);
 
   Status ReadFirstRecord(const WalFileType type, const uint64_t number,
-                         WriteBatch* const result);
+                         SequenceNumber* sequence);
 
-  Status ReadFirstLine(const std::string& fname, WriteBatch* const batch);
+  Status ReadFirstLine(const std::string& fname, SequenceNumber* sequence);
+#endif  // ROCKSDB_LITE
 
   void PrintStatistics();
 
   // dump rocksdb.stats to LOG
   void MaybeDumpStats();
 
+  // Return true if the current db supports snapshot.  If the current
+  // DB does not support snapshot, then calling GetSnapshot() will always
+  // return nullptr.
+  //
+  // @see GetSnapshot()
+  virtual bool IsSnapshotSupported() const;
+
   // Return the minimum empty level that could hold the total data in the
   // input level. Return the input level, if such level could not be found.
-  int FindMinimumEmptyLevelFitting(int level);
+  int FindMinimumEmptyLevelFitting(ColumnFamilyData* cfd, int level);
 
   // Move the files in the input level to the target level.
   // If target_level < 0, automatically calculate the minimum level that could
   // hold the data set.
-  Status ReFitLevel(int level, int target_level = -1);
-
-  // Returns the current SuperVersion number.
-  uint64_t CurrentVersionNumber() const;
-
-  // Returns a pair of iterators (mutable-only and immutable-only) used
-  // internally by TailingIterator and stores CurrentVersionNumber() in
-  // *superversion_number. These iterators are always up-to-date, i.e. can
-  // be used to read new data.
-  std::pair<Iterator*, Iterator*> GetTailingIteratorPair(
-    const ReadOptions& options,
-    uint64_t* superversion_number);
-
-  // Constant after construction
-  const InternalFilterPolicy internal_filter_policy_;
-  bool owns_info_log_;
+  Status ReFitLevel(ColumnFamilyData* cfd, int level, int target_level = -1);
 
   // table_cache_ provides its own synchronization
-  unique_ptr<TableCache> table_cache_;
+  std::shared_ptr<Cache> table_cache_;
 
   // Lock over the persistent DB state.  Non-nullptr iff successfully acquired.
   FileLock* db_lock_;
@@ -455,20 +450,24 @@ class DBImpl : public DB {
   port::Mutex mutex_;
   port::AtomicPointer shutting_down_;
   port::CondVar bg_cv_;          // Signalled when background work finishes
-  MemTable* mem_;
-  MemTableList imm_;             // Memtable that are not changing
   uint64_t logfile_number_;
   unique_ptr<log::Writer> log_;
-
-  SuperVersion* super_version_;
-
-  // An ordinal representing the current SuperVersion. Updated by
-  // InstallSuperVersion(), i.e. incremented every time super_version_
-  // changes.
-  std::atomic<uint64_t> super_version_number_;
-  // Thread's local copy of SuperVersion pointer
-  // This needs to be destructed after mutex_
-  ThreadLocalPtr* local_sv_;
+  bool log_empty_;
+  ColumnFamilyHandleImpl* default_cf_handle_;
+  unique_ptr<ColumnFamilyMemTablesImpl> column_family_memtables_;
+  struct LogFileNumberSize {
+    explicit LogFileNumberSize(uint64_t _number)
+        : number(_number), size(0), getting_flushed(false) {}
+    void AddSize(uint64_t new_size) { size += new_size; }
+    uint64_t number;
+    uint64_t size;
+    bool getting_flushed;
+  };
+  std::deque<LogFileNumberSize> alive_log_files_;
+  uint64_t total_log_size_;
+  // only used for dynamically adjusting max_total_wal_size. it is a sum of
+  // [write_buffer_size * max_write_buffer_number] over all column families
+  uint64_t max_total_in_memory_state_;
 
   std::string host_name_;
 
@@ -480,6 +479,10 @@ class DBImpl : public DB {
 
   SnapshotList snapshots_;
 
+  // cache for ReadFirstRecord() calls
+  std::unordered_map<uint64_t, SequenceNumber> read_first_record_cache_;
+  port::Mutex read_first_record_cache_mutex_;
+
   // Set of table files to protect from deletion because they are
   // part of ongoing compactions.
   std::set<uint64_t> pending_outputs_;
@@ -504,6 +507,7 @@ class DBImpl : public DB {
 
   // Information for a manual compaction
   struct ManualCompaction {
+    ColumnFamilyData* cfd;
     int input_level;
     int output_level;
     bool done;
@@ -545,8 +549,6 @@ class DBImpl : public DB {
 
   bool flush_on_destroy_; // Used when disableWAL is true.
 
-  InternalStats internal_stats_;
-
   static const int KEEP_LOG_FILE_NUM = 1000;
   std::string db_absolute_path_;
 
@@ -579,28 +581,23 @@ class DBImpl : public DB {
     std::vector<SequenceNumber>& snapshots,
     SequenceNumber* prev_snapshot);
 
-  // will return a pointer to SuperVersion* if previous SuperVersion
-  // if its reference count is zero and needs deletion or nullptr if not
-  // As argument takes a pointer to allocated SuperVersion
-  // Foreground threads call this function directly (they don't carry
-  // deletion state and have to handle their own creation and deletion
-  // of SuperVersion)
-  SuperVersion* InstallSuperVersion(SuperVersion* new_superversion);
   // Background threads call this function, which is just a wrapper around
-  // the InstallSuperVersion() function above. Background threads carry
+  // the cfd->InstallSuperVersion() function. Background threads carry
   // deletion_state which can have new_superversion already allocated.
-  void InstallSuperVersion(DeletionState& deletion_state);
+  void InstallSuperVersion(ColumnFamilyData* cfd,
+                           DeletionState& deletion_state);
 
-  void ResetThreadLocalSuperVersions(DeletionState* deletion_state);
-
-  virtual Status GetPropertiesOfAllTables(TablePropertiesCollection* props)
+#ifndef ROCKSDB_LITE
+  using DB::GetPropertiesOfAllTables;
+  virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
+                                          TablePropertiesCollection* props)
       override;
+#endif  // ROCKSDB_LITE
 
   // Function that Get and KeyMayExist call with no_io true or false
   // Note: 'value_found' from KeyMayExist propagates here
-  Status GetImpl(const ReadOptions& options,
-                 const Slice& key,
-                 std::string* value,
+  Status GetImpl(const ReadOptions& options, ColumnFamilyHandle* column_family,
+                 const Slice& key, std::string* value,
                  bool* value_found = nullptr);
 };
 
@@ -610,7 +607,7 @@ extern Options SanitizeOptions(const std::string& db,
                                const InternalKeyComparator* icmp,
                                const InternalFilterPolicy* ipolicy,
                                const Options& src);
-
+extern DBOptions SanitizeOptions(const std::string& db, const DBOptions& src);
 
 // Determine compression type, based on user options, level of the output
 // file and whether compression is disabled.

db/db_impl_debug.cc

@@ -0,0 +1,132 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#ifndef ROCKSDB_LITE
+
+#include "db/db_impl.h"
+
+namespace rocksdb {
+
+void DBImpl::TEST_PurgeObsoleteteWAL() { PurgeObsoleteWALFiles(); }
+
+uint64_t DBImpl::TEST_GetLevel0TotalSize() {
+  MutexLock l(&mutex_);
+  return default_cf_handle_->cfd()->current()->NumLevelBytes(0);
+}
+
+Iterator* DBImpl::TEST_NewInternalIterator(ColumnFamilyHandle* column_family) {
+  ColumnFamilyData* cfd;
+  if (column_family == nullptr) {
+    cfd = default_cf_handle_->cfd();
+  } else {
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+    cfd = cfh->cfd();
+  }
+
+  mutex_.Lock();
+  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
+  mutex_.Unlock();
+  ReadOptions roptions;
+  return NewInternalIterator(roptions, cfd, super_version);
+}
+
+int64_t DBImpl::TEST_MaxNextLevelOverlappingBytes(
+    ColumnFamilyHandle* column_family) {
+  ColumnFamilyData* cfd;
+  if (column_family == nullptr) {
+    cfd = default_cf_handle_->cfd();
+  } else {
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+    cfd = cfh->cfd();
+  }
+  MutexLock l(&mutex_);
+  return cfd->current()->MaxNextLevelOverlappingBytes();
+}
+
+void DBImpl::TEST_GetFilesMetaData(
+    ColumnFamilyHandle* column_family,
+    std::vector<std::vector<FileMetaData>>* metadata) {
+  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+  auto cfd = cfh->cfd();
+  MutexLock l(&mutex_);
+  metadata->resize(NumberLevels());
+  for (int level = 0; level < NumberLevels(); level++) {
+    const std::vector<FileMetaData*>& files = cfd->current()->files_[level];
+
+    (*metadata)[level].clear();
+    for (const auto& f : files) {
+      (*metadata)[level].push_back(*f);
+    }
+  }
+}
+
+uint64_t DBImpl::TEST_Current_Manifest_FileNo() {
+  return versions_->ManifestFileNumber();
+}
+
+Status DBImpl::TEST_CompactRange(int level, const Slice* begin,
+                                 const Slice* end,
+                                 ColumnFamilyHandle* column_family) {
+  ColumnFamilyData* cfd;
+  if (column_family == nullptr) {
+    cfd = default_cf_handle_->cfd();
+  } else {
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+    cfd = cfh->cfd();
+  }
+  int output_level =
+      (cfd->options()->compaction_style == kCompactionStyleUniversal)
+          ? level
+          : level + 1;
+  return RunManualCompaction(cfd, level, output_level, begin, end);
+}
+
+Status DBImpl::TEST_FlushMemTable(bool wait) {
+  FlushOptions fo;
+  fo.wait = wait;
+  return FlushMemTable(default_cf_handle_->cfd(), fo);
+}
+
+Status DBImpl::TEST_WaitForFlushMemTable(ColumnFamilyHandle* column_family) {
+  ColumnFamilyData* cfd;
+  if (column_family == nullptr) {
+    cfd = default_cf_handle_->cfd();
+  } else {
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+    cfd = cfh->cfd();
+  }
+  return WaitForFlushMemTable(cfd);
+}
+
+Status DBImpl::TEST_WaitForCompact() {
+  // Wait until the compaction completes
+
+  // TODO: a bug here. This function actually does not necessarily
+  // wait for compact. It actually waits for scheduled compaction
+  // OR flush to finish.
+
+  MutexLock l(&mutex_);
+  while ((bg_compaction_scheduled_ || bg_flush_scheduled_) && bg_error_.ok()) {
+    bg_cv_.Wait();
+  }
+  return bg_error_;
+}
+
+Status DBImpl::TEST_ReadFirstRecord(const WalFileType type,
+                                    const uint64_t number,
+                                    SequenceNumber* sequence) {
+  return ReadFirstRecord(type, number, sequence);
+}
+
+Status DBImpl::TEST_ReadFirstLine(const std::string& fname,
+                                  SequenceNumber* sequence) {
+  return ReadFirstLine(fname, sequence);
+}
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

db/db_impl_readonly.cc

@@ -42,8 +42,8 @@
 
 namespace rocksdb {
 
-DBImplReadOnly::DBImplReadOnly(const Options& options,
-    const std::string& dbname)
+DBImplReadOnly::DBImplReadOnly(const DBOptions& options,
+                               const std::string& dbname)
     : DBImpl(options, dbname) {
   Log(options_.info_log, "Opening the db in read only mode");
 }
@@ -53,50 +53,102 @@ DBImplReadOnly::~DBImplReadOnly() {
 
 // Implementations of the DB interface
 Status DBImplReadOnly::Get(const ReadOptions& options,
-                   const Slice& key,
-                   std::string* value) {
+                           ColumnFamilyHandle* column_family, const Slice& key,
+                           std::string* value) {
   Status s;
   SequenceNumber snapshot = versions_->LastSequence();
-  SuperVersion* super_version = GetSuperVersion();
+  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+  auto cfd = cfh->cfd();
+  SuperVersion* super_version = cfd->GetSuperVersion();
   MergeContext merge_context;
   LookupKey lkey(key, snapshot);
-  if (super_version->mem->Get(lkey, value, &s, merge_context, options_)) {
+  if (super_version->mem->Get(lkey, value, &s, merge_context,
+                              *cfd->options())) {
   } else {
     Version::GetStats stats;
     super_version->current->Get(options, lkey, value, &s, &merge_context,
-                                &stats, options_);
+                                &stats);
   }
   return s;
 }
 
-Iterator* DBImplReadOnly::NewIterator(const ReadOptions& options) {
-  SequenceNumber latest_snapshot;
-  Iterator* internal_iter = NewInternalIterator(options, &latest_snapshot);
+Iterator* DBImplReadOnly::NewIterator(const ReadOptions& options,
+                                      ColumnFamilyHandle* column_family) {
+  auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+  auto cfd = cfh->cfd();
+  SuperVersion* super_version = cfd->GetSuperVersion()->Ref();
+  SequenceNumber latest_snapshot = versions_->LastSequence();
+  Iterator* internal_iter = NewInternalIterator(options, cfd, super_version);
   return NewDBIterator(
-    &dbname_, env_, options_,  user_comparator(),internal_iter,
+      env_, *cfd->options(), cfd->user_comparator(), internal_iter,
       (options.snapshot != nullptr
-      ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
-      : latest_snapshot));
+           ? reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_
+           : latest_snapshot));
 }
 
-
 Status DB::OpenForReadOnly(const Options& options, const std::string& dbname,
-                DB** dbptr, bool error_if_log_file_exist) {
+                           DB** dbptr, bool error_if_log_file_exist) {
   *dbptr = nullptr;
 
-  DBImplReadOnly* impl = new DBImplReadOnly(options, dbname);
-  impl->mutex_.Lock();
-  Status s = impl->Recover(true /* read only */, error_if_log_file_exist);
+  DBOptions db_options(options);
+  ColumnFamilyOptions cf_options(options);
+  std::vector<ColumnFamilyDescriptor> column_families;
+  column_families.push_back(
+      ColumnFamilyDescriptor(kDefaultColumnFamilyName, cf_options));
+  std::vector<ColumnFamilyHandle*> handles;
+
+  Status s =
+      DB::OpenForReadOnly(db_options, dbname, column_families, &handles, dbptr);
   if (s.ok()) {
-    delete impl->InstallSuperVersion(new DBImpl::SuperVersion());
+    assert(handles.size() == 1);
+    // i can delete the handle since DBImpl is always holding a
+    // reference to default column family
+    delete handles[0];
+  }
+  return s;
+}
+
+Status DB::OpenForReadOnly(
+    const DBOptions& db_options, const std::string& dbname,
+    const std::vector<ColumnFamilyDescriptor>& column_families,
+    std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+    bool error_if_log_file_exist) {
+  *dbptr = nullptr;
+  handles->clear();
+
+  DBImplReadOnly* impl = new DBImplReadOnly(db_options, dbname);
+  impl->mutex_.Lock();
+  Status s = impl->Recover(column_families, true /* read only */,
+                           error_if_log_file_exist);
+  if (s.ok()) {
+    // set column family handles
+    for (auto cf : column_families) {
+      auto cfd =
+          impl->versions_->GetColumnFamilySet()->GetColumnFamily(cf.name);
+      if (cfd == nullptr) {
+        s = Status::InvalidArgument("Column family not found: ", cf.name);
+        break;
+      }
+      handles->push_back(new ColumnFamilyHandleImpl(cfd, impl, &impl->mutex_));
+    }
+  }
+  if (s.ok()) {
+    for (auto cfd : *impl->versions_->GetColumnFamilySet()) {
+      delete cfd->InstallSuperVersion(new SuperVersion(), &impl->mutex_);
+    }
   }
   impl->mutex_.Unlock();
   if (s.ok()) {
     *dbptr = impl;
   } else {
+    for (auto h : *handles) {
+      delete h;
+    }
+    handles->clear();
     delete impl;
   }
   return s;
 }
 
+
 }   // namespace rocksdb

db/db_impl_readonly.h

@@ -12,6 +12,8 @@
 
 #include <deque>
 #include <set>
+#include <vector>
+#include <string>
 #include "db/dbformat.h"
 #include "db/log_writer.h"
 #include "db/snapshot.h"
@@ -23,57 +25,79 @@
 namespace rocksdb {
 
 class DBImplReadOnly : public DBImpl {
-public:
-  DBImplReadOnly(const Options& options, const std::string& dbname);
- virtual ~DBImplReadOnly();
+ public:
+  DBImplReadOnly(const DBOptions& options, const std::string& dbname);
+  virtual ~DBImplReadOnly();
 
- // Implementations of the DB interface
- virtual Status Get(const ReadOptions& options,
-                    const Slice& key,
-                    std::string* value);
+  // Implementations of the DB interface
+  using DB::Get;
+  virtual Status Get(const ReadOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     std::string* value);
 
- // TODO: Implement ReadOnly MultiGet?
+  // TODO: Implement ReadOnly MultiGet?
 
- virtual Iterator* NewIterator(const ReadOptions&);
+  using DBImpl::NewIterator;
+  virtual Iterator* NewIterator(const ReadOptions&,
+                                ColumnFamilyHandle* column_family);
 
- virtual Status Put(const WriteOptions&, const Slice& key, const Slice& value) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status Merge(const WriteOptions&, const Slice& key,
-                      const Slice& value) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status Delete(const WriteOptions&, const Slice& key) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status Write(const WriteOptions& options, WriteBatch* updates) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status CompactRange(const Slice* begin, const Slice* end,
-                             bool reduce_level = false, int target_level = -1) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status DisableFileDeletions() {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status EnableFileDeletions(bool force) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status GetLiveFiles(std::vector<std::string>&,
-                             uint64_t* manifest_file_size,
-                             bool flush_memtable = true) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
- virtual Status Flush(const FlushOptions& options) {
-   return Status::NotSupported("Not supported operation in read only mode.");
- }
+  virtual Status NewIterators(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_family,
+      std::vector<Iterator*>* iterators) {
+   // TODO
+    return Status::NotSupported("Not supported yet.");
+  }
 
-private:
- friend class DB;
+  using DBImpl::Put;
+  virtual Status Put(const WriteOptions& options,
+                     ColumnFamilyHandle* column_family, const Slice& key,
+                     const Slice& value) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  using DBImpl::Merge;
+  virtual Status Merge(const WriteOptions& options,
+                       ColumnFamilyHandle* column_family, const Slice& key,
+                       const Slice& value) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  using DBImpl::Delete;
+  virtual Status Delete(const WriteOptions& options,
+                        ColumnFamilyHandle* column_family, const Slice& key) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  virtual Status Write(const WriteOptions& options, WriteBatch* updates) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  using DBImpl::CompactRange;
+  virtual Status CompactRange(ColumnFamilyHandle* column_family,
+                              const Slice* begin, const Slice* end,
+                              bool reduce_level = false,
+                              int target_level = -1) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  virtual Status DisableFileDeletions() {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  virtual Status EnableFileDeletions(bool force) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  virtual Status GetLiveFiles(std::vector<std::string>&,
+                              uint64_t* manifest_file_size,
+                              bool flush_memtable = true) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
+  using DBImpl::Flush;
+  virtual Status Flush(const FlushOptions& options,
+                       ColumnFamilyHandle* column_family) {
+    return Status::NotSupported("Not supported operation in read only mode.");
+  }
 
- // No copying allowed
- DBImplReadOnly(const DBImplReadOnly&);
- void operator=(const DBImplReadOnly&);
+ private:
+  friend class DB;
+
+  // No copying allowed
+  DBImplReadOnly(const DBImplReadOnly&);
+  void operator=(const DBImplReadOnly&);
 };
-
 }

db/db_iter.cc

@@ -39,71 +39,6 @@ static void DumpInternalIter(Iterator* iter) {
 
 namespace {
 
-class IterLookupKey {
- public:
-  IterLookupKey() : key_(space_), buf_size_(sizeof(space_)), key_size_(0) {}
-
-  ~IterLookupKey() { Clear(); }
-
-  Slice GetKey() const {
-    if (key_ != nullptr) {
-      return Slice(key_, key_size_);
-    } else {
-      return Slice();
-    }
-  }
-
-  bool Valid() const { return key_ != nullptr; }
-
-  void Clear() {
-    if (key_ != nullptr && key_ != space_) {
-      delete[] key_;
-    }
-    key_ = space_;
-    buf_size_ = sizeof(buf_size_);
-  }
-
-  // Enlarge the buffer size if needed based on key_size.
-  // By default, static allocated buffer is used. Once there is a key
-  // larger than the static allocated buffer, another buffer is dynamically
-  // allocated, until a larger key buffer is requested. In that case, we
-  // reallocate buffer and delete the old one.
-  void EnlargeBufferIfNeeded(size_t key_size) {
-    // If size is smaller than buffer size, continue using current buffer,
-    // or the static allocated one, as default
-    if (key_size > buf_size_) {
-      // Need to enlarge the buffer.
-      Clear();
-      key_ = new char[key_size];
-      buf_size_ = key_size;
-    }
-    key_size_ = key_size;
-  }
-
-  void SetUserKey(const Slice& user_key) {
-    size_t size = user_key.size();
-    EnlargeBufferIfNeeded(size);
-    memcpy(key_, user_key.data(), size);
-  }
-
-  void SetInternalKey(const Slice& user_key, SequenceNumber s) {
-    size_t usize = user_key.size();
-    EnlargeBufferIfNeeded(usize + sizeof(uint64_t));
-    memcpy(key_, user_key.data(), usize);
-    EncodeFixed64(key_ + usize, PackSequenceAndType(s, kValueTypeForSeek));
-  }
-
- private:
-  char* key_;
-  size_t buf_size_;
-  size_t key_size_;
-  char space_[32];  // Avoid allocation for short keys
-
-  // No copying allowed
-  IterLookupKey(const IterLookupKey&) = delete;
-  void operator=(const LookupKey&) = delete;
-};
-
 // Memtables and sstables that make the DB representation contain
 // (userkey,seq,type) => uservalue entries.  DBIter
 // combines multiple entries for the same userkey found in the DB
@@ -122,10 +57,9 @@ class DBIter: public Iterator {
     kReverse
   };
 
-  DBIter(const std::string* dbname, Env* env, const Options& options,
+  DBIter(Env* env, const Options& options,
          const Comparator* cmp, Iterator* iter, SequenceNumber s)
-      : dbname_(dbname),
-        env_(env),
+      : env_(env),
         logger_(options.info_log.get()),
         user_comparator_(cmp),
         user_merge_operator_(options.merge_operator.get()),
@@ -182,7 +116,6 @@ class DBIter: public Iterator {
     }
   }
 
-  const std::string* const dbname_;
   Env* const env_;
   Logger* logger_;
   const Comparator* const user_comparator_;
@@ -191,7 +124,7 @@ class DBIter: public Iterator {
   SequenceNumber const sequence_;
 
   Status status_;
-  IterLookupKey saved_key_;   // == current key when direction_==kReverse
+  IterKey saved_key_;   // == current key when direction_==kReverse
   std::string saved_value_;   // == current raw value when direction_==kReverse
   std::string skip_key_;
   Direction direction_;
@@ -254,10 +187,9 @@ void DBIter::Next() {
 // NOTE: In between, saved_key_ can point to a user key that has
 //       a delete marker
 inline void DBIter::FindNextUserEntry(bool skipping) {
-  StopWatchNano timer(env_, false);
-  StartPerfTimer(&timer);
+  PERF_TIMER_AUTO(find_next_user_entry_time);
   FindNextUserEntryInternal(skipping);
-  BumpPerfTime(&perf_context.find_next_user_entry_time, &timer);
+  PERF_TIMER_STOP(find_next_user_entry_time);
 }
 
 // Actual implementation of DBIter::FindNextUserEntry()
@@ -273,7 +205,7 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {
       if (skipping &&
           user_comparator_->Compare(ikey.user_key, saved_key_.GetKey()) <= 0) {
         num_skipped++; // skip this entry
-        BumpPerfCount(&perf_context.internal_key_skipped_count);
+        PERF_COUNTER_ADD(internal_key_skipped_count, 1);
       } else {
         skipping = false;
         switch (ikey.type) {
@@ -283,7 +215,7 @@ void DBIter::FindNextUserEntryInternal(bool skipping) {
             saved_key_.SetUserKey(ikey.user_key);
             skipping = true;
             num_skipped = 0;
-            BumpPerfCount(&perf_context.internal_delete_skipped_count);
+            PERF_COUNTER_ADD(internal_delete_skipped_count, 1);
             break;
           case kTypeValue:
             valid_ = true;
@@ -488,10 +420,9 @@ void DBIter::Seek(const Slice& target) {
   saved_key_.Clear();
   // now savved_key is used to store internal key.
   saved_key_.SetInternalKey(target, sequence_);
-  StopWatchNano internal_seek_timer(env_, false);
-  StartPerfTimer(&internal_seek_timer);
+  PERF_TIMER_AUTO(seek_internal_seek_time);
   iter_->Seek(saved_key_.GetKey());
-  BumpPerfTime(&perf_context.seek_internal_seek_time, &internal_seek_timer);
+  PERF_TIMER_STOP(seek_internal_seek_time);
   if (iter_->Valid()) {
     direction_ = kForward;
     ClearSavedValue();
@@ -504,10 +435,9 @@ void DBIter::Seek(const Slice& target) {
 void DBIter::SeekToFirst() {
   direction_ = kForward;
   ClearSavedValue();
-  StopWatchNano internal_seek_timer(env_, false);
-  StartPerfTimer(&internal_seek_timer);
+  PERF_TIMER_AUTO(seek_internal_seek_time);
   iter_->SeekToFirst();
-  BumpPerfTime(&perf_context.seek_internal_seek_time, &internal_seek_timer);
+  PERF_TIMER_STOP(seek_internal_seek_time);
   if (iter_->Valid()) {
     FindNextUserEntry(false /* not skipping */);
   } else {
@@ -526,23 +456,21 @@ void DBIter::SeekToLast() {
 
   direction_ = kReverse;
   ClearSavedValue();
-  StopWatchNano internal_seek_timer(env_, false);
-  StartPerfTimer(&internal_seek_timer);
+  PERF_TIMER_AUTO(seek_internal_seek_time);
   iter_->SeekToLast();
-  BumpPerfTime(&perf_context.seek_internal_seek_time, &internal_seek_timer);
+  PERF_TIMER_STOP(seek_internal_seek_time);
   FindPrevUserEntry();
 }
 
 }  // anonymous namespace
 
 Iterator* NewDBIterator(
-    const std::string* dbname,
     Env* env,
     const Options& options,
     const Comparator *user_key_comparator,
     Iterator* internal_iter,
     const SequenceNumber& sequence) {
-  return new DBIter(dbname, env, options, user_key_comparator,
+  return new DBIter(env, options, user_key_comparator,
                     internal_iter, sequence);
 }
 

db/db_iter.h

@@ -18,7 +18,6 @@ namespace rocksdb {
 // "*internal_iter") that were live at the specified "sequence" number
 // into appropriate user keys.
 extern Iterator* NewDBIterator(
-    const std::string* dbname,
     Env* env,
     const Options& options,
     const Comparator *user_key_comparator,

db/db_stats_logger.cc

@@ -20,7 +20,8 @@
 namespace rocksdb {
 
 void DBImpl::MaybeScheduleLogDBDeployStats() {
-
+// we did say maybe
+#ifndef ROCKSDB_LITE
   // There is a lock in the actual logger.
   if (!logger_ || options_.db_stats_log_interval < 0
       || host_name_.empty()) {
@@ -65,7 +66,7 @@ void DBImpl::LogDBDeployStats() {
 
   uint64_t file_total_size = 0;
   uint32_t file_total_num = 0;
-  Version* current = versions_->current();
+  Version* current = default_cf_handle_->cfd()->current();
   for (int i = 0; i < current->NumberLevels(); i++) {
     file_total_num += current->NumLevelFiles(i);
     file_total_size += current->NumLevelBytes(i);
@@ -89,6 +90,6 @@ void DBImpl::LogDBDeployStats() {
   bg_logstats_scheduled_ = false;
   bg_cv_.SignalAll();
   mutex_.Unlock();
+#endif
 }
-
 }

db/dbformat.cc

@@ -59,7 +59,7 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
   //    decreasing sequence number
   //    decreasing type (though sequence# should be enough to disambiguate)
   int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
-  BumpPerfCount(&perf_context.user_key_comparison_count);
+  PERF_COUNTER_ADD(user_key_comparison_count, 1);
   if (r == 0) {
     const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
     const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);
@@ -79,7 +79,7 @@ int InternalKeyComparator::Compare(const ParsedInternalKey& a,
   //    decreasing sequence number
   //    decreasing type (though sequence# should be enough to disambiguate)
   int r = user_comparator_->Compare(a.user_key, b.user_key);
-  BumpPerfCount(&perf_context.user_key_comparison_count);
+  PERF_COUNTER_ADD(user_key_comparison_count, 1);
   if (r == 0) {
     if (a.sequence > b.sequence) {
       r = -1;

db/dbformat.h

@@ -13,6 +13,7 @@
 #include "rocksdb/db.h"
 #include "rocksdb/filter_policy.h"
 #include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
 #include "rocksdb/table.h"
 #include "rocksdb/types.h"
 #include "util/coding.h"
@@ -31,7 +32,12 @@ enum ValueType : unsigned char {
   kTypeDeletion = 0x0,
   kTypeValue = 0x1,
   kTypeMerge = 0x2,
+  // Following types are used only in write ahead logs. They are not used in
+  // memtables or sst files:
   kTypeLogData = 0x3,
+  kTypeColumnFamilyDeletion = 0x4,
+  kTypeColumnFamilyValue = 0x5,
+  kTypeColumnFamilyMerge = 0x6,
   kMaxValue = 0x7F
 };
 
@@ -235,4 +241,100 @@ inline LookupKey::~LookupKey() {
   if (start_ != space_) delete[] start_;
 }
 
+class IterKey {
+ public:
+  IterKey() : key_(space_), buf_size_(sizeof(space_)), key_size_(0) {}
+
+  ~IterKey() { ResetBuffer(); }
+
+  Slice GetKey() const { return Slice(key_, key_size_); }
+
+  void Clear() { key_size_ = 0; }
+
+  void SetUserKey(const Slice& user_key) {
+    size_t size = user_key.size();
+    EnlargeBufferIfNeeded(size);
+    memcpy(key_, user_key.data(), size);
+    key_size_ = size;
+  }
+
+  void SetInternalKey(const Slice& user_key, SequenceNumber s,
+                      ValueType value_type = kValueTypeForSeek) {
+    size_t usize = user_key.size();
+    EnlargeBufferIfNeeded(usize + sizeof(uint64_t));
+    memcpy(key_, user_key.data(), usize);
+    EncodeFixed64(key_ + usize, PackSequenceAndType(s, value_type));
+    key_size_ = usize + sizeof(uint64_t);
+  }
+
+  void SetInternalKey(const ParsedInternalKey& parsed_key) {
+    SetInternalKey(parsed_key.user_key, parsed_key.sequence, parsed_key.type);
+  }
+
+ private:
+  char* key_;
+  size_t buf_size_;
+  size_t key_size_;
+  char space_[32];  // Avoid allocation for short keys
+
+  void ResetBuffer() {
+    if (key_ != nullptr && key_ != space_) {
+      delete[] key_;
+    }
+    key_ = space_;
+    buf_size_ = sizeof(space_);
+    key_size_ = 0;
+  }
+
+  // Enlarge the buffer size if needed based on key_size.
+  // By default, static allocated buffer is used. Once there is a key
+  // larger than the static allocated buffer, another buffer is dynamically
+  // allocated, until a larger key buffer is requested. In that case, we
+  // reallocate buffer and delete the old one.
+  void EnlargeBufferIfNeeded(size_t key_size) {
+    // If size is smaller than buffer size, continue using current buffer,
+    // or the static allocated one, as default
+    if (key_size > buf_size_) {
+      // Need to enlarge the buffer.
+      ResetBuffer();
+      key_ = new char[key_size];
+      buf_size_ = key_size;
+    }
+  }
+
+  // No copying allowed
+  IterKey(const IterKey&) = delete;
+  void operator=(const IterKey&) = delete;
+};
+
+class InternalKeySliceTransform : public SliceTransform {
+ public:
+  explicit InternalKeySliceTransform(const SliceTransform* transform)
+      : transform_(transform) {}
+
+  virtual const char* Name() const { return transform_->Name(); }
+
+  virtual Slice Transform(const Slice& src) const {
+    auto user_key = ExtractUserKey(src);
+    return transform_->Transform(user_key);
+  }
+
+  virtual bool InDomain(const Slice& src) const {
+    auto user_key = ExtractUserKey(src);
+    return transform_->InDomain(user_key);
+  }
+
+  virtual bool InRange(const Slice& dst) const {
+    auto user_key = ExtractUserKey(dst);
+    return transform_->InRange(user_key);
+  }
+
+  const SliceTransform* user_prefix_extractor() const { return transform_; }
+
+ private:
+  // Like comparator, InternalKeySliceTransform will not take care of the
+  // deletion of transform_
+  const SliceTransform* const transform_;
+};
+
 }  // namespace rocksdb

db/file_indexer.cc

@@ -0,0 +1,202 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "db/file_indexer.h"
+#include <algorithm>
+#include "rocksdb/comparator.h"
+#include "db/version_edit.h"
+
+namespace rocksdb {
+
+FileIndexer::FileIndexer(const uint32_t num_levels,
+                         const Comparator* ucmp)
+  : num_levels_(num_levels),
+    ucmp_(ucmp),
+    next_level_index_(num_levels),
+    level_rb_(num_levels, -1) {
+}
+
+
+uint32_t FileIndexer::NumLevelIndex() {
+  return next_level_index_.size();
+}
+
+uint32_t FileIndexer::LevelIndexSize(uint32_t level) {
+  return next_level_index_[level].size();
+}
+
+void FileIndexer::GetNextLevelIndex(
+    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
+    const int cmp_largest, int32_t* left_bound, int32_t* right_bound) {
+  assert(level > 0);
+
+  // Last level, no hint
+  if (level == num_levels_ - 1) {
+    *left_bound = 0;
+    *right_bound = -1;
+    return;
+  }
+
+  assert(level < num_levels_ - 1);
+  assert(static_cast<int32_t>(file_index) <= level_rb_[level]);
+
+  const auto& index = next_level_index_[level][file_index];
+
+  if (cmp_smallest < 0) {
+    *left_bound = (level > 0 && file_index > 0) ?
+      next_level_index_[level][file_index - 1].largest_lb : 0;
+    *right_bound = index.smallest_rb;
+  } else if (cmp_smallest == 0) {
+    *left_bound = index.smallest_lb;
+    *right_bound = index.smallest_rb;
+  } else if (cmp_smallest > 0 && cmp_largest < 0) {
+    *left_bound = index.smallest_lb;
+    *right_bound = index.largest_rb;
+  } else if (cmp_largest == 0) {
+    *left_bound = index.largest_lb;
+    *right_bound = index.largest_rb;
+  } else if (cmp_largest > 0) {
+    *left_bound = index.largest_lb;
+    *right_bound = level_rb_[level + 1];
+  } else {
+    assert(false);
+  }
+
+  assert(*left_bound >= 0);
+  assert(*left_bound <= *right_bound + 1);
+  assert(*right_bound <= level_rb_[level + 1]);
+}
+
+void FileIndexer::ClearIndex() {
+  for (uint32_t level = 1; level < num_levels_; ++level) {
+    next_level_index_[level].clear();
+  }
+}
+
+void FileIndexer::UpdateIndex(std::vector<FileMetaData*>* const files) {
+  if (files == nullptr) {
+    return;
+  }
+
+  // L1 - Ln-1
+  for (uint32_t level = 1; level < num_levels_ - 1; ++level) {
+    const auto& upper_files = files[level];
+    const int32_t upper_size = upper_files.size();
+    const auto& lower_files = files[level + 1];
+    level_rb_[level] = upper_files.size() - 1;
+    if (upper_size == 0) {
+      continue;
+    }
+    auto& index = next_level_index_[level];
+    index.resize(upper_size);
+
+    CalculateLB(upper_files, lower_files, &index,
+        [this](const FileMetaData* a, const FileMetaData* b) -> int {
+          return ucmp_->Compare(a->smallest.user_key(), b->largest.user_key());
+        },
+        [](IndexUnit* index, int32_t f_idx) {
+          index->smallest_lb = f_idx;
+        });
+    CalculateLB(upper_files, lower_files, &index,
+        [this](const FileMetaData* a, const FileMetaData* b) -> int {
+          return ucmp_->Compare(a->largest.user_key(), b->largest.user_key());
+        },
+        [](IndexUnit* index, int32_t f_idx) {
+          index->largest_lb = f_idx;
+        });
+    CalculateRB(upper_files, lower_files, &index,
+        [this](const FileMetaData* a, const FileMetaData* b) -> int {
+          return ucmp_->Compare(a->smallest.user_key(), b->smallest.user_key());
+        },
+        [](IndexUnit* index, int32_t f_idx) {
+          index->smallest_rb = f_idx;
+        });
+    CalculateRB(upper_files, lower_files, &index,
+        [this](const FileMetaData* a, const FileMetaData* b) -> int {
+          return ucmp_->Compare(a->largest.user_key(), b->smallest.user_key());
+        },
+        [](IndexUnit* index, int32_t f_idx) {
+          index->largest_rb = f_idx;
+        });
+  }
+  level_rb_[num_levels_ - 1] = files[num_levels_ - 1].size() - 1;
+}
+
+void FileIndexer::CalculateLB(const std::vector<FileMetaData*>& upper_files,
+    const std::vector<FileMetaData*>& lower_files,
+    std::vector<IndexUnit>* index,
+    std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+    std::function<void(IndexUnit*, int32_t)> set_index) {
+  const int32_t upper_size = upper_files.size();
+  const int32_t lower_size = lower_files.size();
+  int32_t upper_idx = 0;
+  int32_t lower_idx = 0;
+  while (upper_idx < upper_size && lower_idx < lower_size) {
+    int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]);
+
+    if (cmp == 0) {
+      set_index(&(*index)[upper_idx], lower_idx);
+      ++upper_idx;
+      ++lower_idx;
+    } else if (cmp > 0) {
+      // Lower level's file (largest) is smaller, a key won't hit in that
+      // file. Move to next lower file
+      ++lower_idx;
+    } else {
+      // Lower level's file becomes larger, update the index, and
+      // move to the next upper file
+      set_index(&(*index)[upper_idx], lower_idx);
+      ++upper_idx;
+    }
+  }
+
+  while (upper_idx < upper_size) {
+    // Lower files are exhausted, that means the remaining upper files are
+    // greater than any lower files. Set the index to be the lower level size.
+    set_index(&(*index)[upper_idx], lower_size);
+    ++upper_idx;
+  }
+}
+
+void FileIndexer::CalculateRB(const std::vector<FileMetaData*>& upper_files,
+    const std::vector<FileMetaData*>& lower_files,
+    std::vector<IndexUnit>* index,
+    std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+    std::function<void(IndexUnit*, int32_t)> set_index) {
+  const int32_t upper_size = upper_files.size();
+  const int32_t lower_size = lower_files.size();
+  int32_t upper_idx = upper_size - 1;
+  int32_t lower_idx = lower_size - 1;
+  while (upper_idx >= 0 && lower_idx >= 0) {
+    int cmp = cmp_op(upper_files[upper_idx], lower_files[lower_idx]);
+
+    if (cmp == 0) {
+      set_index(&(*index)[upper_idx], lower_idx);
+      --upper_idx;
+      --lower_idx;
+    } else if (cmp < 0) {
+      // Lower level's file (smallest) is larger, a key won't hit in that
+      // file. Move to next lower file.
+      --lower_idx;
+    } else {
+      // Lower level's file becomes smaller, update the index, and move to
+      // the next the upper file
+      set_index(&(*index)[upper_idx], lower_idx);
+      --upper_idx;
+    }
+  }
+  while (upper_idx >= 0) {
+    // Lower files are exhausted, that means the remaining upper files are
+    // smaller than any lower files. Set it to -1.
+    set_index(&(*index)[upper_idx], -1);
+    --upper_idx;
+  }
+}
+
+}  // namespace rocksdb

db/file_indexer.h

@@ -0,0 +1,129 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#pragma once
+#include <cstdint>
+#include <functional>
+#include <limits>
+#include <vector>
+
+namespace rocksdb {
+
+class Comparator;
+struct FileMetaData;
+
+// The file tree structure in Version is prebuilt and the range of each file
+// is known. On Version::Get(), it uses binary search to find a potential file
+// and then check if a target key can be found in the file by comparing the key
+// to each file's smallest and largest key. The results of these comparisions
+// can be reused beyond checking if a key falls into a file's range.
+// With some pre-calculated knowledge, each key comparision that has been done
+// can serve as a hint to narrow down further searches: if a key compared to
+// be smaller than a file's smallest or largest, that comparison can be used
+// to find out the right bound of next binary search. Similarly, if a key
+// compared to be larger than a file's smallest or largest, it can be utilized
+// to find out the left bound of next binary search.
+// With these hints: it can greatly reduce the range of binary search,
+// especially for bottom levels, given that one file most likely overlaps with
+// only N files from level below (where N is max_bytes_for_level_multiplier).
+// So on level L, we will only look at ~N files instead of N^L files on the
+// naive approach.
+class FileIndexer {
+ public:
+  FileIndexer(const uint32_t num_levels, const Comparator* ucmp);
+
+  uint32_t NumLevelIndex();
+
+  uint32_t LevelIndexSize(uint32_t level);
+
+  // Return a file index range in the next level to search for a key based on
+  // smallest and largest key comparision for the current file specified by
+  // level and file_index. When *left_index < *right_index, both index should
+  // be valid and fit in the vector size.
+  void GetNextLevelIndex(
+    const uint32_t level, const uint32_t file_index, const int cmp_smallest,
+    const int cmp_largest, int32_t* left_bound, int32_t* right_bound);
+
+  void ClearIndex();
+
+  void UpdateIndex(std::vector<FileMetaData*>* const files);
+
+  enum {
+    kLevelMaxIndex = std::numeric_limits<int32_t>::max()
+  };
+
+ private:
+  const uint32_t num_levels_;
+  const Comparator* ucmp_;
+
+  struct IndexUnit {
+    IndexUnit()
+      : smallest_lb(0), largest_lb(0), smallest_rb(-1), largest_rb(-1) {}
+    // During file search, a key is compared against smallest and largest
+    // from a FileMetaData. It can have 3 possible outcomes:
+    // (1) key is smaller than smallest, implying it is also smaller than
+    //     larger. Precalculated index based on "smallest < smallest" can
+    //     be used to provide right bound.
+    // (2) key is in between smallest and largest.
+    //     Precalculated index based on "smallest > greatest" can be used to
+    //     provide left bound.
+    //     Precalculated index based on "largest < smallest" can be used to
+    //     provide right bound.
+    // (3) key is larger than largest, implying it is also larger than smallest.
+    //     Precalculated index based on "largest > largest" can be used to
+    //     provide left bound.
+    //
+    // As a result, we will need to do:
+    // Compare smallest (<=) and largest keys from upper level file with
+    // smallest key from lower level to get a right bound.
+    // Compare smallest (>=) and largest keys from upper level file with
+    // largest key from lower level to get a left bound.
+    //
+    // Example:
+    //    level 1:              [50 - 60]
+    //    level 2:        [1 - 40], [45 - 55], [58 - 80]
+    // A key 35, compared to be less than 50, 3rd file on level 2 can be
+    // skipped according to rule (1). LB = 0, RB = 1.
+    // A key 53, sits in the middle 50 and 60. 1st file on level 2 can be
+    // skipped according to rule (2)-a, but the 3rd file cannot be skipped
+    // because 60 is greater than 58. LB = 1, RB = 2.
+    // A key 70, compared to be larger than 60. 1st and 2nd file can be skipped
+    // according to rule (3). LB = 2, RB = 2.
+    //
+    // Point to a left most file in a lower level that may contain a key,
+    // which compares greater than smallest of a FileMetaData (upper level)
+    int32_t smallest_lb;
+    // Point to a left most file in a lower level that may contain a key,
+    // which compares greater than largest of a FileMetaData (upper level)
+    int32_t largest_lb;
+    // Point to a right most file in a lower level that may contain a key,
+    // which compares smaller than smallest of a FileMetaData (upper level)
+    int32_t smallest_rb;
+    // Point to a right most file in a lower level that may contain a key,
+    // which compares smaller than largest of a FileMetaData (upper level)
+    int32_t largest_rb;
+  };
+
+  void CalculateLB(const std::vector<FileMetaData*>& upper_files,
+    const std::vector<FileMetaData*>& lower_files,
+    std::vector<IndexUnit>* index,
+    std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+    std::function<void(IndexUnit*, int32_t)> set_index);
+
+  void CalculateRB(const std::vector<FileMetaData*>& upper_files,
+    const std::vector<FileMetaData*>& lower_files,
+    std::vector<IndexUnit>* index,
+    std::function<int(const FileMetaData*, const FileMetaData*)> cmp_op,
+    std::function<void(IndexUnit*, int32_t)> set_index);
+
+  std::vector<std::vector<IndexUnit>> next_level_index_;
+  std::vector<int32_t> level_rb_;
+};
+
+}  // namespace rocksdb

db/file_indexer_test.cc

@@ -0,0 +1,330 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+//
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include <string>
+#include "db/file_indexer.h"
+#include "db/dbformat.h"
+#include "db/version_edit.h"
+#include "rocksdb/comparator.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+
+namespace rocksdb {
+
+class IntComparator : public Comparator {
+ public:
+  int Compare(const Slice& a, const Slice& b) const {
+    assert(a.size() == 8);
+    assert(b.size() == 8);
+    return *reinterpret_cast<const int64_t*>(a.data()) -
+      *reinterpret_cast<const int64_t*>(b.data());
+  }
+
+  const char* Name() const {
+    return "IntComparator";
+  }
+
+  void FindShortestSeparator(std::string* start, const Slice& limit) const {}
+
+  void FindShortSuccessor(std::string* key) const {}
+};
+
+
+struct FileIndexerTest {
+ public:
+  FileIndexerTest() :
+    kNumLevels(4), indexer(kNumLevels, &ucmp),
+    files(new std::vector<FileMetaData*>[kNumLevels]) {
+  }
+
+  ~FileIndexerTest() {
+    Reset();
+    delete[] files;
+  }
+
+  void AddFile(int level, int64_t smallest, int64_t largest) {
+    auto* f = new FileMetaData();
+    f->smallest = IntKey(smallest);
+    f->largest = IntKey(largest);
+    files[level].push_back(f);
+  }
+
+  InternalKey IntKey(int64_t v) {
+    return InternalKey(Slice(reinterpret_cast<char*>(&v), 8), 0, kTypeValue);
+  }
+
+  void Reset() {
+    for (uint32_t i = 0; i < kNumLevels; ++i) {
+      for (auto* f : files[i]) {
+        delete f;
+      }
+      files[i].clear();
+    }
+    indexer.ClearIndex();
+  }
+
+  void GetNextLevelIndex(const uint32_t level, const uint32_t file_index,
+      const int cmp_smallest, const int cmp_largest, int32_t* left_index,
+      int32_t* right_index) {
+    *left_index = 100;
+    *right_index = 100;
+    indexer.GetNextLevelIndex(level, file_index, cmp_smallest, cmp_largest,
+        left_index, right_index);
+  }
+
+  const uint32_t kNumLevels;
+  IntComparator ucmp;
+  FileIndexer indexer;
+
+  std::vector<FileMetaData*>* files;
+};
+
+TEST(FileIndexerTest, next_level_hint) {
+  for (uint32_t i = 0; i < kNumLevels; ++i) {
+    ASSERT_EQ(0U, indexer.LevelIndexSize(i));
+  }
+
+  // Case 1: no overlap, files are on the left of next level files
+  // level 1
+  AddFile(1, 100, 200);
+  AddFile(1, 300, 400);
+  AddFile(1, 500, 600);
+  // level 2
+  AddFile(2, 1500, 1600);
+  AddFile(2, 1601, 1699);
+  AddFile(2, 1700, 1800);
+  // level 3
+  AddFile(3, 2500, 2600);
+  AddFile(3, 2601, 2699);
+  AddFile(3, 2700, 2800);
+  indexer.UpdateIndex(files);
+  int32_t left = 100;
+  int32_t right = 100;
+  for (uint32_t level = 1; level < 3; ++level) {
+    for (uint32_t f = 0; f < 3; ++f) {
+      GetNextLevelIndex(level, f, -1, -1, &left, &right);
+      ASSERT_EQ(0, left);
+      ASSERT_EQ(-1, right);
+      GetNextLevelIndex(level, f, 0, -1, &left, &right);
+      ASSERT_EQ(0, left);
+      ASSERT_EQ(-1, right);
+      GetNextLevelIndex(level, f, 1, -1, &left, &right);
+      ASSERT_EQ(0, left);
+      ASSERT_EQ(-1, right);
+      GetNextLevelIndex(level, f, 1, 0, &left, &right);
+      ASSERT_EQ(0, left);
+      ASSERT_EQ(-1, right);
+      GetNextLevelIndex(level, f, 1, 1, &left, &right);
+      ASSERT_EQ(0, left);
+      ASSERT_EQ(2, right);
+    }
+  }
+
+  // Case 2: no overlap, files are on the right of next level files
+  Reset();
+  for (uint32_t i = 1; i < kNumLevels; ++i) {
+    ASSERT_EQ(0U, indexer.LevelIndexSize(i));
+  }
+  // level 1
+  AddFile(1, 2100, 2200);
+  AddFile(1, 2300, 2400);
+  AddFile(1, 2500, 2600);
+  // level 2
+  AddFile(2, 1500, 1600);
+  AddFile(2, 1501, 1699);
+  AddFile(2, 1700, 1800);
+  // level 3
+  AddFile(3, 500, 600);
+  AddFile(3, 501, 699);
+  AddFile(3, 700, 800);
+  indexer.UpdateIndex(files);
+  for (uint32_t level = 1; level < 3; ++level) {
+    for (uint32_t f = 0; f < 3; ++f) {
+      GetNextLevelIndex(level, f, -1, -1, &left, &right);
+      ASSERT_EQ(f == 0 ? 0 : 3, left);
+      ASSERT_EQ(2, right);
+      GetNextLevelIndex(level, f, 0, -1, &left, &right);
+      ASSERT_EQ(3, left);
+      ASSERT_EQ(2, right);
+      GetNextLevelIndex(level, f, 1, -1, &left, &right);
+      ASSERT_EQ(3, left);
+      ASSERT_EQ(2, right);
+      GetNextLevelIndex(level, f, 1, -1, &left, &right);
+      ASSERT_EQ(3, left);
+      ASSERT_EQ(2, right);
+      GetNextLevelIndex(level, f, 1, 0, &left, &right);
+      ASSERT_EQ(3, left);
+      ASSERT_EQ(2, right);
+      GetNextLevelIndex(level, f, 1, 1, &left, &right);
+      ASSERT_EQ(3, left);
+      ASSERT_EQ(2, right);
+    }
+  }
+
+  // Case 3: empty L2
+  Reset();
+  for (uint32_t i = 1; i < kNumLevels; ++i) {
+    ASSERT_EQ(0U, indexer.LevelIndexSize(i));
+  }
+  // level 1
+  AddFile(1, 2100, 2200);
+  AddFile(1, 2300, 2400);
+  AddFile(1, 2500, 2600);
+  // level 3
+  AddFile(3, 500, 600);
+  AddFile(3, 501, 699);
+  AddFile(3, 700, 800);
+  indexer.UpdateIndex(files);
+  for (uint32_t f = 0; f < 3; ++f) {
+    GetNextLevelIndex(1, f, -1, -1, &left, &right);
+    ASSERT_EQ(0, left);
+    ASSERT_EQ(-1, right);
+    GetNextLevelIndex(1, f, 0, -1, &left, &right);
+    ASSERT_EQ(0, left);
+    ASSERT_EQ(-1, right);
+    GetNextLevelIndex(1, f, 1, -1, &left, &right);
+    ASSERT_EQ(0, left);
+    ASSERT_EQ(-1, right);
+    GetNextLevelIndex(1, f, 1, -1, &left, &right);
+    ASSERT_EQ(0, left);
+    ASSERT_EQ(-1, right);
+    GetNextLevelIndex(1, f, 1, 0, &left, &right);
+    ASSERT_EQ(0, left);
+    ASSERT_EQ(-1, right);
+    GetNextLevelIndex(1, f, 1, 1, &left, &right);
+    ASSERT_EQ(0, left);
+    ASSERT_EQ(-1, right);
+  }
+
+
+  // Case 4: mixed
+  Reset();
+  for (uint32_t i = 1; i < kNumLevels; ++i) {
+    ASSERT_EQ(0U, indexer.LevelIndexSize(i));
+  }
+  // level 1
+  AddFile(1, 100, 200);
+  AddFile(1, 250, 400);
+  AddFile(1, 450, 500);
+  // level 2
+  AddFile(2, 100, 150);  // 0
+  AddFile(2, 200, 250);  // 1
+  AddFile(2, 251, 300);  // 2
+  AddFile(2, 301, 350);  // 3
+  AddFile(2, 500, 600);  // 4
+  // level 3
+  AddFile(3, 0, 50);
+  AddFile(3, 100, 200);
+  AddFile(3, 201, 250);
+  indexer.UpdateIndex(files);
+  // level 1, 0
+  GetNextLevelIndex(1, 0, -1, -1, &left, &right);
+  ASSERT_EQ(0, left);
+  ASSERT_EQ(0, right);
+  GetNextLevelIndex(1, 0, 0, -1, &left, &right);
+  ASSERT_EQ(0, left);
+  ASSERT_EQ(0, right);
+  GetNextLevelIndex(1, 0, 1, -1, &left, &right);
+  ASSERT_EQ(0, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(1, 0, 1, 0, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(1, 0, 1, 1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(4, right);
+  // level 1, 1
+  GetNextLevelIndex(1, 1, -1, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(1, 1, 0, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(1, 1, 1, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(3, right);
+  GetNextLevelIndex(1, 1, 1, 0, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(3, right);
+  GetNextLevelIndex(1, 1, 1, 1, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(4, right);
+  // level 1, 2
+  GetNextLevelIndex(1, 2, -1, -1, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(3, right);
+  GetNextLevelIndex(1, 2, 0, -1, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(3, right);
+  GetNextLevelIndex(1, 2, 1, -1, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(4, right);
+  GetNextLevelIndex(1, 2, 1, 0, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(4, right);
+  GetNextLevelIndex(1, 2, 1, 1, &left, &right);
+  ASSERT_EQ(4, left);
+  ASSERT_EQ(4, right);
+  // level 2, 0
+  GetNextLevelIndex(2, 0, -1, -1, &left, &right);
+  ASSERT_EQ(0, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(2, 0, 0, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(2, 0, 1, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(2, 0, 1, 0, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(2, 0, 1, 1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(2, right);
+  // level 2, 1
+  GetNextLevelIndex(2, 1, -1, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(2, 1, 0, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(1, right);
+  GetNextLevelIndex(2, 1, 1, -1, &left, &right);
+  ASSERT_EQ(1, left);
+  ASSERT_EQ(2, right);
+  GetNextLevelIndex(2, 1, 1, 0, &left, &right);
+  ASSERT_EQ(2, left);
+  ASSERT_EQ(2, right);
+  GetNextLevelIndex(2, 1, 1, 1, &left, &right);
+  ASSERT_EQ(2, left);
+  ASSERT_EQ(2, right);
+  // level 2, [2 - 4], no overlap
+  for (uint32_t f = 2; f <= 4; ++f) {
+    GetNextLevelIndex(2, f, -1, -1, &left, &right);
+    ASSERT_EQ(f == 2 ? 2 : 3, left);
+    ASSERT_EQ(2, right);
+    GetNextLevelIndex(2, f, 0, -1, &left, &right);
+    ASSERT_EQ(3, left);
+    ASSERT_EQ(2, right);
+    GetNextLevelIndex(2, f, 1, -1, &left, &right);
+    ASSERT_EQ(3, left);
+    ASSERT_EQ(2, right);
+    GetNextLevelIndex(2, f, 1, 0, &left, &right);
+    ASSERT_EQ(3, left);
+    ASSERT_EQ(2, right);
+    GetNextLevelIndex(2, f, 1, 1, &left, &right);
+    ASSERT_EQ(3, left);
+    ASSERT_EQ(2, right);
+  }
+}
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  return rocksdb::test::RunAllTests();
+}

db/filename.cc

@@ -44,10 +44,6 @@ static int FlattenPath(const std::string& path, char* dest, int len) {
   return write_idx;
 }
 
-// A utility routine: write "data" to the named file and Sync() it.
-extern Status WriteStringToFileSync(Env* env, const Slice& data,
-                                    const std::string& fname);
-
 static std::string MakeFileName(const std::string& name, uint64_t number,
                                 const char* suffix) {
   char buf[100];
@@ -92,7 +88,6 @@ std::string LockFileName(const std::string& dbname) {
 }
 
 std::string TempFileName(const std::string& dbname, uint64_t number) {
-  assert(number >= 0);
   return MakeFileName(dbname, number, "dbtmp");
 }
 
@@ -238,7 +233,7 @@ Status SetCurrentFile(Env* env, const std::string& dbname,
   assert(contents.starts_with(dbname + "/"));
   contents.remove_prefix(dbname.size() + 1);
   std::string tmp = TempFileName(dbname, descriptor_number);
-  Status s = WriteStringToFileSync(env, contents.ToString() + "\n", tmp);
+  Status s = WriteStringToFile(env, contents.ToString() + "\n", tmp, true);
   if (s.ok()) {
     s = env->RenameFile(tmp, CurrentFileName(dbname));
   }
@@ -253,7 +248,7 @@ Status SetIdentityFile(Env* env, const std::string& dbname) {
   assert(!id.empty());
   // Reserve the filename dbname/000000.dbtmp for the temporary identity file
   std::string tmp = TempFileName(dbname, 0);
-  Status s = WriteStringToFileSync(env, id, tmp);
+  Status s = WriteStringToFile(env, id, tmp, true);
   if (s.ok()) {
     s = env->RenameFile(tmp, IdentityFileName(dbname));
   }

db/internal_stats.cc

@@ -7,8 +7,7 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include "db/internal_stats.h"
-#include "db/db_impl.h"
-#include "db/memtable_list.h"
+#include "db/column_family.h"
 
 #include <vector>
 
@@ -38,16 +37,18 @@ DBPropertyType GetPropertyType(const Slice& property) {
     return kBackgroundErrors;
   } else if (in == "cur-size-active-mem-table") {
     return kCurSizeActiveMemTable;
+  } else if (in == "num-entries-active-mem-table") {
+    return kNumEntriesInMutableMemtable;
+  } else if (in == "num-entries-imm-mem-tables") {
+    return kNumEntriesInImmutableMemtable;
   }
   return kUnknown;
 }
 
 bool InternalStats::GetProperty(DBPropertyType property_type,
                                 const Slice& property, std::string* value,
-                                DBImpl* db) {
-  VersionSet* version_set = db->versions_.get();
-  Version* current = version_set->current();
-  const MemTableList& imm = db->imm_;
+                                ColumnFamilyData* cfd) {
+  Version* current = cfd->current();
   Slice in = property;
 
   switch (property_type) {
@@ -110,7 +111,6 @@ bool InternalStats::GetProperty(DBPropertyType property_type,
         write_with_wal = statistics_->getTickerCount(WRITE_WITH_WAL);
       }
 
-      // Pardon the long line but I think it is easier to read this way.
       snprintf(
           buf, sizeof(buf),
           "                               Compactions\n"
@@ -159,7 +159,7 @@ bool InternalStats::GetProperty(DBPropertyType property_type,
                    "%9lu\n",
                    level, files, current->NumLevelBytes(level) / 1048576.0,
                    current->NumLevelBytes(level) /
-                       version_set->MaxBytesForLevel(level),
+                       cfd->compaction_picker()->MaxBytesForLevel(level),
                    compaction_stats_[level].micros / 1e6,
                    bytes_read / 1048576.0,
                    compaction_stats_[level].bytes_written / 1048576.0,
@@ -334,11 +334,11 @@ bool InternalStats::GetProperty(DBPropertyType property_type,
       *value = current->DebugString();
       return true;
     case kNumImmutableMemTable:
-      *value = std::to_string(imm.size());
+      *value = std::to_string(cfd->imm()->size());
       return true;
     case kMemtableFlushPending:
       // Return number of mem tables that are ready to flush (made immutable)
-      *value = std::to_string(imm.IsFlushPending() ? 1 : 0);
+      *value = std::to_string(cfd->imm()->IsFlushPending() ? 1 : 0);
       return true;
     case kCompactionPending:
       // 1 if the system already determines at least one compacdtion is needed.
@@ -351,7 +351,15 @@ bool InternalStats::GetProperty(DBPropertyType property_type,
       return true;
     case kCurSizeActiveMemTable:
       // Current size of the active memtable
-      *value = std::to_string(db->mem_->ApproximateMemoryUsage());
+      *value = std::to_string(cfd->mem()->ApproximateMemoryUsage());
+      return true;
+    case kNumEntriesInMutableMemtable:
+      // Current size of the active memtable
+      *value = std::to_string(cfd->mem()->GetNumEntries());
+      return true;
+    case kNumEntriesInImmutableMemtable:
+      // Current size of the active memtable
+      *value = std::to_string(cfd->imm()->current()->GetTotalNumEntries());
       return true;
     default:
       return false;

db/internal_stats.h

@@ -16,6 +16,8 @@
 #include <vector>
 #include <string>
 
+class ColumnFamilyData;
+
 namespace rocksdb {
 
 class MemTableList;
@@ -26,13 +28,16 @@ enum DBPropertyType {
   kLevelStats,       // Return number of files and total sizes of each level
   kStats,            // Return general statitistics of DB
   kSsTables,         // Return a human readable string of current SST files
-  kNumImmutableMemTable,  // Return number of immutable mem tables
-  kMemtableFlushPending,  // Return 1 if mem table flushing is pending,
-                          // otherwise
-                          // 0.
-  kCompactionPending,     // Return 1 if a compaction is pending. Otherwise 0.
-  kBackgroundErrors,      // Return accumulated background errors encountered.
+  kNumImmutableMemTable,   // Return number of immutable mem tables
+  kMemtableFlushPending,   // Return 1 if mem table flushing is pending,
+                           // otherwise 0.
+  kCompactionPending,      // Return 1 if a compaction is pending. Otherwise 0.
+  kBackgroundErrors,       // Return accumulated background errors encountered.
   kCurSizeActiveMemTable,  // Return current size of the active memtable
+  kNumEntriesInMutableMemtable,    // Return number of entries in the mutable
+                                   // memtable.
+  kNumEntriesInImmutableMemtable,  // Return sum of number of entries in all
+                                   // the immutable mem tables.
   kUnknown,
 };
 
@@ -126,7 +131,7 @@ class InternalStats {
   uint64_t BumpAndGetBackgroundErrorCount() { return ++bg_error_count_; }
 
   bool GetProperty(DBPropertyType property_type, const Slice& property,
-                   std::string* value, DBImpl* db);
+                   std::string* value, ColumnFamilyData* cfd);
 
  private:
   std::vector<CompactionStats> compaction_stats_;

db/log_and_apply_bench.cc

@@ -0,0 +1,79 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+
+#include <vector>
+
+#include "util/testharness.h"
+#include "util/benchharness.h"
+#include "db/version_set.h"
+#include "util/mutexlock.h"
+
+namespace rocksdb {
+
+std::string MakeKey(unsigned int num) {
+  char buf[30];
+  snprintf(buf, sizeof(buf), "%016u", num);
+  return std::string(buf);
+}
+
+void BM_LogAndApply(int iters, int num_base_files) {
+  VersionSet* vset;
+  ColumnFamilyData* default_cfd;
+  uint64_t fnum = 1;
+  port::Mutex mu;
+  MutexLock l(&mu);
+
+  BENCHMARK_SUSPEND {
+    std::string dbname = test::TmpDir() + "/rocksdb_test_benchmark";
+    ASSERT_OK(DestroyDB(dbname, Options()));
+
+    DB* db = nullptr;
+    Options opts;
+    opts.create_if_missing = true;
+    Status s = DB::Open(opts, dbname, &db);
+    ASSERT_OK(s);
+    ASSERT_TRUE(db != nullptr);
+
+    delete db;
+    db = nullptr;
+
+    Options options;
+    EnvOptions sopt;
+    vset = new VersionSet(dbname, &options, sopt, nullptr);
+    std::vector<ColumnFamilyDescriptor> dummy;
+    dummy.push_back(ColumnFamilyDescriptor());
+    ASSERT_OK(vset->Recover(dummy));
+    default_cfd = vset->GetColumnFamilySet()->GetDefault();
+    VersionEdit vbase;
+    for (int i = 0; i < num_base_files; i++) {
+      InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
+      InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
+      vbase.AddFile(2, ++fnum, 1 /* file size */, start, limit, 1, 1);
+    }
+    ASSERT_OK(vset->LogAndApply(default_cfd, &vbase, &mu));
+  }
+
+  for (int i = 0; i < iters; i++) {
+    VersionEdit vedit;
+    vedit.DeleteFile(2, fnum);
+    InternalKey start(MakeKey(2 * fnum), 1, kTypeValue);
+    InternalKey limit(MakeKey(2 * fnum + 1), 1, kTypeDeletion);
+    vedit.AddFile(2, ++fnum, 1 /* file size */, start, limit, 1, 1);
+    vset->LogAndApply(default_cfd, &vedit, &mu);
+  }
+}
+
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 1000_iters_1_file, 1000, 1)
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 1000_iters_100_files, 1000, 100)
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 1000_iters_10000_files, 1000, 10000)
+BENCHMARK_NAMED_PARAM(BM_LogAndApply, 100_iters_100000_files, 100, 100000)
+
+}  // namespace rocksdb
+
+int main(int argc, char** argv) {
+  rocksdb::benchmark::RunBenchmarks();
+  return 0;
+}

db/memtable.cc

@@ -11,6 +11,7 @@
 
 #include <memory>
 #include <algorithm>
+#include <limits>
 
 #include "db/dbformat.h"
 #include "db/merge_context.h"
@@ -37,12 +38,12 @@ MemTable::MemTable(const InternalKeyComparator& cmp, const Options& options)
       arena_(options.arena_block_size),
       table_(options.memtable_factory->CreateMemTableRep(
           comparator_, &arena_, options.prefix_extractor.get())),
+      num_entries_(0),
       flush_in_progress_(false),
       flush_completed_(false),
       file_number_(0),
       first_seqno_(0),
       mem_next_logfile_number_(0),
-      mem_logfile_number_(0),
       locks_(options.inplace_update_support ? options.inplace_update_num_locks
                                             : 0),
       prefix_extractor_(options.prefix_extractor.get()),
@@ -51,9 +52,10 @@ MemTable::MemTable(const InternalKeyComparator& cmp, const Options& options)
   // gone wrong already.
   assert(!should_flush_);
   if (prefix_extractor_ && options.memtable_prefix_bloom_bits > 0) {
-    prefix_bloom_.reset(new DynamicBloom(options.memtable_prefix_bloom_bits,
-                                         options.bloom_locality,
-                                         options.memtable_prefix_bloom_probes));
+    prefix_bloom_.reset(new DynamicBloom(
+        options.memtable_prefix_bloom_bits, options.bloom_locality,
+        options.memtable_prefix_bloom_probes, nullptr,
+        options.memtable_prefix_bloom_huge_page_tlb_size));
   }
 }
 
@@ -62,7 +64,16 @@ MemTable::~MemTable() {
 }
 
 size_t MemTable::ApproximateMemoryUsage() {
-  return arena_.ApproximateMemoryUsage() + table_->ApproximateMemoryUsage();
+  size_t arena_usage = arena_.ApproximateMemoryUsage();
+  size_t table_usage = table_->ApproximateMemoryUsage();
+  // let MAX_USAGE =  std::numeric_limits<size_t>::max()
+  // then if arena_usage + total_usage >= MAX_USAGE, return MAX_USAGE.
+  // the following variation is to avoid numeric overflow.
+  if (arena_usage >= std::numeric_limits<size_t>::max() - table_usage) {
+    return std::numeric_limits<size_t>::max();
+  }
+  // otherwise, return the actual usage
+  return arena_usage + table_usage;
 }
 
 bool MemTable::ShouldFlushNow() const {
@@ -159,14 +170,12 @@ const char* EncodeKey(std::string* scratch, const Slice& target) {
 
 class MemTableIterator: public Iterator {
  public:
-  MemTableIterator(const MemTable& mem, const ReadOptions& options)
+  MemTableIterator(const MemTable& mem, const ReadOptions& options,
+                   bool enforce_total_order)
       : bloom_(nullptr),
         prefix_extractor_(mem.prefix_extractor_),
-        iter_(),
         valid_(false) {
-    if (options.prefix) {
-      iter_.reset(mem.table_->GetPrefixIterator(*options.prefix));
-    } else if (options.prefix_seek) {
+    if (prefix_extractor_ != nullptr && !enforce_total_order) {
       bloom_ = mem.prefix_bloom_.get();
       iter_.reset(mem.table_->GetDynamicPrefixIterator());
     } else {
@@ -217,7 +226,7 @@ class MemTableIterator: public Iterator {
  private:
   DynamicBloom* bloom_;
   const SliceTransform* const prefix_extractor_;
-  std::shared_ptr<MemTableRep::Iterator> iter_;
+  std::unique_ptr<MemTableRep::Iterator> iter_;
   bool valid_;
 
   // No copying allowed
@@ -225,8 +234,9 @@ class MemTableIterator: public Iterator {
   void operator=(const MemTableIterator&);
 };
 
-Iterator* MemTable::NewIterator(const ReadOptions& options) {
-  return new MemTableIterator(*this, options);
+Iterator* MemTable::NewIterator(const ReadOptions& options,
+    bool enforce_total_order) {
+  return new MemTableIterator(*this, options, enforce_total_order);
 }
 
 port::RWMutex* MemTable::GetLock(const Slice& key) {
@@ -260,6 +270,7 @@ void MemTable::Add(SequenceNumber s, ValueType type,
   memcpy(p, value.data(), val_size);
   assert((unsigned)(p + val_size - buf) == (unsigned)encoded_len);
   table_->Insert(handle);
+  num_entries_++;
 
   if (prefix_bloom_) {
     assert(prefix_extractor_);
@@ -377,8 +388,7 @@ static bool SaveValue(void* arg, const char* entry) {
 
 bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
                    MergeContext& merge_context, const Options& options) {
-  StopWatchNano memtable_get_timer(options.env, false);
-  StartPerfTimer(&memtable_get_timer);
+  PERF_TIMER_AUTO(get_from_memtable_time);
 
   Slice user_key = key.user_key();
   bool found_final_value = false;
@@ -408,8 +418,8 @@ bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
   if (!found_final_value && merge_in_progress) {
     *s = Status::MergeInProgress("");
   }
-  BumpPerfTime(&perf_context.get_from_memtable_time, &memtable_get_timer);
-  BumpPerfCount(&perf_context.get_from_memtable_count);
+  PERF_TIMER_STOP(get_from_memtable_time);
+  PERF_COUNTER_ADD(get_from_memtable_count, 1);
   return found_final_value;
 }
 
@@ -478,7 +488,7 @@ bool MemTable::UpdateCallback(SequenceNumber seq,
   LookupKey lkey(key, seq);
   Slice memkey = lkey.memtable_key();
 
-  std::shared_ptr<MemTableRep::Iterator> iter(
+  std::unique_ptr<MemTableRep::Iterator> iter(
     table_->GetIterator(lkey.user_key()));
   iter->Seek(lkey.internal_key(), memkey.data());
 

db/memtable.h

@@ -13,7 +13,7 @@
 #include <deque>
 #include "db/dbformat.h"
 #include "db/skiplist.h"
-#include "db/version_set.h"
+#include "db/version_edit.h"
 #include "rocksdb/db.h"
 #include "rocksdb/memtablerep.h"
 #include "util/arena.h"
@@ -39,7 +39,7 @@ class MemTable {
   // MemTables are reference counted.  The initial reference count
   // is zero and the caller must call Ref() at least once.
   explicit MemTable(const InternalKeyComparator& comparator,
-                    const Options& options = Options());
+                    const Options& options);
 
   ~MemTable();
 
@@ -75,14 +75,10 @@ class MemTable {
   // iterator are internal keys encoded by AppendInternalKey in the
   // db/dbformat.{h,cc} module.
   //
-  // If options.prefix is supplied, it is passed to the underlying MemTableRep
-  // as a hint that the iterator only need to support access to keys with that
-  // specific prefix.
-  // If options.prefix is not supplied and options.prefix_seek is set, the
-  // iterator is not bound to a specific prefix. However, the semantics of
-  // Seek is changed - the result might only include keys with the same prefix
-  // as the seek-key.
-  Iterator* NewIterator(const ReadOptions& options = ReadOptions());
+  // By default, it returns an iterator for prefix seek if prefix_extractor
+  // is configured in Options.
+  Iterator* NewIterator(const ReadOptions& options,
+                        bool enforce_total_order = false);
 
   // Add an entry into memtable that maps key to value at the
   // specified sequence number and with the specified type.
@@ -132,6 +128,9 @@ class MemTable {
   // key in the memtable.
   size_t CountSuccessiveMergeEntries(const LookupKey& key);
 
+  // Get total number of entries in the mem table.
+  uint64_t GetNumEntries() const { return num_entries_; }
+
   // Returns the edits area that is needed for flushing the memtable
   VersionEdit* GetEdits() { return &edit_; }
 
@@ -147,17 +146,17 @@ class MemTable {
   // be flushed to storage
   void SetNextLogNumber(uint64_t num) { mem_next_logfile_number_ = num; }
 
-  // Returns the logfile number that can be safely deleted when this
-  // memstore is flushed to storage
-  uint64_t GetLogNumber() { return mem_logfile_number_; }
-
-  // Sets the logfile number that can be safely deleted when this
-  // memstore is flushed to storage
-  void SetLogNumber(uint64_t num) { mem_logfile_number_ = num; }
-
   // Notify the underlying storage that no more items will be added
   void MarkImmutable() { table_->MarkReadOnly(); }
 
+  // return true if the current MemTableRep supports merge operator.
+  bool IsMergeOperatorSupported() const {
+    return table_->IsMergeOperatorSupported();
+  }
+
+  // return true if the current MemTableRep supports snapshots.
+  bool IsSnapshotSupported() const { return table_->IsSnapshotSupported(); }
+
   // Get the lock associated for the key
   port::RWMutex* GetLock(const Slice& key);
 
@@ -182,6 +181,8 @@ class MemTable {
   Arena arena_;
   unique_ptr<MemTableRep> table_;
 
+  uint64_t num_entries_;
+
   // These are used to manage memtable flushes to storage
   bool flush_in_progress_; // started the flush
   bool flush_completed_;   // finished the flush
@@ -197,10 +198,6 @@ class MemTable {
   // The log files earlier than this number can be deleted.
   uint64_t mem_next_logfile_number_;
 
-  // The log file that backs this memtable (to be deleted when
-  // memtable flush is done)
-  uint64_t mem_logfile_number_;
-
   // rw locks for inplace updates
   std::vector<port::RWMutex> locks_;
 

db/memtable_list.cc

@@ -8,6 +8,7 @@
 #include <string>
 #include "rocksdb/db.h"
 #include "db/memtable.h"
+#include "db/version_set.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
 #include "util/coding.h"
@@ -77,6 +78,14 @@ void MemTableListVersion::AddIterators(const ReadOptions& options,
   }
 }
 
+uint64_t MemTableListVersion::GetTotalNumEntries() const {
+  uint64_t total_num = 0;
+  for (auto& m : memlist_) {
+    total_num += m->GetNumEntries();
+  }
+  return total_num;
+}
+
 // caller is responsible for referencing m
 void MemTableListVersion::Add(MemTable* m) {
   assert(refs_ == 1);  // only when refs_ == 1 is MemTableListVersion mutable
@@ -121,7 +130,8 @@ void MemTableList::PickMemtablesToFlush(autovector<MemTable*>* ret) {
 }
 
 void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
-     uint64_t file_number, std::set<uint64_t>* pending_outputs) {
+                                         uint64_t file_number,
+                                         std::set<uint64_t>* pending_outputs) {
   assert(!mems.empty());
 
   // If the flush was not successful, then just reset state.
@@ -141,10 +151,10 @@ void MemTableList::RollbackMemtableFlush(const autovector<MemTable*>& mems,
 
 // Record a successful flush in the manifest file
 Status MemTableList::InstallMemtableFlushResults(
-    const autovector<MemTable*>& mems, VersionSet* vset, port::Mutex* mu,
-    Logger* info_log, uint64_t file_number, std::set<uint64_t>& pending_outputs,
-    autovector<MemTable*>* to_delete, Directory* db_directory,
-    LogBuffer* log_buffer) {
+    ColumnFamilyData* cfd, const autovector<MemTable*>& mems, VersionSet* vset,
+    port::Mutex* mu, Logger* info_log, uint64_t file_number,
+    std::set<uint64_t>& pending_outputs, autovector<MemTable*>* to_delete,
+    Directory* db_directory, LogBuffer* log_buffer) {
   mu->AssertHeld();
 
   // flush was sucessful
@@ -174,11 +184,11 @@ Status MemTableList::InstallMemtableFlushResults(
       break;
     }
 
-    LogToBuffer(log_buffer, "Level-0 commit table #%lu started",
-                (unsigned long)m->file_number_);
+    LogToBuffer(log_buffer, "[%s] Level-0 commit table #%lu started",
+                cfd->GetName().c_str(), (unsigned long)m->file_number_);
 
     // this can release and reacquire the mutex.
-    s = vset->LogAndApply(&m->edit_, mu, db_directory);
+    s = vset->LogAndApply(cfd, &m->edit_, mu, db_directory);
 
     // we will be changing the version in the next code path,
     // so we better create a new one, since versions are immutable
@@ -189,8 +199,10 @@ Status MemTableList::InstallMemtableFlushResults(
     uint64_t mem_id = 1;  // how many memtables has been flushed.
     do {
       if (s.ok()) { // commit new state
-        LogToBuffer(log_buffer, "Level-0 commit table #%lu: memtable #%lu done",
-                    (unsigned long)m->file_number_, (unsigned long)mem_id);
+        LogToBuffer(log_buffer,
+                    "[%s] Level-0 commit table #%lu: memtable #%lu done",
+                    cfd->GetName().c_str(), (unsigned long)m->file_number_,
+                    (unsigned long)mem_id);
         current_->Remove(m);
         assert(m->file_number_ > 0);
 

db/memtable_list.h

@@ -7,19 +7,25 @@
 
 #include <string>
 #include <list>
+#include <vector>
+#include <set>
 #include <deque>
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
+#include "rocksdb/iterator.h"
 
 #include "db/dbformat.h"
-#include "db/memtable.h"
 #include "db/skiplist.h"
-#include "rocksdb/db.h"
+#include "db/memtable.h"
 #include "rocksdb/db.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 #include "util/autovector.h"
+#include "util/log_buffer.h"
 
 namespace rocksdb {
 
+class ColumnFamilyData;
 class InternalKeyComparator;
 class Mutex;
 
@@ -43,6 +49,8 @@ class MemTableListVersion {
   void AddIterators(const ReadOptions& options,
                     std::vector<Iterator*>* iterator_list);
 
+  uint64_t GetTotalNumEntries() const;
+
  private:
   // REQUIRE: m is mutable memtable
   void Add(MemTable* m);
@@ -99,7 +107,8 @@ class MemTableList {
                              std::set<uint64_t>* pending_outputs);
 
   // Commit a successful flush in the manifest file
-  Status InstallMemtableFlushResults(const autovector<MemTable*>& m,
+  Status InstallMemtableFlushResults(ColumnFamilyData* cfd,
+                                     const autovector<MemTable*>& m,
                                      VersionSet* vset, port::Mutex* mu,
                                      Logger* info_log, uint64_t file_number,
                                      std::set<uint64_t>& pending_outputs,

db/merge_test.cc

@@ -17,7 +17,7 @@
 #include "db/write_batch_internal.h"
 #include "utilities/merge_operators.h"
 #include "util/testharness.h"
-#include "utilities/utility_db.h"
+#include "utilities/db_ttl.h"
 
 using namespace std;
 using namespace rocksdb;
@@ -75,11 +75,11 @@ class CountMergeOperator : public AssociativeMergeOperator {
   std::shared_ptr<MergeOperator> mergeOperator_;
 };
 
+namespace {
 std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false,
                            const size_t max_successive_merges = 0,
                            const uint32_t min_partial_merge_operands = 2) {
   DB* db;
-  StackableDB* sdb;
   Options options;
   options.create_if_missing = true;
   options.merge_operator = std::make_shared<CountMergeOperator>();
@@ -89,8 +89,9 @@ std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false,
   DestroyDB(dbname, Options());
   if (ttl) {
     cout << "Opening database with TTL\n";
-    s = UtilityDB::OpenTtlDB(options, dbname, &sdb);
-    db = sdb;
+    DBWithTTL* db_with_ttl;
+    s = DBWithTTL::Open(options, dbname, &db_with_ttl);
+    db = db_with_ttl;
   } else {
     s = DB::Open(options, dbname, &db);
   }
@@ -100,6 +101,7 @@ std::shared_ptr<DB> OpenDb(const string& dbname, const bool ttl = false,
   }
   return std::shared_ptr<DB>(db);
 }
+}  // namespace
 
 // Imagine we are maintaining a set of uint64 counters.
 // Each counter has a distinct name. And we would like
@@ -201,12 +203,16 @@ class Counters {
 
   uint64_t assert_get(const string& key) {
     uint64_t value = default_;
-    assert(get(key, &value));
+    int result = get(key, &value);
+    assert(result);
+    if (result == 0) exit(1); // Disable unused variable warning.
     return value;
   }
 
   void assert_add(const string& key, uint64_t value) {
-    assert(add(key, value));
+    int result = add(key, value);
+    assert(result);
+    if (result == 0) exit(1); // Disable unused variable warning. 
   }
 };
 
@@ -237,6 +243,7 @@ class MergeBasedCounters : public Counters {
   }
 };
 
+namespace {
 void dumpDb(DB* db) {
   auto it = unique_ptr<Iterator>(db->NewIterator(ReadOptions()));
   for (it->SeekToFirst(); it->Valid(); it->Next()) {
@@ -454,6 +461,7 @@ void runTest(int argc, const string& dbname, const bool use_ttl = false) {
     }
   }
 }
+}  // namespace
 
 int main(int argc, char *argv[]) {
   //TODO: Make this test like a general rocksdb unit-test

db/plain_table_db_test.cc

@@ -47,7 +47,6 @@ class PlainTableDBTest {
 
  public:
   PlainTableDBTest() : env_(Env::Default()) {
-    ro_.prefix_seek = true;
     dbname_ = test::TmpDir() + "/plain_table_db_test";
     ASSERT_OK(DestroyDB(dbname_, Options()));
     db_ = nullptr;
@@ -59,8 +58,6 @@ class PlainTableDBTest {
     ASSERT_OK(DestroyDB(dbname_, Options()));
   }
 
-  ReadOptions ro_;
-
   // Return the current option configuration.
   Options CurrentOptions() {
     Options options;
@@ -123,7 +120,7 @@ class PlainTableDBTest {
   }
 
   std::string Get(const std::string& k, const Snapshot* snapshot = nullptr) {
-    ReadOptions options = ro_;
+    ReadOptions options;
     options.snapshot = snapshot;
     std::string result;
     Status s = db_->Get(options, k, &result);
@@ -188,9 +185,9 @@ class TestPlainTableReader : public PlainTableReader {
                        const Options& options, bool* expect_bloom_not_match)
       : PlainTableReader(options, std::move(file), storage_options, icomparator,
                          file_size, bloom_bits_per_key, hash_table_ratio,
-                         index_sparseness, table_properties),
+                         index_sparseness, table_properties, 2 * 1024 * 1024),
         expect_bloom_not_match_(expect_bloom_not_match) {
-    Status s = PopulateIndex();
+    Status s = PopulateIndex(const_cast<TableProperties*>(table_properties));
     ASSERT_TRUE(s.ok());
   }
 
@@ -209,13 +206,12 @@ extern const uint64_t kPlainTableMagicNumber;
 class TestPlainTableFactory : public PlainTableFactory {
  public:
   explicit TestPlainTableFactory(bool* expect_bloom_not_match,
-                                 uint32_t user_key_len =
-                                     kPlainTableVariableLength,
-                                 int bloom_bits_per_key = 0,
-                                 double hash_table_ratio = 0.75,
-                                 size_t index_sparseness = 16)
+                                 uint32_t user_key_len, int bloom_bits_per_key,
+                                 double hash_table_ratio,
+                                 size_t index_sparseness,
+                                 size_t huge_page_tlb_size)
       : PlainTableFactory(user_key_len, user_key_len, hash_table_ratio,
-                          hash_table_ratio),
+                          index_sparseness, huge_page_tlb_size),
         bloom_bits_per_key_(bloom_bits_per_key),
         hash_table_ratio_(hash_table_ratio),
         index_sparseness_(index_sparseness),
@@ -247,188 +243,257 @@ class TestPlainTableFactory : public PlainTableFactory {
 };
 
 TEST(PlainTableDBTest, Flush) {
-  for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
-    for (int total_order = 0; total_order <= 1; total_order++) {
-      Options options = CurrentOptions();
-      options.create_if_missing = true;
-      // Set only one bucket to force bucket conflict.
-      // Test index interval for the same prefix to be 1, 2 and 4
-      if (total_order) {
-        options.table_factory.reset(
-            NewTotalOrderPlainTableFactory(16, bloom_bits, 2));
-      } else {
-        options.table_factory.reset(NewPlainTableFactory(16, bloom_bits));
-      }
-      DestroyAndReopen(&options);
+  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
+       huge_page_tlb_size += 2 * 1024 * 1024) {
+    for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
+      for (int total_order = 0; total_order <= 1; total_order++) {
+        Options options = CurrentOptions();
+        options.create_if_missing = true;
+        // Set only one bucket to force bucket conflict.
+        // Test index interval for the same prefix to be 1, 2 and 4
+        if (total_order) {
+          options.table_factory.reset(NewTotalOrderPlainTableFactory(
+              16, bloom_bits, 2, huge_page_tlb_size));
+        } else {
+          options.table_factory.reset(NewPlainTableFactory(
+              16, bloom_bits, 0.75, 16, huge_page_tlb_size));
+        }
+        DestroyAndReopen(&options);
 
-      ASSERT_OK(Put("1000000000000foo", "v1"));
-      ASSERT_OK(Put("0000000000000bar", "v2"));
-      ASSERT_OK(Put("1000000000000foo", "v3"));
-      dbfull()->TEST_FlushMemTable();
-      ASSERT_EQ("v3", Get("1000000000000foo"));
-      ASSERT_EQ("v2", Get("0000000000000bar"));
+        ASSERT_OK(Put("1000000000000foo", "v1"));
+        ASSERT_OK(Put("0000000000000bar", "v2"));
+        ASSERT_OK(Put("1000000000000foo", "v3"));
+        dbfull()->TEST_FlushMemTable();
+
+        TablePropertiesCollection ptc;
+        reinterpret_cast<DB*>(dbfull())->GetPropertiesOfAllTables(&ptc);
+        ASSERT_EQ(1U, ptc.size());
+        auto row = ptc.begin();
+        auto tp = row->second;
+        ASSERT_EQ(total_order ? "4" : "12", (tp->user_collected_properties).at(
+                                                "plain_table_hash_table_size"));
+        ASSERT_EQ(total_order ? "9" : "0", (tp->user_collected_properties).at(
+                                               "plain_table_sub_index_size"));
+
+        ASSERT_EQ("v3", Get("1000000000000foo"));
+        ASSERT_EQ("v2", Get("0000000000000bar"));
+      }
     }
   }
 }
 
 TEST(PlainTableDBTest, Flush2) {
-  for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
-    for (int total_order = 0; total_order <= 1; total_order++) {
-      bool expect_bloom_not_match = false;
-      Options options = CurrentOptions();
-      options.create_if_missing = true;
-      // Set only one bucket to force bucket conflict.
-      // Test index interval for the same prefix to be 1, 2 and 4
-      if (total_order) {
-        options.prefix_extractor = nullptr;
-        options.table_factory.reset(new TestPlainTableFactory(
-            &expect_bloom_not_match, 16, bloom_bits, 0, 2));
-      } else {
-        options.table_factory.reset(
-            new TestPlainTableFactory(&expect_bloom_not_match, 16, bloom_bits));
-      }
-      DestroyAndReopen(&options);
-      ASSERT_OK(Put("0000000000000bar", "b"));
-      ASSERT_OK(Put("1000000000000foo", "v1"));
-      dbfull()->TEST_FlushMemTable();
-
-      ASSERT_OK(Put("1000000000000foo", "v2"));
-      dbfull()->TEST_FlushMemTable();
-      ASSERT_EQ("v2", Get("1000000000000foo"));
-
-      ASSERT_OK(Put("0000000000000eee", "v3"));
-      dbfull()->TEST_FlushMemTable();
-      ASSERT_EQ("v3", Get("0000000000000eee"));
-
-      ASSERT_OK(Delete("0000000000000bar"));
-      dbfull()->TEST_FlushMemTable();
-      ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
-
-      ASSERT_OK(Put("0000000000000eee", "v5"));
-      ASSERT_OK(Put("9000000000000eee", "v5"));
-      dbfull()->TEST_FlushMemTable();
-      ASSERT_EQ("v5", Get("0000000000000eee"));
-
-      // Test Bloom Filter
-      if (bloom_bits > 0) {
-        // Neither key nor value should exist.
-        expect_bloom_not_match = true;
-        ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
-
-        // Key doesn't exist any more but prefix exists.
+  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
+       huge_page_tlb_size += 2 * 1024 * 1024) {
+    for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
+      for (int total_order = 0; total_order <= 1; total_order++) {
+        bool expect_bloom_not_match = false;
+        Options options = CurrentOptions();
+        options.create_if_missing = true;
+        // Set only one bucket to force bucket conflict.
+        // Test index interval for the same prefix to be 1, 2 and 4
         if (total_order) {
-          ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
-          ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
+          options.prefix_extractor = nullptr;
+          options.table_factory.reset(
+              new TestPlainTableFactory(&expect_bloom_not_match, 16, bloom_bits,
+                                        0, 2, huge_page_tlb_size));
+        } else {
+          options.table_factory.reset(
+              new TestPlainTableFactory(&expect_bloom_not_match, 16, bloom_bits,
+                                        0.75, 16, huge_page_tlb_size));
+        }
+        DestroyAndReopen(&options);
+        ASSERT_OK(Put("0000000000000bar", "b"));
+        ASSERT_OK(Put("1000000000000foo", "v1"));
+        dbfull()->TEST_FlushMemTable();
+
+        ASSERT_OK(Put("1000000000000foo", "v2"));
+        dbfull()->TEST_FlushMemTable();
+        ASSERT_EQ("v2", Get("1000000000000foo"));
+
+        ASSERT_OK(Put("0000000000000eee", "v3"));
+        dbfull()->TEST_FlushMemTable();
+        ASSERT_EQ("v3", Get("0000000000000eee"));
+
+        ASSERT_OK(Delete("0000000000000bar"));
+        dbfull()->TEST_FlushMemTable();
+        ASSERT_EQ("NOT_FOUND", Get("0000000000000bar"));
+
+        ASSERT_OK(Put("0000000000000eee", "v5"));
+        ASSERT_OK(Put("9000000000000eee", "v5"));
+        dbfull()->TEST_FlushMemTable();
+        ASSERT_EQ("v5", Get("0000000000000eee"));
+
+        // Test Bloom Filter
+        if (bloom_bits > 0) {
+          // Neither key nor value should exist.
+          expect_bloom_not_match = true;
+          ASSERT_EQ("NOT_FOUND", Get("5_not00000000bar"));
+
+          // Key doesn't exist any more but prefix exists.
+          if (total_order) {
+            ASSERT_EQ("NOT_FOUND", Get("1000000000000not"));
+            ASSERT_EQ("NOT_FOUND", Get("0000000000000not"));
+          }
+          expect_bloom_not_match = false;
         }
-        expect_bloom_not_match = false;
       }
     }
   }
 }
 
 TEST(PlainTableDBTest, Iterator) {
-  for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
-    for (int total_order = 0; total_order <= 1; total_order++) {
-      bool expect_bloom_not_match = false;
-      Options options = CurrentOptions();
-      options.create_if_missing = true;
-      // Set only one bucket to force bucket conflict.
-      // Test index interval for the same prefix to be 1, 2 and 4
-      if (total_order) {
-        options.prefix_extractor = nullptr;
-        options.table_factory.reset(new TestPlainTableFactory(
-            &expect_bloom_not_match, 16, bloom_bits, 0, 2));
-      } else {
-        options.table_factory.reset(
-            new TestPlainTableFactory(&expect_bloom_not_match, 16, bloom_bits));
-      }
-      DestroyAndReopen(&options);
+  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
+       huge_page_tlb_size += 2 * 1024 * 1024) {
+    for (int bloom_bits = 0; bloom_bits <= 117; bloom_bits += 117) {
+      for (int total_order = 0; total_order <= 1; total_order++) {
+        bool expect_bloom_not_match = false;
+        Options options = CurrentOptions();
+        options.create_if_missing = true;
+        // Set only one bucket to force bucket conflict.
+        // Test index interval for the same prefix to be 1, 2 and 4
+        if (total_order) {
+          options.prefix_extractor = nullptr;
+          options.table_factory.reset(
+              new TestPlainTableFactory(&expect_bloom_not_match, 16, bloom_bits,
+                                        0, 2, huge_page_tlb_size));
+        } else {
+          options.table_factory.reset(
+              new TestPlainTableFactory(&expect_bloom_not_match, 16, bloom_bits,
+                                        0.75, 16, huge_page_tlb_size));
+        }
+        DestroyAndReopen(&options);
 
-      ASSERT_OK(Put("1000000000foo002", "v_2"));
-      ASSERT_OK(Put("0000000000000bar", "random"));
-      ASSERT_OK(Put("1000000000foo001", "v1"));
-      ASSERT_OK(Put("3000000000000bar", "bar_v"));
-      ASSERT_OK(Put("1000000000foo003", "v__3"));
-      ASSERT_OK(Put("1000000000foo004", "v__4"));
-      ASSERT_OK(Put("1000000000foo005", "v__5"));
-      ASSERT_OK(Put("1000000000foo007", "v__7"));
-      ASSERT_OK(Put("1000000000foo008", "v__8"));
-      dbfull()->TEST_FlushMemTable();
-      ASSERT_EQ("v1", Get("1000000000foo001"));
-      ASSERT_EQ("v__3", Get("1000000000foo003"));
-      Iterator* iter = dbfull()->NewIterator(ro_);
-      iter->Seek("1000000000foo000");
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo001", iter->key().ToString());
-      ASSERT_EQ("v1", iter->value().ToString());
+        ASSERT_OK(Put("1000000000foo002", "v_2"));
+        ASSERT_OK(Put("0000000000000bar", "random"));
+        ASSERT_OK(Put("1000000000foo001", "v1"));
+        ASSERT_OK(Put("3000000000000bar", "bar_v"));
+        ASSERT_OK(Put("1000000000foo003", "v__3"));
+        ASSERT_OK(Put("1000000000foo004", "v__4"));
+        ASSERT_OK(Put("1000000000foo005", "v__5"));
+        ASSERT_OK(Put("1000000000foo007", "v__7"));
+        ASSERT_OK(Put("1000000000foo008", "v__8"));
+        dbfull()->TEST_FlushMemTable();
+        ASSERT_EQ("v1", Get("1000000000foo001"));
+        ASSERT_EQ("v__3", Get("1000000000foo003"));
+        Iterator* iter = dbfull()->NewIterator(ReadOptions());
+        iter->Seek("1000000000foo000");
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo001", iter->key().ToString());
+        ASSERT_EQ("v1", iter->value().ToString());
 
-      iter->Next();
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo002", iter->key().ToString());
-      ASSERT_EQ("v_2", iter->value().ToString());
+        iter->Next();
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo002", iter->key().ToString());
+        ASSERT_EQ("v_2", iter->value().ToString());
 
-      iter->Next();
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo003", iter->key().ToString());
-      ASSERT_EQ("v__3", iter->value().ToString());
+        iter->Next();
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo003", iter->key().ToString());
+        ASSERT_EQ("v__3", iter->value().ToString());
 
-      iter->Next();
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo004", iter->key().ToString());
-      ASSERT_EQ("v__4", iter->value().ToString());
+        iter->Next();
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo004", iter->key().ToString());
+        ASSERT_EQ("v__4", iter->value().ToString());
 
-      iter->Seek("3000000000000bar");
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("3000000000000bar", iter->key().ToString());
-      ASSERT_EQ("bar_v", iter->value().ToString());
-
-      iter->Seek("1000000000foo000");
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo001", iter->key().ToString());
-      ASSERT_EQ("v1", iter->value().ToString());
-
-      iter->Seek("1000000000foo005");
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo005", iter->key().ToString());
-      ASSERT_EQ("v__5", iter->value().ToString());
-
-      iter->Seek("1000000000foo006");
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo007", iter->key().ToString());
-      ASSERT_EQ("v__7", iter->value().ToString());
-
-      iter->Seek("1000000000foo008");
-      ASSERT_TRUE(iter->Valid());
-      ASSERT_EQ("1000000000foo008", iter->key().ToString());
-      ASSERT_EQ("v__8", iter->value().ToString());
-
-      if (total_order == 0) {
-        iter->Seek("1000000000foo009");
+        iter->Seek("3000000000000bar");
         ASSERT_TRUE(iter->Valid());
         ASSERT_EQ("3000000000000bar", iter->key().ToString());
-      }
+        ASSERT_EQ("bar_v", iter->value().ToString());
 
-      // Test Bloom Filter
-      if (bloom_bits > 0) {
-        if (!total_order) {
-          // Neither key nor value should exist.
-          expect_bloom_not_match = true;
-          iter->Seek("2not000000000bar");
-          ASSERT_TRUE(!iter->Valid());
-          ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
-          expect_bloom_not_match = false;
-        } else {
-          expect_bloom_not_match = true;
-          ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
-          expect_bloom_not_match = false;
+        iter->Seek("1000000000foo000");
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo001", iter->key().ToString());
+        ASSERT_EQ("v1", iter->value().ToString());
+
+        iter->Seek("1000000000foo005");
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo005", iter->key().ToString());
+        ASSERT_EQ("v__5", iter->value().ToString());
+
+        iter->Seek("1000000000foo006");
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo007", iter->key().ToString());
+        ASSERT_EQ("v__7", iter->value().ToString());
+
+        iter->Seek("1000000000foo008");
+        ASSERT_TRUE(iter->Valid());
+        ASSERT_EQ("1000000000foo008", iter->key().ToString());
+        ASSERT_EQ("v__8", iter->value().ToString());
+
+        if (total_order == 0) {
+          iter->Seek("1000000000foo009");
+          ASSERT_TRUE(iter->Valid());
+          ASSERT_EQ("3000000000000bar", iter->key().ToString());
         }
-      }
 
-      delete iter;
+        // Test Bloom Filter
+        if (bloom_bits > 0) {
+          if (!total_order) {
+            // Neither key nor value should exist.
+            expect_bloom_not_match = true;
+            iter->Seek("2not000000000bar");
+            ASSERT_TRUE(!iter->Valid());
+            ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
+            expect_bloom_not_match = false;
+          } else {
+            expect_bloom_not_match = true;
+            ASSERT_EQ("NOT_FOUND", Get("2not000000000bar"));
+            expect_bloom_not_match = false;
+          }
+        }
+
+        delete iter;
+      }
     }
   }
 }
 
+namespace {
+std::string MakeLongKey(size_t length, char c) {
+  return std::string(length, c);
+}
+}  // namespace
+
+TEST(PlainTableDBTest, IteratorLargeKeys) {
+  Options options = CurrentOptions();
+  options.table_factory.reset(NewTotalOrderPlainTableFactory(0, 0, 16));
+  options.create_if_missing = true;
+  options.prefix_extractor.reset();
+  DestroyAndReopen(&options);
+
+  std::string key_list[] = {
+      MakeLongKey(30, '0'),
+      MakeLongKey(16, '1'),
+      MakeLongKey(32, '2'),
+      MakeLongKey(60, '3'),
+      MakeLongKey(90, '4'),
+      MakeLongKey(50, '5'),
+      MakeLongKey(26, '6')
+  };
+
+  for (size_t i = 0; i < 7; i++) {
+    ASSERT_OK(Put(key_list[i], std::to_string(i)));
+  }
+
+  dbfull()->TEST_FlushMemTable();
+
+  Iterator* iter = dbfull()->NewIterator(ReadOptions());
+  iter->Seek(key_list[0]);
+
+  for (size_t i = 0; i < 7; i++) {
+    ASSERT_TRUE(iter->Valid());
+    ASSERT_EQ(key_list[i], iter->key().ToString());
+    ASSERT_EQ(std::to_string(i), iter->value().ToString());
+    iter->Next();
+  }
+
+  ASSERT_TRUE(!iter->Valid());
+
+  delete iter;
+}
+
 // A test comparator which compare two strings in this way:
 // (1) first compare prefix of 8 bytes in alphabet order,
 // (2) if two strings share the same prefix, sort the other part of the string
@@ -478,7 +543,7 @@ TEST(PlainTableDBTest, IteratorReverseSuffixComparator) {
   dbfull()->TEST_FlushMemTable();
   ASSERT_EQ("v1", Get("1000000000foo001"));
   ASSERT_EQ("v__3", Get("1000000000foo003"));
-  Iterator* iter = dbfull()->NewIterator(ro_);
+  Iterator* iter = dbfull()->NewIterator(ReadOptions());
   iter->Seek("1000000000foo009");
   ASSERT_TRUE(iter->Valid());
   ASSERT_EQ("1000000000foo008", iter->key().ToString());
@@ -527,165 +592,173 @@ TEST(PlainTableDBTest, IteratorReverseSuffixComparator) {
 }
 
 TEST(PlainTableDBTest, HashBucketConflict) {
-  for (unsigned char i = 1; i <= 3; i++) {
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    // Set only one bucket to force bucket conflict.
-    // Test index interval for the same prefix to be 1, 2 and 4
-    options.table_factory.reset(NewTotalOrderPlainTableFactory(16, 0, 2 ^ i));
-    DestroyAndReopen(&options);
-    ASSERT_OK(Put("5000000000000fo0", "v1"));
-    ASSERT_OK(Put("5000000000000fo1", "v2"));
-    ASSERT_OK(Put("5000000000000fo2", "v"));
-    ASSERT_OK(Put("2000000000000fo0", "v3"));
-    ASSERT_OK(Put("2000000000000fo1", "v4"));
-    ASSERT_OK(Put("2000000000000fo2", "v"));
-    ASSERT_OK(Put("2000000000000fo3", "v"));
+  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
+       huge_page_tlb_size += 2 * 1024 * 1024) {
+    for (unsigned char i = 1; i <= 3; i++) {
+      Options options = CurrentOptions();
+      options.create_if_missing = true;
+      // Set only one bucket to force bucket conflict.
+      // Test index interval for the same prefix to be 1, 2 and 4
+      options.table_factory.reset(
+          NewTotalOrderPlainTableFactory(16, 0, 2 ^ i, huge_page_tlb_size));
+      DestroyAndReopen(&options);
+      ASSERT_OK(Put("5000000000000fo0", "v1"));
+      ASSERT_OK(Put("5000000000000fo1", "v2"));
+      ASSERT_OK(Put("5000000000000fo2", "v"));
+      ASSERT_OK(Put("2000000000000fo0", "v3"));
+      ASSERT_OK(Put("2000000000000fo1", "v4"));
+      ASSERT_OK(Put("2000000000000fo2", "v"));
+      ASSERT_OK(Put("2000000000000fo3", "v"));
 
-    dbfull()->TEST_FlushMemTable();
+      dbfull()->TEST_FlushMemTable();
 
-    ASSERT_EQ("v1", Get("5000000000000fo0"));
-    ASSERT_EQ("v2", Get("5000000000000fo1"));
-    ASSERT_EQ("v3", Get("2000000000000fo0"));
-    ASSERT_EQ("v4", Get("2000000000000fo1"));
+      ASSERT_EQ("v1", Get("5000000000000fo0"));
+      ASSERT_EQ("v2", Get("5000000000000fo1"));
+      ASSERT_EQ("v3", Get("2000000000000fo0"));
+      ASSERT_EQ("v4", Get("2000000000000fo1"));
 
-    ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
-    ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
-    ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
-    ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
+      ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
+      ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
+      ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
+      ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
 
-    ReadOptions ro;
-    Iterator* iter = dbfull()->NewIterator(ro);
+      ReadOptions ro;
+      Iterator* iter = dbfull()->NewIterator(ro);
 
-    iter->Seek("5000000000000fo0");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo0", iter->key().ToString());
-    iter->Next();
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo1", iter->key().ToString());
+      iter->Seek("5000000000000fo0");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo0", iter->key().ToString());
+      iter->Next();
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo1", iter->key().ToString());
 
-    iter->Seek("5000000000000fo1");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo1", iter->key().ToString());
+      iter->Seek("5000000000000fo1");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo1", iter->key().ToString());
 
-    iter->Seek("2000000000000fo0");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo0", iter->key().ToString());
-    iter->Next();
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo1", iter->key().ToString());
+      iter->Seek("2000000000000fo0");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo0", iter->key().ToString());
+      iter->Next();
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo1", iter->key().ToString());
 
-    iter->Seek("2000000000000fo1");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo1", iter->key().ToString());
+      iter->Seek("2000000000000fo1");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo1", iter->key().ToString());
 
-    iter->Seek("2000000000000bar");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo0", iter->key().ToString());
+      iter->Seek("2000000000000bar");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo0", iter->key().ToString());
 
-    iter->Seek("5000000000000bar");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo0", iter->key().ToString());
+      iter->Seek("5000000000000bar");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo0", iter->key().ToString());
 
-    iter->Seek("2000000000000fo8");
-    ASSERT_TRUE(!iter->Valid() ||
-                options.comparator->Compare(iter->key(), "20000001") > 0);
+      iter->Seek("2000000000000fo8");
+      ASSERT_TRUE(!iter->Valid() ||
+                  options.comparator->Compare(iter->key(), "20000001") > 0);
 
-    iter->Seek("5000000000000fo8");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("5000000000000fo8");
+      ASSERT_TRUE(!iter->Valid());
 
-    iter->Seek("1000000000000fo2");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("1000000000000fo2");
+      ASSERT_TRUE(!iter->Valid());
 
-    iter->Seek("3000000000000fo2");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("3000000000000fo2");
+      ASSERT_TRUE(!iter->Valid());
 
-    iter->Seek("8000000000000fo2");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("8000000000000fo2");
+      ASSERT_TRUE(!iter->Valid());
 
-    delete iter;
+      delete iter;
+    }
   }
 }
 
 TEST(PlainTableDBTest, HashBucketConflictReverseSuffixComparator) {
-  for (unsigned char i = 1; i <= 3; i++) {
-    Options options = CurrentOptions();
-    options.create_if_missing = true;
-    SimpleSuffixReverseComparator comp;
-    options.comparator = &comp;
-    // Set only one bucket to force bucket conflict.
-    // Test index interval for the same prefix to be 1, 2 and 4
-    options.table_factory.reset(NewTotalOrderPlainTableFactory(16, 0, 2 ^ i));
-    DestroyAndReopen(&options);
-    ASSERT_OK(Put("5000000000000fo0", "v1"));
-    ASSERT_OK(Put("5000000000000fo1", "v2"));
-    ASSERT_OK(Put("5000000000000fo2", "v"));
-    ASSERT_OK(Put("2000000000000fo0", "v3"));
-    ASSERT_OK(Put("2000000000000fo1", "v4"));
-    ASSERT_OK(Put("2000000000000fo2", "v"));
-    ASSERT_OK(Put("2000000000000fo3", "v"));
+  for (size_t huge_page_tlb_size = 0; huge_page_tlb_size <= 2 * 1024 * 1024;
+       huge_page_tlb_size += 2 * 1024 * 1024) {
+    for (unsigned char i = 1; i <= 3; i++) {
+      Options options = CurrentOptions();
+      options.create_if_missing = true;
+      SimpleSuffixReverseComparator comp;
+      options.comparator = &comp;
+      // Set only one bucket to force bucket conflict.
+      // Test index interval for the same prefix to be 1, 2 and 4
+      options.table_factory.reset(
+          NewTotalOrderPlainTableFactory(16, 0, 2 ^ i, huge_page_tlb_size));
+      DestroyAndReopen(&options);
+      ASSERT_OK(Put("5000000000000fo0", "v1"));
+      ASSERT_OK(Put("5000000000000fo1", "v2"));
+      ASSERT_OK(Put("5000000000000fo2", "v"));
+      ASSERT_OK(Put("2000000000000fo0", "v3"));
+      ASSERT_OK(Put("2000000000000fo1", "v4"));
+      ASSERT_OK(Put("2000000000000fo2", "v"));
+      ASSERT_OK(Put("2000000000000fo3", "v"));
 
-    dbfull()->TEST_FlushMemTable();
+      dbfull()->TEST_FlushMemTable();
 
-    ASSERT_EQ("v1", Get("5000000000000fo0"));
-    ASSERT_EQ("v2", Get("5000000000000fo1"));
-    ASSERT_EQ("v3", Get("2000000000000fo0"));
-    ASSERT_EQ("v4", Get("2000000000000fo1"));
+      ASSERT_EQ("v1", Get("5000000000000fo0"));
+      ASSERT_EQ("v2", Get("5000000000000fo1"));
+      ASSERT_EQ("v3", Get("2000000000000fo0"));
+      ASSERT_EQ("v4", Get("2000000000000fo1"));
 
-    ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
-    ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
-    ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
-    ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
+      ASSERT_EQ("NOT_FOUND", Get("5000000000000bar"));
+      ASSERT_EQ("NOT_FOUND", Get("2000000000000bar"));
+      ASSERT_EQ("NOT_FOUND", Get("5000000000000fo8"));
+      ASSERT_EQ("NOT_FOUND", Get("2000000000000fo8"));
 
-    ReadOptions ro;
-    Iterator* iter = dbfull()->NewIterator(ro);
+      ReadOptions ro;
+      Iterator* iter = dbfull()->NewIterator(ro);
 
-    iter->Seek("5000000000000fo1");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo1", iter->key().ToString());
-    iter->Next();
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo0", iter->key().ToString());
+      iter->Seek("5000000000000fo1");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo1", iter->key().ToString());
+      iter->Next();
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo0", iter->key().ToString());
 
-    iter->Seek("5000000000000fo1");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo1", iter->key().ToString());
+      iter->Seek("5000000000000fo1");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo1", iter->key().ToString());
 
-    iter->Seek("2000000000000fo1");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo1", iter->key().ToString());
-    iter->Next();
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo0", iter->key().ToString());
+      iter->Seek("2000000000000fo1");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo1", iter->key().ToString());
+      iter->Next();
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo0", iter->key().ToString());
 
-    iter->Seek("2000000000000fo1");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo1", iter->key().ToString());
+      iter->Seek("2000000000000fo1");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo1", iter->key().ToString());
 
-    iter->Seek("2000000000000var");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("2000000000000fo3", iter->key().ToString());
+      iter->Seek("2000000000000var");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("2000000000000fo3", iter->key().ToString());
 
-    iter->Seek("5000000000000var");
-    ASSERT_TRUE(iter->Valid());
-    ASSERT_EQ("5000000000000fo2", iter->key().ToString());
+      iter->Seek("5000000000000var");
+      ASSERT_TRUE(iter->Valid());
+      ASSERT_EQ("5000000000000fo2", iter->key().ToString());
 
-    std::string seek_key = "2000000000000bar";
-    iter->Seek(seek_key);
-    ASSERT_TRUE(!iter->Valid() ||
-                options.prefix_extractor->Transform(iter->key()) !=
-                    options.prefix_extractor->Transform(seek_key));
+      std::string seek_key = "2000000000000bar";
+      iter->Seek(seek_key);
+      ASSERT_TRUE(!iter->Valid() ||
+                  options.prefix_extractor->Transform(iter->key()) !=
+                      options.prefix_extractor->Transform(seek_key));
 
-    iter->Seek("1000000000000fo2");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("1000000000000fo2");
+      ASSERT_TRUE(!iter->Valid());
 
-    iter->Seek("3000000000000fo2");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("3000000000000fo2");
+      ASSERT_TRUE(!iter->Valid());
 
-    iter->Seek("8000000000000fo2");
-    ASSERT_TRUE(!iter->Valid());
+      iter->Seek("8000000000000fo2");
+      ASSERT_TRUE(!iter->Valid());
 
-    delete iter;
+      delete iter;
+    }
   }
 }
 
@@ -709,7 +782,7 @@ TEST(PlainTableDBTest, NonExistingKeyToNonEmptyBucket) {
   ASSERT_EQ("NOT_FOUND", Get("8000000000000bar"));
   ASSERT_EQ("NOT_FOUND", Get("1000000000000bar"));
 
-  Iterator* iter = dbfull()->NewIterator(ro_);
+  Iterator* iter = dbfull()->NewIterator(ReadOptions());
 
   iter->Seek("5000000000000bar");
   ASSERT_TRUE(iter->Valid());

db/prefix_filter_iterator.h

@@ -1,75 +0,0 @@
-//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under the BSD-style license found in the
-//  LICENSE file in the root directory of this source tree. An additional grant
-//  of patent rights can be found in the PATENTS file in the same directory.
-//
-// Wrap an underlying iterator, but exclude any results not starting
-// with a given prefix.  Seeking to keys not beginning with the prefix
-// is invalid, and SeekToLast is not implemented (that would be
-// non-trivial), but otherwise this iterator will behave just like the
-// underlying iterator would if there happened to be no non-matching
-// keys in the dataset.
-
-#pragma once
-#include "rocksdb/iterator.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/slice_transform.h"
-
-namespace rocksdb {
-
-class PrefixFilterIterator : public Iterator {
- private:
-  Iterator* iter_;
-  const Slice &prefix_;
-  const SliceTransform *prefix_extractor_;
-  Status status_;
-
- public:
-  PrefixFilterIterator(Iterator* iter,
-                       const Slice &prefix,
-                       const SliceTransform* prefix_extractor)
-                             : iter_(iter), prefix_(prefix),
-                               prefix_extractor_(prefix_extractor),
-                               status_(Status::OK()) {
-    if (prefix_extractor == nullptr) {
-      status_ = Status::InvalidArgument("A prefix filter may not be used "
-                                        "unless a function is also defined "
-                                        "for extracting prefixes");
-    } else if (!prefix_extractor_->InRange(prefix)) {
-      status_ = Status::InvalidArgument("Must provide a slice for prefix which"
-                                        "is a prefix for some key");
-    }
-  }
-  ~PrefixFilterIterator() {
-    delete iter_;
-  }
-  Slice key() const { return iter_->key(); }
-  Slice value() const { return iter_->value(); }
-  Status status() const {
-    if (!status_.ok()) {
-      return status_;
-    }
-    return iter_->status();
-  }
-  void Next() { iter_->Next(); }
-  void Prev() { iter_->Prev(); }
-  void Seek(const Slice& k) {
-    if (prefix_extractor_->Transform(k) == prefix_) {
-      iter_->Seek(k);
-    } else {
-      status_ = Status::InvalidArgument("Seek must begin with target prefix");
-    }
-  }
-  void SeekToFirst() {
-    Seek(prefix_);
-  }
-  void SeekToLast() {
-    status_ = Status::NotSupported("SeekToLast is incompatible with prefixes");
-  }
-  bool Valid() const {
-    return (status_.ok() && iter_->Valid() &&
-            prefix_extractor_->Transform(iter_->key()) == prefix_);
-  }
-};
-
-}  // namespace rocksdb

db/prefix_test.cc

@@ -17,7 +17,6 @@
 #include "util/stop_watch.h"
 #include "util/testharness.h"
 
-DEFINE_bool(use_prefix_hash_memtable, true, "");
 DEFINE_bool(trigger_deadlock, false,
             "issue delete in range scan to trigger PrefixHashMap deadlock");
 DEFINE_uint64(bucket_count, 100000, "number of buckets");
@@ -31,6 +30,7 @@ DEFINE_int64(min_write_buffer_number_to_merge, 1, "");
 DEFINE_int32(skiplist_height, 4, "");
 DEFINE_int32(memtable_prefix_bloom_bits, 10000000, "");
 DEFINE_int32(memtable_prefix_bloom_probes, 10, "");
+DEFINE_int32(memtable_prefix_bloom_huge_page_tlb_size, 2 * 1024 * 1024, "");
 DEFINE_int32(value_size, 40, "");
 
 // Path to the database on file system
@@ -104,6 +104,7 @@ class TestKeyComparator : public Comparator {
 
 };
 
+namespace {
 void PutKey(DB* db, WriteOptions write_options, uint64_t prefix,
             uint64_t suffix, const Slice& value) {
   TestKey test_key(prefix, suffix);
@@ -133,6 +134,7 @@ std::string Get(DB* db, const ReadOptions& read_options, uint64_t prefix,
   }
   return result;
 }
+}  // namespace
 
 class PrefixTest {
  public:
@@ -147,6 +149,8 @@ class PrefixTest {
 
     options.memtable_prefix_bloom_bits = FLAGS_memtable_prefix_bloom_bits;
     options.memtable_prefix_bloom_probes = FLAGS_memtable_prefix_bloom_probes;
+    options.memtable_prefix_bloom_huge_page_tlb_size =
+        FLAGS_memtable_prefix_bloom_huge_page_tlb_size;
 
     Status s = DB::Open(options, kDbName,  &db);
     ASSERT_OK(s);
@@ -171,6 +175,10 @@ class PrefixTest {
           options.memtable_factory.reset(
               NewHashLinkListRepFactory(bucket_count));
           return true;
+        case kHashLinkListHugePageTlb:
+          options.memtable_factory.reset(
+              NewHashLinkListRepFactory(bucket_count, 2 * 1024 * 1024));
+          return true;
         default:
           return false;
       }
@@ -189,6 +197,7 @@ class PrefixTest {
     kBegin,
     kHashSkipList,
     kHashLinkList,
+    kHashLinkListHugePageTlb,
     kEnd
   };
   int option_config_;
@@ -206,7 +215,6 @@ TEST(PrefixTest, TestResult) {
       auto db = OpenDb();
       WriteOptions write_options;
       ReadOptions read_options;
-      read_options.prefix_seek = true;
 
       // 1. Insert one row.
       Slice v16("v16");
@@ -369,43 +377,6 @@ TEST(PrefixTest, TestResult) {
   }
 }
 
-TEST(PrefixTest, FullIterator) {
-  while (NextOptions(1000000)) {
-    DestroyDB(kDbName, Options());
-    auto db = OpenDb();
-    WriteOptions write_options;
-
-    std::vector<uint64_t> prefixes;
-    for (uint64_t i = 0; i < 100; ++i) {
-      prefixes.push_back(i);
-    }
-    std::random_shuffle(prefixes.begin(), prefixes.end());
-
-    for (auto prefix : prefixes) {
-      for (uint64_t i = 0; i < 200; ++i) {
-        TestKey test_key(prefix, i);
-        Slice key = TestKeyToSlice(test_key);
-        ASSERT_OK(db->Put(write_options, key, Slice("0")));
-      }
-    }
-
-    auto func = [](void* db_void) {
-      auto db = reinterpret_cast<DB*>(db_void);
-      std::unique_ptr<Iterator> iter(db->NewIterator(ReadOptions()));
-      iter->SeekToFirst();
-      for (int i = 0; i < 3; ++i) {
-        iter->Next();
-      }
-    };
-
-    auto env = Env::Default();
-    for (int i = 0; i < 16; ++i) {
-      env->StartThread(func, reinterpret_cast<void*>(db.get()));
-    }
-    env->WaitForJoin();
-  }
-}
-
 TEST(PrefixTest, DynamicPrefixIterator) {
   while (NextOptions(FLAGS_bucket_count)) {
     std::cout << "*** Mem table: " << options.memtable_factory->Name()
@@ -450,9 +421,6 @@ TEST(PrefixTest, DynamicPrefixIterator) {
     HistogramImpl hist_seek_time;
     HistogramImpl hist_seek_comparison;
 
-    if (FLAGS_use_prefix_hash_memtable) {
-      read_options.prefix_seek = true;
-    }
     std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
 
     for (auto prefix : prefixes) {
@@ -462,14 +430,15 @@ TEST(PrefixTest, DynamicPrefixIterator) {
 
       perf_context.Reset();
       StopWatchNano timer(Env::Default(), true);
+      auto key_prefix = options.prefix_extractor->Transform(key);
       uint64_t total_keys = 0;
-      for (iter->Seek(key); iter->Valid(); iter->Next()) {
+      for (iter->Seek(key);
+           iter->Valid() && iter->key().starts_with(key_prefix);
+           iter->Next()) {
         if (FLAGS_trigger_deadlock) {
           std::cout << "Behold the deadlock!\n";
           db->Delete(write_options, iter->key());
         }
-        auto test_key = SliceToTestKey(iter->key());
-        if (test_key->prefix != prefix) break;
         total_keys++;
       }
       hist_seek_time.Add(timer.ElapsedNanos());
@@ -507,116 +476,6 @@ TEST(PrefixTest, DynamicPrefixIterator) {
   }
 }
 
-TEST(PrefixTest, PrefixHash) {
-  while (NextOptions(FLAGS_bucket_count)) {
-    std::cout << "*** Mem table: " << options.memtable_factory->Name()
-        << std::endl;
-    DestroyDB(kDbName, Options());
-    auto db = OpenDb();
-    WriteOptions write_options;
-    ReadOptions read_options;
-
-    std::vector<uint64_t> prefixes;
-    for (uint64_t i = 0; i < FLAGS_total_prefixes; ++i) {
-      prefixes.push_back(i);
-    }
-
-    if (FLAGS_random_prefix) {
-      std::random_shuffle(prefixes.begin(), prefixes.end());
-    }
-
-    // insert x random prefix, each with y continuous element.
-    HistogramImpl hist_put_time;
-    HistogramImpl hist_put_comparison;
-
-    for (auto prefix : prefixes) {
-       for (uint64_t sorted = 0; sorted < FLAGS_items_per_prefix; sorted++) {
-        TestKey test_key(prefix, sorted);
-
-        Slice key = TestKeyToSlice(test_key);
-        std::string value = "v" + std::to_string(sorted);
-
-        perf_context.Reset();
-        StopWatchNano timer(Env::Default(), true);
-        ASSERT_OK(db->Put(write_options, key, value));
-        hist_put_time.Add(timer.ElapsedNanos());
-        hist_put_comparison.Add(perf_context.user_key_comparison_count);
-      }
-    }
-
-    std::cout << "Put key comparison: \n" << hist_put_comparison.ToString()
-              << "Put time: \n" << hist_put_time.ToString();
-
-
-    // test seek existing keys
-    HistogramImpl hist_seek_time;
-    HistogramImpl hist_seek_comparison;
-
-    for (auto prefix : prefixes) {
-      TestKey test_key(prefix, 0);
-      Slice key = TestKeyToSlice(test_key);
-      std::string value = "v" + std::to_string(0);
-
-      Slice key_prefix;
-      if (FLAGS_use_prefix_hash_memtable) {
-        key_prefix = options.prefix_extractor->Transform(key);
-        read_options.prefix = &key_prefix;
-      }
-      std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
-
-      perf_context.Reset();
-      StopWatchNano timer(Env::Default(), true);
-      uint64_t total_keys = 0;
-      for (iter->Seek(key); iter->Valid(); iter->Next()) {
-        if (FLAGS_trigger_deadlock) {
-          std::cout << "Behold the deadlock!\n";
-          db->Delete(write_options, iter->key());
-        }
-        auto test_key = SliceToTestKey(iter->key());
-        if (test_key->prefix != prefix) break;
-        total_keys++;
-      }
-      hist_seek_time.Add(timer.ElapsedNanos());
-      hist_seek_comparison.Add(perf_context.user_key_comparison_count);
-      ASSERT_EQ(total_keys, FLAGS_items_per_prefix);
-    }
-
-    std::cout << "Seek key comparison: \n"
-              << hist_seek_comparison.ToString()
-              << "Seek time: \n"
-              << hist_seek_time.ToString();
-
-    // test non-existing keys
-    HistogramImpl hist_no_seek_time;
-    HistogramImpl hist_no_seek_comparison;
-
-    for (auto prefix = FLAGS_total_prefixes;
-         prefix < FLAGS_total_prefixes + 100;
-         prefix++) {
-      TestKey test_key(prefix, 0);
-      Slice key = TestKeyToSlice(test_key);
-
-      if (FLAGS_use_prefix_hash_memtable) {
-        Slice key_prefix = options.prefix_extractor->Transform(key);
-        read_options.prefix = &key_prefix;
-      }
-      std::unique_ptr<Iterator> iter(db->NewIterator(read_options));
-
-      perf_context.Reset();
-      StopWatchNano timer(Env::Default(), true);
-      iter->Seek(key);
-      hist_no_seek_time.Add(timer.ElapsedNanos());
-      hist_no_seek_comparison.Add(perf_context.user_key_comparison_count);
-      ASSERT_TRUE(!iter->Valid());
-    }
-
-    std::cout << "non-existing Seek key comparison: \n"
-              << hist_no_seek_comparison.ToString()
-              << "non-existing Seek time: \n"
-              << hist_no_seek_time.ToString();
-  }
-}
-
 }
 
 int main(int argc, char** argv) {

db/repair.cc

@@ -29,6 +29,8 @@
 //   Store per-table metadata (smallest, largest, largest-seq#, ...)
 //   in the table's meta section to speed up ScanTable.
 
+#ifndef ROCKSDB_LITE
+
 #include "db/builder.h"
 #include "db/db_impl.h"
 #include "db/dbformat.h"
@@ -55,14 +57,20 @@ class Repairer {
         icmp_(options.comparator),
         ipolicy_(options.filter_policy),
         options_(SanitizeOptions(dbname, &icmp_, &ipolicy_, options)),
+        raw_table_cache_(
+            // TableCache can be small since we expect each table to be opened
+            // once.
+            NewLRUCache(10, options_.table_cache_numshardbits,
+                        options_.table_cache_remove_scan_count_limit)),
         next_file_number_(1) {
-    // TableCache can be small since we expect each table to be opened once.
-    table_cache_ = new TableCache(dbname_, &options_, storage_options_, 10);
+    table_cache_ = new TableCache(dbname_, &options_, storage_options_,
+                                  raw_table_cache_.get());
     edit_ = new VersionEdit();
   }
 
   ~Repairer() {
     delete table_cache_;
+    raw_table_cache_.reset();
     delete edit_;
   }
 
@@ -102,6 +110,7 @@ class Repairer {
   InternalKeyComparator const icmp_;
   InternalFilterPolicy const ipolicy_;
   Options const options_;
+  std::shared_ptr<Cache> raw_table_cache_;
   TableCache* table_cache_;
   VersionEdit* edit_;
 
@@ -197,6 +206,7 @@ class Repairer {
     Slice record;
     WriteBatch batch;
     MemTable* mem = new MemTable(icmp_, options_);
+    auto cf_mems_default = new ColumnFamilyMemTablesDefault(mem, &options_);
     mem->Ref();
     int counter = 0;
     while (reader.ReadRecord(&record, &scratch)) {
@@ -206,7 +216,7 @@ class Repairer {
         continue;
       }
       WriteBatchInternal::SetContents(&batch, record);
-      status = WriteBatchInternal::InsertInto(&batch, mem, &options_);
+      status = WriteBatchInternal::InsertInto(&batch, cf_mems_default);
       if (status.ok()) {
         counter += WriteBatchInternal::Count(&batch);
       } else {
@@ -221,11 +231,13 @@ class Repairer {
     // since ExtractMetaData() will also generate edits.
     FileMetaData meta;
     meta.number = next_file_number_++;
-    Iterator* iter = mem->NewIterator();
+    ReadOptions ro;
+    Iterator* iter = mem->NewIterator(ro, true /* enforce_total_order */);
     status = BuildTable(dbname_, env_, options_, storage_options_, table_cache_,
                         iter, &meta, icmp_, 0, 0, kNoCompression);
     delete iter;
     delete mem->Unref();
+    delete cf_mems_default;
     mem = nullptr;
     if (status.ok()) {
       if (meta.file_size > 0) {
@@ -387,3 +399,5 @@ Status RepairDB(const std::string& dbname, const Options& options) {
 }
 
 }  // namespace rocksdb
+
+#endif  // ROCKSDB_LITE

db/simple_table_db_test.cc

@@ -83,8 +83,6 @@ public:
                      unique_ptr<RandomAccessFile> && file, uint64_t file_size,
                      unique_ptr<TableReader>* table_reader);
 
-  bool PrefixMayMatch(const Slice& internal_prefix) override;
-
   Iterator* NewIterator(const ReadOptions&) override;
 
   Status Get(const ReadOptions&, const Slice& key, void* arg,
@@ -220,10 +218,6 @@ std::shared_ptr<const TableProperties> SimpleTableReader::GetTableProperties()
   return rep_->table_properties;
 }
 
-bool SimpleTableReader::PrefixMayMatch(const Slice& internal_prefix) {
-  return true;
-}
-
 Iterator* SimpleTableReader::NewIterator(const ReadOptions& options) {
   return new SimpleTableIterator(this);
 }

db/table_cache.cc

@@ -35,18 +35,13 @@ static Slice GetSliceForFileNumber(uint64_t* file_number) {
                sizeof(*file_number));
 }
 
-TableCache::TableCache(const std::string& dbname,
-                       const Options* options,
-                       const EnvOptions& storage_options,
-                       int entries)
+TableCache::TableCache(const std::string& dbname, const Options* options,
+                       const EnvOptions& storage_options, Cache* const cache)
     : env_(options->env),
       dbname_(dbname),
       options_(options),
       storage_options_(storage_options),
-      cache_(
-        NewLRUCache(entries, options->table_cache_numshardbits,
-                    options->table_cache_remove_scan_count_limit)) {
-}
+      cache_(cache) {}
 
 TableCache::~TableCache() {
 }
@@ -111,20 +106,21 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
   if (table_reader_ptr != nullptr) {
     *table_reader_ptr = nullptr;
   }
-  Cache::Handle* handle = file_meta.table_reader_handle;
+  TableReader* table_reader = file_meta.table_reader;
+  Cache::Handle* handle = nullptr;
   Status s;
-  if (!handle) {
+  if (table_reader == nullptr) {
     s = FindTable(toptions, icomparator, file_meta.number, file_meta.file_size,
                   &handle, nullptr, options.read_tier == kBlockCacheTier);
-  }
-  if (!s.ok()) {
-    return NewErrorIterator(s);
+    if (!s.ok()) {
+      return NewErrorIterator(s);
+    }
+    table_reader = GetTableReaderFromHandle(handle);
   }
 
-  TableReader* table_reader = GetTableReaderFromHandle(handle);
   Iterator* result = table_reader->NewIterator(options);
-  if (!file_meta.table_reader_handle) {
-    result->RegisterCleanup(&UnrefEntry, cache_.get(), handle);
+  if (handle != nullptr) {
+    result->RegisterCleanup(&UnrefEntry, cache_, handle);
   }
   if (table_reader_ptr != nullptr) {
     *table_reader_ptr = table_reader;
@@ -143,17 +139,20 @@ Status TableCache::Get(const ReadOptions& options,
                        bool (*saver)(void*, const ParsedInternalKey&,
                                      const Slice&, bool),
                        bool* table_io, void (*mark_key_may_exist)(void*)) {
-  Cache::Handle* handle = file_meta.table_reader_handle;
+  TableReader* t = file_meta.table_reader;
   Status s;
-  if (!handle) {
+  Cache::Handle* handle = nullptr;
+  if (!t) {
     s = FindTable(storage_options_, internal_comparator, file_meta.number,
                   file_meta.file_size, &handle, table_io,
                   options.read_tier == kBlockCacheTier);
+    if (s.ok()) {
+      t = GetTableReaderFromHandle(handle);
+    }
   }
   if (s.ok()) {
-    TableReader* t = GetTableReaderFromHandle(handle);
     s = t->Get(options, k, arg, saver, mark_key_may_exist);
-    if (!file_meta.table_reader_handle) {
+    if (handle != nullptr) {
       ReleaseHandle(handle);
     }
   } else if (options.read_tier && s.IsIncomplete()) {
@@ -169,15 +168,16 @@ Status TableCache::GetTableProperties(
     const FileMetaData& file_meta,
     std::shared_ptr<const TableProperties>* properties, bool no_io) {
   Status s;
-  auto table_handle = file_meta.table_reader_handle;
+  auto table_reader = file_meta.table_reader;
   // table already been pre-loaded?
-  if (table_handle) {
-    auto table = GetTableReaderFromHandle(table_handle);
-    *properties = table->GetTableProperties();
+  if (table_reader) {
+    *properties = table_reader->GetTableProperties();
+
     return s;
   }
 
   bool table_io;
+  Cache::Handle* table_handle = nullptr;
   s = FindTable(toptions, internal_comparator, file_meta.number,
                 file_meta.file_size, &table_handle, &table_io, no_io);
   if (!s.ok()) {
@@ -190,34 +190,8 @@ Status TableCache::GetTableProperties(
   return s;
 }
 
-bool TableCache::PrefixMayMatch(const ReadOptions& options,
-                                const InternalKeyComparator& icomparator,
-                                const FileMetaData& file_meta,
-                                const Slice& internal_prefix, bool* table_io) {
-  bool may_match = true;
-  auto table_handle = file_meta.table_reader_handle;
-  if (table_handle == nullptr) {
-    // Need to get table handle from file number
-    Status s = FindTable(storage_options_, icomparator, file_meta.number,
-                         file_meta.file_size, &table_handle, table_io);
-    if (!s.ok()) {
-      return may_match;
-    }
-  }
-
-  auto table = GetTableReaderFromHandle(table_handle);
-  may_match = table->PrefixMayMatch(internal_prefix);
-
-  if (file_meta.table_reader_handle == nullptr) {
-    // Need to release handle if it is generated from here.
-    ReleaseHandle(table_handle);
-  }
-
-  return may_match;
-}
-
-void TableCache::Evict(uint64_t file_number) {
-  cache_->Erase(GetSliceForFileNumber(&file_number));
+void TableCache::Evict(Cache* cache, uint64_t file_number) {
+  cache->Erase(GetSliceForFileNumber(&file_number));
 }
 
 }  // namespace rocksdb

db/table_cache.h

@@ -30,7 +30,7 @@ struct FileMetaData;
 class TableCache {
  public:
   TableCache(const std::string& dbname, const Options* options,
-             const EnvOptions& storage_options, int entries);
+             const EnvOptions& storage_options, Cache* cache);
   ~TableCache();
 
   // Return an iterator for the specified file number (the corresponding
@@ -56,15 +56,8 @@ class TableCache {
                                    const Slice&, bool),
              bool* table_io, void (*mark_key_may_exist)(void*) = nullptr);
 
-  // Determine whether the table may contain the specified prefix.  If
-  // the table index or blooms are not in memory, this may cause an I/O
-  bool PrefixMayMatch(const ReadOptions& options,
-                      const InternalKeyComparator& internal_comparator,
-                      const FileMetaData& file_meta,
-                      const Slice& internal_prefix, bool* table_io);
-
   // Evict any entry for the specified file number
-  void Evict(uint64_t file_number);
+  static void Evict(Cache* cache, uint64_t file_number);
 
   // Find table reader
   Status FindTable(const EnvOptions& toptions,
@@ -95,7 +88,7 @@ class TableCache {
   const std::string dbname_;
   const Options* options_;
   const EnvOptions& storage_options_;
-  std::shared_ptr<Cache> cache_;
+  Cache* const cache_;
 };
 
 }  // namespace rocksdb

db/table_properties_collector_test.cc

@@ -83,6 +83,7 @@ class DumbLogger : public Logger {
 };
 
 // Utilities test functions
+namespace {
 void MakeBuilder(const Options& options,
                  const InternalKeyComparator& internal_comparator,
                  std::unique_ptr<FakeWritableFile>* writable,
@@ -91,6 +92,7 @@ void MakeBuilder(const Options& options,
   builder->reset(options.table_factory->NewTableBuilder(
       options, internal_comparator, writable->get(), options.compression));
 }
+}  // namespace
 
 // Collects keys that starts with "A" in a table.
 class RegularKeysStartWithA: public TablePropertiesCollector {
@@ -126,6 +128,7 @@ class RegularKeysStartWithA: public TablePropertiesCollector {
 
 extern uint64_t kBlockBasedTableMagicNumber;
 extern uint64_t kPlainTableMagicNumber;
+namespace {
 void TestCustomizedTablePropertiesCollector(
     uint64_t magic_number, bool encode_as_internal, const Options& options,
     const InternalKeyComparator& internal_comparator) {
@@ -178,6 +181,7 @@ void TestCustomizedTablePropertiesCollector(
   ASSERT_TRUE(GetVarint32(&key, &starts_with_A));
   ASSERT_EQ(3u, starts_with_A);
 }
+}  // namespace
 
 TEST(TablePropertiesTest, CustomizedTablePropertiesCollector) {
   // Test properties collectors with internal keys or regular keys
@@ -209,6 +213,7 @@ TEST(TablePropertiesTest, CustomizedTablePropertiesCollector) {
                                          ikc);
 }
 
+namespace {
 void TestInternalKeyPropertiesCollector(
     uint64_t magic_number,
     bool sanitized,
@@ -280,6 +285,7 @@ void TestInternalKeyPropertiesCollector(
     ASSERT_EQ(1u, starts_with_A);
   }
 }
+}  // namespace
 
 TEST(TablePropertiesTest, InternalKeyPropertiesCollector) {
   TestInternalKeyPropertiesCollector(

db/tailing_iter.cc

@@ -3,22 +3,36 @@
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
 
+#ifndef ROCKSDB_LITE
 #include "db/tailing_iter.h"
 
 #include <string>
 #include <utility>
+#include <vector>
 #include "db/db_impl.h"
+#include "db/db_iter.h"
+#include "db/column_family.h"
+#include "rocksdb/env.h"
 #include "rocksdb/slice.h"
 #include "rocksdb/slice_transform.h"
+#include "table/merger.h"
 
 namespace rocksdb {
 
-TailingIterator::TailingIterator(DBImpl* db, const ReadOptions& options,
-                                 const Comparator* comparator)
-    : db_(db), options_(options), comparator_(comparator),
-      version_number_(0), current_(nullptr),
+TailingIterator::TailingIterator(Env* const env, DBImpl* db,
+    const ReadOptions& read_options, ColumnFamilyData* cfd)
+    : env_(env),
+      db_(db),
+      read_options_(read_options),
+      cfd_(cfd),
+      super_version_(nullptr),
+      current_(nullptr),
       status_(Status::InvalidArgument("Seek() not called on this iterator")) {}
 
+TailingIterator::~TailingIterator() {
+  Cleanup();
+}
+
 bool TailingIterator::Valid() const {
   return current_ != nullptr;
 }
@@ -46,18 +60,17 @@ void TailingIterator::Seek(const Slice& target) {
   // need to do a seek if 'target' belongs to that interval (i.e. immutable_ is
   // already at the correct position)!
   //
-  // If options.prefix_seek is used and immutable_ is not valid, seek if target
-  // has a different prefix than prev_key.
+  // If prefix seek is used and immutable_ is not valid, seek if target has a
+  // different prefix than prev_key.
   //
   // prev_key_ is updated by Next(). SeekImmutable() sets prev_key_ to
   // 'target' -- in this case, prev_key_ is included in the interval, so
   // prev_inclusive_ has to be set.
 
-  if (!is_prev_set_ ||
-      comparator_->Compare(prev_key_, target) >= !is_prev_inclusive_ ||
-      (immutable_->Valid() &&
-       comparator_->Compare(target, immutable_->key()) > 0) ||
-      (options_.prefix_seek && !IsSamePrefix(target))) {
+  const Comparator* cmp = cfd_->user_comparator();
+  if (!is_prev_set_ || cmp->Compare(prev_key_, target) >= !is_prev_inclusive_ ||
+      (immutable_->Valid() && cmp->Compare(target, immutable_->key()) > 0) ||
+      (cfd_->options()->prefix_extractor != nullptr && !IsSamePrefix(target))) {
     SeekImmutable(target);
   }
 
@@ -119,14 +132,45 @@ void TailingIterator::SeekToLast() {
   status_ = Status::NotSupported("This iterator doesn't support SeekToLast()");
 }
 
+void TailingIterator::Cleanup() {
+  // Release old super version if necessary
+  mutable_.reset();
+  immutable_.reset();
+  if (super_version_ != nullptr && super_version_->Unref()) {
+    DBImpl::DeletionState deletion_state;
+    db_->mutex_.Lock();
+    super_version_->Cleanup();
+    db_->FindObsoleteFiles(deletion_state, false, true);
+    db_->mutex_.Unlock();
+    delete super_version_;
+    if (deletion_state.HaveSomethingToDelete()) {
+      db_->PurgeObsoleteFiles(deletion_state);
+    }
+  }
+}
+
 void TailingIterator::CreateIterators() {
-  std::pair<Iterator*, Iterator*> iters =
-    db_->GetTailingIteratorPair(options_, &version_number_);
+  Cleanup();
+  super_version_= cfd_->GetReferencedSuperVersion(&(db_->mutex_));
 
-  assert(iters.first && iters.second);
+  Iterator* mutable_iter = super_version_->mem->NewIterator(read_options_);
+  // create a DBIter that only uses memtable content; see NewIterator()
+  mutable_.reset(
+      NewDBIterator(env_, *cfd_->options(), cfd_->user_comparator(),
+                    mutable_iter, kMaxSequenceNumber));
+
+  std::vector<Iterator*> list;
+  super_version_->imm->AddIterators(read_options_, &list);
+  super_version_->current->AddIterators(
+      read_options_, *cfd_->soptions(), &list);
+  Iterator* immutable_iter =
+      NewMergingIterator(&cfd_->internal_comparator(), &list[0], list.size());
+
+  // create a DBIter that only uses memtable content; see NewIterator()
+  immutable_.reset(
+      NewDBIterator(env_, *cfd_->options(), cfd_->user_comparator(),
+                    immutable_iter, kMaxSequenceNumber));
 
-  mutable_.reset(iters.first);
-  immutable_.reset(iters.second);
   current_ = nullptr;
   is_prev_set_ = false;
 }
@@ -137,9 +181,10 @@ void TailingIterator::UpdateCurrent() {
   if (mutable_->Valid()) {
     current_ = mutable_.get();
   }
+  const Comparator* cmp = cfd_->user_comparator();
   if (immutable_->Valid() &&
       (current_ == nullptr ||
-       comparator_->Compare(immutable_->key(), current_->key()) < 0)) {
+       cmp->Compare(immutable_->key(), current_->key()) < 0)) {
     current_ = immutable_.get();
   }
 
@@ -150,12 +195,12 @@ void TailingIterator::UpdateCurrent() {
 }
 
 bool TailingIterator::IsCurrentVersion() const {
-  return mutable_ != nullptr && immutable_ != nullptr &&
-    version_number_ == db_->CurrentVersionNumber();
+  return super_version_ != nullptr &&
+         super_version_->version_number == cfd_->GetSuperVersionNumber();
 }
 
 bool TailingIterator::IsSamePrefix(const Slice& target) const {
-  const SliceTransform* extractor = db_->options_.prefix_extractor.get();
+  const SliceTransform* extractor = cfd_->options()->prefix_extractor.get();
 
   assert(extractor);
   assert(is_prev_set_);
@@ -173,3 +218,4 @@ void TailingIterator::SeekImmutable(const Slice& target) {
 }
 
 }  // namespace rocksdb
+#endif  // ROCKSDB_LITE

db/tailing_iter.h

@@ -4,6 +4,8 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 #pragma once
 
+#ifndef ROCKSDB_LITE
+
 #include <string>
 
 #include "rocksdb/db.h"
@@ -13,6 +15,9 @@
 namespace rocksdb {
 
 class DBImpl;
+class Env;
+struct SuperVersion;
+class ColumnFamilyData;
 
 /**
  * TailingIterator is a special type of iterator that doesn't use an (implicit)
@@ -24,9 +29,9 @@ class DBImpl;
  */
 class TailingIterator : public Iterator {
  public:
-  TailingIterator(DBImpl* db, const ReadOptions& options,
-                  const Comparator* comparator);
-  virtual ~TailingIterator() {}
+  TailingIterator(Env* const env, DBImpl* db, const ReadOptions& read_options,
+                  ColumnFamilyData* cfd);
+  virtual ~TailingIterator();
 
   virtual bool Valid() const override;
   virtual void SeekToFirst() override;
@@ -39,10 +44,13 @@ class TailingIterator : public Iterator {
   virtual Status status() const override;
 
  private:
+  void Cleanup();
+
+  Env* const env_;
   DBImpl* const db_;
-  const ReadOptions options_;
-  const Comparator* const comparator_;
-  uint64_t version_number_;
+  const ReadOptions read_options_;
+  ColumnFamilyData* const cfd_;
+  SuperVersion* super_version_;
 
   // TailingIterator merges the contents of the two iterators below (one using
   // mutable memtable contents only, other over SSTs and immutable memtables).
@@ -72,7 +80,7 @@ class TailingIterator : public Iterator {
   bool IsCurrentVersion() const;
 
   // check if SeekImmutable() is needed due to target having a different prefix
-  // than prev_key_ (used when options.prefix_seek is set)
+  // than prev_key_ (used when in prefix seek mode)
   bool IsSamePrefix(const Slice& target) const;
 
   // creates mutable_ and immutable_ iterators and updates version_number_
@@ -86,3 +94,4 @@ class TailingIterator : public Iterator {
 };
 
 }  // namespace rocksdb
+#endif  // ROCKSDB_LITE

db/transaction_log_impl.cc

@@ -2,14 +2,15 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//
+
+#ifndef ROCKSDB_LITE
 #include "db/transaction_log_impl.h"
 #include "db/write_batch_internal.h"
 
 namespace rocksdb {
 
 TransactionLogIteratorImpl::TransactionLogIteratorImpl(
-    const std::string& dir, const Options* options,
+    const std::string& dir, const DBOptions* options,
     const TransactionLogIterator::ReadOptions& read_options,
     const EnvOptions& soptions, const SequenceNumber seq,
     std::unique_ptr<VectorLogPtr> files, DBImpl const* const dbimpl)
@@ -257,3 +258,4 @@ Status TransactionLogIteratorImpl::OpenLogReader(const LogFile* logFile) {
   return Status::OK();
 }
 }  //  namespace rocksdb
+#endif  // ROCKSDB_LITE

db/transaction_log_impl.h

@@ -2,7 +2,8 @@
 //  This source code is licensed under the BSD-style license found in the
 //  LICENSE file in the root directory of this source tree. An additional grant
 //  of patent rights can be found in the PATENTS file in the same directory.
-//
+
+#ifndef ROCKSDB_LITE
 #pragma once
 #include <vector>
 
@@ -67,7 +68,7 @@ class LogFileImpl : public LogFile {
 class TransactionLogIteratorImpl : public TransactionLogIterator {
  public:
   TransactionLogIteratorImpl(
-      const std::string& dir, const Options* options,
+      const std::string& dir, const DBOptions* options,
       const TransactionLogIterator::ReadOptions& read_options,
       const EnvOptions& soptions, const SequenceNumber seqNum,
       std::unique_ptr<VectorLogPtr> files, DBImpl const* const dbimpl);
@@ -82,7 +83,7 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
 
  private:
   const std::string& dir_;
-  const Options* options_;
+  const DBOptions* options_;
   const TransactionLogIterator::ReadOptions read_options_;
   const EnvOptions& soptions_;
   SequenceNumber startingSequenceNumber_;
@@ -116,3 +117,4 @@ class TransactionLogIteratorImpl : public TransactionLogIterator {
   Status OpenLogReader(const LogFile* file);
 };
 }  //  namespace rocksdb
+#endif  // ROCKSDB_LITE

db/version_edit.cc

@@ -11,6 +11,7 @@
 
 #include "db/version_set.h"
 #include "util/coding.h"
+#include "rocksdb/slice.h"
 
 namespace rocksdb {
 
@@ -29,6 +30,11 @@ enum Tag {
 
   // these are new formats divergent from open source leveldb
   kNewFile2             = 100,  // store smallest & largest seqno
+
+  kColumnFamily         = 200,  // specify column family for version edit
+  kColumnFamilyAdd      = 201,
+  kColumnFamilyDrop     = 202,
+  kMaxColumnFamily      = 203,
 };
 
 void VersionEdit::Clear() {
@@ -38,13 +44,19 @@ void VersionEdit::Clear() {
   prev_log_number_ = 0;
   last_sequence_ = 0;
   next_file_number_ = 0;
+  max_column_family_ = 0;
   has_comparator_ = false;
   has_log_number_ = false;
   has_prev_log_number_ = false;
   has_next_file_number_ = false;
   has_last_sequence_ = false;
+  has_max_column_family_ = false;
   deleted_files_.clear();
   new_files_.clear();
+  column_family_ = 0;
+  is_column_family_add_ = 0;
+  is_column_family_drop_ = 0;
+  column_family_name_.clear();
 }
 
 void VersionEdit::EncodeTo(std::string* dst) const {
@@ -68,6 +80,10 @@ void VersionEdit::EncodeTo(std::string* dst) const {
     PutVarint32(dst, kLastSequence);
     PutVarint64(dst, last_sequence_);
   }
+  if (has_max_column_family_) {
+    PutVarint32(dst, kMaxColumnFamily);
+    PutVarint32(dst, max_column_family_);
+  }
 
   for (const auto& deleted : deleted_files_) {
     PutVarint32(dst, kDeletedFile);
@@ -86,6 +102,21 @@ void VersionEdit::EncodeTo(std::string* dst) const {
     PutVarint64(dst, f.smallest_seqno);
     PutVarint64(dst, f.largest_seqno);
   }
+
+  // 0 is default and does not need to be explicitly written
+  if (column_family_ != 0) {
+    PutVarint32(dst, kColumnFamily);
+    PutVarint32(dst, column_family_);
+  }
+
+  if (is_column_family_add_) {
+    PutVarint32(dst, kColumnFamilyAdd);
+    PutLengthPrefixedSlice(dst, Slice(column_family_name_));
+  }
+
+  if (is_column_family_drop_) {
+    PutVarint32(dst, kColumnFamilyDrop);
+  }
 }
 
 static bool GetInternalKey(Slice* input, InternalKey* dst) {
@@ -167,6 +198,14 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         }
         break;
 
+      case kMaxColumnFamily:
+        if (GetVarint32(&input, &max_column_family_)) {
+          has_max_column_family_ = true;
+        } else {
+          msg = "max column family";
+        }
+        break;
+
       case kCompactPointer:
         if (GetLevel(&input, &level, &msg) &&
             GetInternalKey(&input, &key)) {
@@ -221,6 +260,29 @@ Status VersionEdit::DecodeFrom(const Slice& src) {
         }
         break;
 
+      case kColumnFamily:
+        if (!GetVarint32(&input, &column_family_)) {
+          if (!msg) {
+            msg = "set column family id";
+          }
+        }
+        break;
+
+      case kColumnFamilyAdd:
+        if (GetLengthPrefixedSlice(&input, &str)) {
+          is_column_family_add_ = true;
+          column_family_name_ = str.ToString();
+        } else {
+          if (!msg) {
+            msg = "column family add";
+          }
+        }
+        break;
+
+      case kColumnFamilyDrop:
+        is_column_family_drop_ = true;
+        break;
+
       default:
         msg = "unknown tag";
         break;
@@ -282,6 +344,19 @@ std::string VersionEdit::DebugString(bool hex_key) const {
     r.append(" .. ");
     r.append(f.largest.DebugString(hex_key));
   }
+  r.append("\n  ColumnFamily: ");
+  AppendNumberTo(&r, column_family_);
+  if (is_column_family_add_) {
+    r.append("\n  ColumnFamilyAdd: ");
+    r.append(column_family_name_);
+  }
+  if (is_column_family_drop_) {
+    r.append("\n  ColumnFamilyDrop");
+  }
+  if (has_max_column_family_) {
+    r.append("\n  MaxColumnFamily: ");
+    AppendNumberTo(&r, max_column_family_);
+  }
   r.append("\n}\n");
   return r;
 }

db/version_edit.h

@@ -11,6 +11,7 @@
 #include <set>
 #include <utility>
 #include <vector>
+#include <string>
 #include "rocksdb/cache.h"
 #include "db/dbformat.h"
 
@@ -31,12 +32,18 @@ struct FileMetaData {
 
   // Needs to be disposed when refs becomes 0.
   Cache::Handle* table_reader_handle;
+  // Table reader in table_reader_handle
+  TableReader* table_reader;
 
-  FileMetaData(uint64_t number, uint64_t file_size) :
-      refs(0), allowed_seeks(1 << 30), number(number), file_size(file_size),
-      being_compacted(false), table_reader_handle(nullptr) {
-  }
-  FileMetaData() : FileMetaData(0, 0) { }
+  FileMetaData(uint64_t number, uint64_t file_size)
+      : refs(0),
+        allowed_seeks(1 << 30),
+        number(number),
+        file_size(file_size),
+        being_compacted(false),
+        table_reader_handle(nullptr),
+        table_reader(nullptr) {}
+  FileMetaData() : FileMetaData(0, 0) {}
 };
 
 class VersionEdit {
@@ -66,6 +73,10 @@ class VersionEdit {
     has_last_sequence_ = true;
     last_sequence_ = seq;
   }
+  void SetMaxColumnFamily(uint32_t max_column_family) {
+    has_max_column_family_ = true;
+    max_column_family_ = max_column_family;
+  }
 
   // Add the specified file at the specified number.
   // REQUIRES: This version has not been saved (see VersionSet::SaveTo)
@@ -97,6 +108,31 @@ class VersionEdit {
     return new_files_.size() + deleted_files_.size();
   }
 
+  bool IsColumnFamilyManipulation() {
+    return is_column_family_add_ || is_column_family_drop_;
+  }
+
+  void SetColumnFamily(uint32_t column_family_id) {
+    column_family_ = column_family_id;
+  }
+
+  // set column family ID by calling SetColumnFamily()
+  void AddColumnFamily(const std::string& name) {
+    assert(!is_column_family_drop_);
+    assert(!is_column_family_add_);
+    assert(NumEntries() == 0);
+    is_column_family_add_ = true;
+    column_family_name_ = name;
+  }
+
+  // set column family ID by calling SetColumnFamily()
+  void DropColumnFamily() {
+    assert(!is_column_family_drop_);
+    assert(!is_column_family_add_);
+    assert(NumEntries() == 0);
+    is_column_family_drop_ = true;
+  }
+
   void EncodeTo(std::string* dst) const;
   Status DecodeFrom(const Slice& src);
 
@@ -114,15 +150,27 @@ class VersionEdit {
   uint64_t log_number_;
   uint64_t prev_log_number_;
   uint64_t next_file_number_;
+  uint32_t max_column_family_;
   SequenceNumber last_sequence_;
   bool has_comparator_;
   bool has_log_number_;
   bool has_prev_log_number_;
   bool has_next_file_number_;
   bool has_last_sequence_;
+  bool has_max_column_family_;
 
   DeletedFileSet deleted_files_;
-  std::vector<std::pair<int, FileMetaData> > new_files_;
+  std::vector<std::pair<int, FileMetaData>> new_files_;
+
+  // Each version edit record should have column_family_id set
+  // If it's not set, it is default (0)
+  uint32_t column_family_;
+  // a version edit can be either column_family add or
+  // column_family drop. If it's column family add,
+  // it also includes column family name.
+  bool is_column_family_drop_;
+  bool is_column_family_add_;
+  std::string column_family_name_;
 };
 
 }  // namespace rocksdb

db/version_edit_test.cc

@@ -45,6 +45,19 @@ TEST(VersionEditTest, EncodeDecode) {
   TestEncodeDecode(edit);
 }
 
+TEST(VersionEditTest, ColumnFamilyTest) {
+  VersionEdit edit;
+  edit.SetColumnFamily(2);
+  edit.AddColumnFamily("column_family");
+  edit.SetMaxColumnFamily(5);
+  TestEncodeDecode(edit);
+
+  edit.Clear();
+  edit.SetColumnFamily(3);
+  edit.DropColumnFamily();
+  TestEncodeDecode(edit);
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

db/version_set.h

@@ -24,12 +24,16 @@
 #include <vector>
 #include <deque>
 #include <atomic>
+#include <limits>
 #include "db/dbformat.h"
 #include "db/version_edit.h"
 #include "port/port.h"
 #include "db/table_cache.h"
 #include "db/compaction.h"
 #include "db/compaction_picker.h"
+#include "db/column_family.h"
+#include "db/log_reader.h"
+#include "db/file_indexer.h"
 
 namespace rocksdb {
 
@@ -41,10 +45,12 @@ class Iterator;
 class LogBuffer;
 class LookupKey;
 class MemTable;
-class MergeContext;
-class TableCache;
 class Version;
 class VersionSet;
+class MergeContext;
+class ColumnFamilyData;
+class ColumnFamilySet;
+class TableCache;
 
 // Return the smallest index i such that files[i]->largest >= key.
 // Return files.size() if there is no such file.
@@ -83,8 +89,7 @@ class Version {
     int seek_file_level;
   };
   void Get(const ReadOptions&, const LookupKey& key, std::string* val,
-           Status* status, MergeContext* merge_context,
-           GetStats* stats, const Options& db_option,
+           Status* status, MergeContext* merge_context, GetStats* stats,
            bool* value_found = nullptr);
 
   // Adds "stats" into the current state.  Returns true if a new
@@ -208,21 +213,28 @@ class Version {
   friend class Compaction;
   friend class VersionSet;
   friend class DBImpl;
+  friend class ColumnFamilyData;
   friend class CompactionPicker;
   friend class LevelCompactionPicker;
   friend class UniversalCompactionPicker;
 
   class LevelFileNumIterator;
-  Iterator* NewConcatenatingIterator(const ReadOptions&,
-                                     const EnvOptions& soptions,
-                                     int level) const;
-  bool PrefixMayMatch(const ReadOptions& options, const EnvOptions& soptions,
-                      const Slice& internal_prefix, Iterator* level_iter) const;
+  class LevelFileIteratorState;
+
+  bool PrefixMayMatch(const ReadOptions& options, Iterator* level_iter,
+                      const Slice& internal_prefix) const;
 
   // Sort all files for this version based on their file size and
   // record results in files_by_size_. The largest files are listed first.
   void UpdateFilesBySize();
 
+  ColumnFamilyData* cfd_;  // ColumnFamilyData to which this Version belongs
+  const InternalKeyComparator* internal_comparator_;
+  const Comparator* user_comparator_;
+  TableCache* table_cache_;
+  const MergeOperator* merge_operator_;
+  Logger* info_log_;
+  Statistics* db_statistics_;
   VersionSet* vset_;            // VersionSet to which this Version belongs
   Version* next_;               // Next version in linked list
   Version* prev_;               // Previous version in linked list
@@ -268,7 +280,8 @@ class Version {
   // used for debugging and logging purposes only.
   uint64_t version_number_;
 
-  explicit Version(VersionSet* vset, uint64_t version_number = 0);
+  Version(ColumnFamilyData* cfd, VersionSet* vset, uint64_t version_number = 0);
+  FileIndexer file_indexer_;
 
   ~Version();
 
@@ -285,23 +298,34 @@ class Version {
 
 class VersionSet {
  public:
-  VersionSet(const std::string& dbname, const Options* options,
-             const EnvOptions& storage_options, TableCache* table_cache,
-             const InternalKeyComparator*);
+  VersionSet(const std::string& dbname, const DBOptions* options,
+             const EnvOptions& storage_options, Cache* table_cache);
   ~VersionSet();
 
   // Apply *edit to the current version to form a new descriptor that
   // is both saved to persistent state and installed as the new
   // current version.  Will release *mu while actually writing to the file.
+  // column_family_options has to be set if edit is column family add
   // REQUIRES: *mu is held on entry.
   // REQUIRES: no other thread concurrently calls LogAndApply()
-  Status LogAndApply(VersionEdit* edit, port::Mutex* mu,
-                     Directory* db_directory = nullptr,
-                     bool new_descriptor_log = false);
+  Status LogAndApply(ColumnFamilyData* column_family_data, VersionEdit* edit,
+                     port::Mutex* mu, Directory* db_directory = nullptr,
+                     bool new_descriptor_log = false,
+                     const ColumnFamilyOptions* column_family_options =
+                         nullptr);
 
   // Recover the last saved descriptor from persistent storage.
-  Status Recover();
+  // If read_only == true, Recover() will not complain if some column families
+  // are not opened
+  Status Recover(const std::vector<ColumnFamilyDescriptor>& column_families,
+                 bool read_only = false);
 
+  // Reads a manifest file and returns a list of column families in
+  // column_families.
+  static Status ListColumnFamilies(std::vector<std::string>* column_families,
+                                   const std::string& dbname, Env* env);
+
+#ifndef ROCKSDB_LITE
   // Try to reduce the number of levels. This call is valid when
   // only one level from the new max level to the old
   // max level containing files.
@@ -316,14 +340,11 @@ class VersionSet {
                                      const EnvOptions& storage_options,
                                      int new_levels);
 
-  // Return the current version.
-  Version* current() const { return current_; }
+  // printf contents (for debugging)
+  Status DumpManifest(Options& options, std::string& manifestFileName,
+                      bool verbose, bool hex = false);
 
-  // A Flag indicating whether write needs to slowdown because of there are
-  // too many number of level0 files.
-  bool NeedSlowdownForNumLevel0Files() const {
-    return need_slowdown_for_num_level0_files_;
-  }
+#endif  // ROCKSDB_LITE
 
   // Return the current manifest file number
   uint64_t ManifestFileNumber() const { return manifest_file_number_; }
@@ -358,37 +379,21 @@ class VersionSet {
   // Mark the specified file number as used.
   void MarkFileNumberUsed(uint64_t number);
 
-  // Return the current log file number.
-  uint64_t LogNumber() const { return log_number_; }
-
   // Return the log file number for the log file that is currently
   // being compacted, or zero if there is no such log file.
   uint64_t PrevLogNumber() const { return prev_log_number_; }
 
-  int NumberLevels() const { return num_levels_; }
-
-  // Pick level and inputs for a new compaction.
-  // Returns nullptr if there is no compaction to be done.
-  // Otherwise returns a pointer to a heap-allocated object that
-  // describes the compaction.  Caller should delete the result.
-  Compaction* PickCompaction(LogBuffer* log_buffer);
-
-  // Return a compaction object for compacting the range [begin,end] in
-  // the specified level.  Returns nullptr if there is nothing in that
-  // level that overlaps the specified range.  Caller should delete
-  // the result.
-  //
-  // The returned Compaction might not include the whole requested range.
-  // In that case, compaction_end will be set to the next key that needs
-  // compacting. In case the compaction will compact the whole range,
-  // compaction_end will be set to nullptr.
-  // Client is responsible for compaction_end storage -- when called,
-  // *compaction_end should point to valid InternalKey!
-  Compaction* CompactRange(int input_level,
-                           int output_level,
-                           const InternalKey* begin,
-                           const InternalKey* end,
-                           InternalKey** compaction_end);
+  // Returns the minimum log number such that all
+  // log numbers less than or equal to it can be deleted
+  uint64_t MinLogNumber() const {
+    uint64_t min_log_num = std::numeric_limits<uint64_t>::max();
+    for (auto cfd : *column_family_set_) {
+      if (min_log_num > cfd->GetLogNumber()) {
+        min_log_num = cfd->GetLogNumber();
+      }
+    }
+    return min_log_num;
+  }
 
   // Create an iterator that reads over the compaction inputs for "*c".
   // The caller should delete the iterator when no longer needed.
@@ -401,10 +406,6 @@ class VersionSet {
   // "key" as of version "v".
   uint64_t ApproximateOffsetOf(Version* v, const InternalKey& key);
 
-  // printf contents (for debugging)
-  Status DumpManifest(Options& options, std::string& manifestFileName,
-                      bool verbose, bool hex = false);
-
   // Return the size of the current manifest file
   uint64_t ManifestFileSize() const { return manifest_file_size_; }
 
@@ -414,62 +415,53 @@ class VersionSet {
   // pick the same files to compact.
   bool VerifyCompactionFileConsistency(Compaction* c);
 
-  double MaxBytesForLevel(int level);
-
-  // Get the max file size in a given level.
-  uint64_t MaxFileSizeForLevel(int level);
-
-  void ReleaseCompactionFiles(Compaction* c, Status status);
-
-  Status GetMetadataForFile(
-    uint64_t number, int *filelevel, FileMetaData **metadata);
+  Status GetMetadataForFile(uint64_t number, int* filelevel,
+                            FileMetaData** metadata, ColumnFamilyData** cfd);
 
   void GetLiveFilesMetaData(
     std::vector<LiveFileMetaData> *metadata);
 
   void GetObsoleteFiles(std::vector<FileMetaData*>* files);
 
+  ColumnFamilySet* GetColumnFamilySet() { return column_family_set_.get(); }
+
  private:
   class Builder;
   struct ManifestWriter;
 
-  friend class Compaction;
   friend class Version;
 
+  struct LogReporter : public log::Reader::Reporter {
+    Status* status;
+    virtual void Corruption(size_t bytes, const Status& s) {
+      if (this->status->ok()) *this->status = s;
+    }
+  };
+
   // Save current contents to *log
   Status WriteSnapshot(log::Writer* log);
 
-  void AppendVersion(Version* v);
+  void AppendVersion(ColumnFamilyData* column_family_data, Version* v);
 
   bool ManifestContains(uint64_t manifest_file_number,
                         const std::string& record) const;
 
+  ColumnFamilyData* CreateColumnFamily(const ColumnFamilyOptions& options,
+                                       VersionEdit* edit);
+
+  std::unique_ptr<ColumnFamilySet> column_family_set_;
+
   Env* const env_;
   const std::string dbname_;
-  const Options* const options_;
-  TableCache* const table_cache_;
-  const InternalKeyComparator icmp_;
+  const DBOptions* const options_;
   uint64_t next_file_number_;
   uint64_t manifest_file_number_;
   uint64_t pending_manifest_file_number_;
   std::atomic<uint64_t> last_sequence_;
-  uint64_t log_number_;
   uint64_t prev_log_number_;  // 0 or backing store for memtable being compacted
 
-  int num_levels_;
-
   // Opened lazily
   unique_ptr<log::Writer> descriptor_log_;
-  Version dummy_versions_;  // Head of circular doubly-linked list of versions.
-  Version* current_;        // == dummy_versions_.prev_
-
-  // A flag indicating whether we should delay writes because
-  // we have too many level 0 files
-  bool need_slowdown_for_num_level0_files_;
-
-  // An object that keeps all the compaction stats
-  // and picks the next compaction
-  std::unique_ptr<CompactionPicker> compaction_picker_;
 
   // generates a increasing version number for every new version
   uint64_t current_version_number_;
@@ -493,8 +485,9 @@ class VersionSet {
   VersionSet(const VersionSet&);
   void operator=(const VersionSet&);
 
-  void LogAndApplyHelper(Builder*b, Version* v,
-                           VersionEdit* edit, port::Mutex* mu);
+  void LogAndApplyCFHelper(VersionEdit* edit);
+  void LogAndApplyHelper(ColumnFamilyData* cfd, Builder* b, Version* v,
+                         VersionEdit* edit, port::Mutex* mu);
 };
 
 }  // namespace rocksdb

db/write_batch.cc

@@ -15,6 +15,9 @@
 //    kTypeValue varstring varstring
 //    kTypeMerge varstring varstring
 //    kTypeDeletion varstring
+//    kTypeColumnFamilyValue varint32 varstring varstring
+//    kTypeColumnFamilyMerge varint32 varstring varstring
+//    kTypeColumnFamilyDeletion varint32 varstring varstring
 // varstring :=
 //    len: varint32
 //    data: uint8[len]
@@ -45,10 +48,20 @@ WriteBatch::~WriteBatch() { }
 
 WriteBatch::Handler::~Handler() { }
 
+void WriteBatch::Handler::Put(const Slice& key, const Slice& value) {
+  // you need to either implement Put or PutCF
+  throw std::runtime_error("Handler::Put not implemented!");
+}
+
 void WriteBatch::Handler::Merge(const Slice& key, const Slice& value) {
   throw std::runtime_error("Handler::Merge not implemented!");
 }
 
+void WriteBatch::Handler::Delete(const Slice& key) {
+  // you need to either implement Delete or DeleteCF
+  throw std::runtime_error("Handler::Delete not implemented!");
+}
+
 void WriteBatch::Handler::LogData(const Slice& blob) {
   // If the user has not specified something to do with blobs, then we ignore
   // them.
@@ -76,31 +89,48 @@ Status WriteBatch::Iterate(Handler* handler) const {
   input.remove_prefix(kHeader);
   Slice key, value, blob;
   int found = 0;
-  while (!input.empty() && handler->Continue()) {
+  Status s;
+  while (s.ok() && !input.empty() && handler->Continue()) {
     char tag = input[0];
     input.remove_prefix(1);
+    uint32_t column_family = 0;  // default
     switch (tag) {
+      case kTypeColumnFamilyValue:
+        if (!GetVarint32(&input, &column_family)) {
+          return Status::Corruption("bad WriteBatch Put");
+        }
+      // intentional fallthrough
       case kTypeValue:
         if (GetLengthPrefixedSlice(&input, &key) &&
             GetLengthPrefixedSlice(&input, &value)) {
-          handler->Put(key, value);
+          s = handler->PutCF(column_family, key, value);
           found++;
         } else {
           return Status::Corruption("bad WriteBatch Put");
         }
         break;
+      case kTypeColumnFamilyDeletion:
+        if (!GetVarint32(&input, &column_family)) {
+          return Status::Corruption("bad WriteBatch Delete");
+        }
+      // intentional fallthrough
       case kTypeDeletion:
         if (GetLengthPrefixedSlice(&input, &key)) {
-          handler->Delete(key);
+          s = handler->DeleteCF(column_family, key);
           found++;
         } else {
           return Status::Corruption("bad WriteBatch Delete");
         }
         break;
+      case kTypeColumnFamilyMerge:
+        if (!GetVarint32(&input, &column_family)) {
+          return Status::Corruption("bad WriteBatch Merge");
+        }
+      // intentional fallthrough
       case kTypeMerge:
         if (GetLengthPrefixedSlice(&input, &key) &&
             GetLengthPrefixedSlice(&input, &value)) {
-          handler->Merge(key, value);
+          s = handler->MergeCF(column_family, key, value);
           found++;
         } else {
           return Status::Corruption("bad WriteBatch Merge");
@@ -117,7 +147,10 @@ Status WriteBatch::Iterate(Handler* handler) const {
         return Status::Corruption("unknown WriteBatch tag");
     }
   }
- if (found != WriteBatchInternal::Count(this)) {
+  if (!s.ok()) {
+    return s;
+  }
+  if (found != WriteBatchInternal::Count(this)) {
     return Status::Corruption("WriteBatch has wrong count");
   } else {
     return Status::OK();
@@ -140,31 +173,85 @@ void WriteBatchInternal::SetSequence(WriteBatch* b, SequenceNumber seq) {
   EncodeFixed64(&b->rep_[0], seq);
 }
 
-void WriteBatch::Put(const Slice& key, const Slice& value) {
-  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
-  rep_.push_back(static_cast<char>(kTypeValue));
-  PutLengthPrefixedSlice(&rep_, key);
-  PutLengthPrefixedSlice(&rep_, value);
+void WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
+                             const Slice& key, const Slice& value) {
+  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+  if (column_family_id == 0) {
+    b->rep_.push_back(static_cast<char>(kTypeValue));
+  } else {
+    b->rep_.push_back(static_cast<char>(kTypeColumnFamilyValue));
+    PutVarint32(&b->rep_, column_family_id);
+  }
+  PutLengthPrefixedSlice(&b->rep_, key);
+  PutLengthPrefixedSlice(&b->rep_, value);
 }
 
-void WriteBatch::Put(const SliceParts& key, const SliceParts& value) {
-  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
-  rep_.push_back(static_cast<char>(kTypeValue));
-  PutLengthPrefixedSliceParts(&rep_, key);
-  PutLengthPrefixedSliceParts(&rep_, value);
+namespace {
+inline uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family) {
+  uint32_t column_family_id = 0;
+  if (column_family != nullptr) {
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
+    column_family_id = cfh->GetID();
+  }
+  return column_family_id;
+}
+}  // namespace
+
+void WriteBatch::Put(ColumnFamilyHandle* column_family, const Slice& key,
+                     const Slice& value) {
+  WriteBatchInternal::Put(this, GetColumnFamilyID(column_family), key, value);
 }
 
-void WriteBatch::Delete(const Slice& key) {
-  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
-  rep_.push_back(static_cast<char>(kTypeDeletion));
-  PutLengthPrefixedSlice(&rep_, key);
+void WriteBatchInternal::Put(WriteBatch* b, uint32_t column_family_id,
+                             const SliceParts& key, const SliceParts& value) {
+  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+  if (column_family_id == 0) {
+    b->rep_.push_back(static_cast<char>(kTypeValue));
+  } else {
+    b->rep_.push_back(static_cast<char>(kTypeColumnFamilyValue));
+    PutVarint32(&b->rep_, column_family_id);
+  }
+  PutLengthPrefixedSliceParts(&b->rep_, key);
+  PutLengthPrefixedSliceParts(&b->rep_, value);
 }
 
-void WriteBatch::Merge(const Slice& key, const Slice& value) {
-  WriteBatchInternal::SetCount(this, WriteBatchInternal::Count(this) + 1);
-  rep_.push_back(static_cast<char>(kTypeMerge));
-  PutLengthPrefixedSlice(&rep_, key);
-  PutLengthPrefixedSlice(&rep_, value);
+void WriteBatch::Put(ColumnFamilyHandle* column_family, const SliceParts& key,
+                     const SliceParts& value) {
+  WriteBatchInternal::Put(this, GetColumnFamilyID(column_family), key, value);
+}
+
+void WriteBatchInternal::Delete(WriteBatch* b, uint32_t column_family_id,
+                                const Slice& key) {
+  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+  if (column_family_id == 0) {
+    b->rep_.push_back(static_cast<char>(kTypeDeletion));
+  } else {
+    b->rep_.push_back(static_cast<char>(kTypeColumnFamilyDeletion));
+    PutVarint32(&b->rep_, column_family_id);
+  }
+  PutLengthPrefixedSlice(&b->rep_, key);
+}
+
+void WriteBatch::Delete(ColumnFamilyHandle* column_family, const Slice& key) {
+  WriteBatchInternal::Delete(this, GetColumnFamilyID(column_family), key);
+}
+
+void WriteBatchInternal::Merge(WriteBatch* b, uint32_t column_family_id,
+                               const Slice& key, const Slice& value) {
+  WriteBatchInternal::SetCount(b, WriteBatchInternal::Count(b) + 1);
+  if (column_family_id == 0) {
+    b->rep_.push_back(static_cast<char>(kTypeMerge));
+  } else {
+    b->rep_.push_back(static_cast<char>(kTypeColumnFamilyMerge));
+    PutVarint32(&b->rep_, column_family_id);
+  }
+  PutLengthPrefixedSlice(&b->rep_, key);
+  PutLengthPrefixedSlice(&b->rep_, value);
+}
+
+void WriteBatch::Merge(ColumnFamilyHandle* column_family, const Slice& key,
+                       const Slice& value) {
+  WriteBatchInternal::Merge(this, GetColumnFamilyID(column_family), key, value);
 }
 
 void WriteBatch::PutLogData(const Slice& blob) {
@@ -176,33 +263,70 @@ namespace {
 class MemTableInserter : public WriteBatch::Handler {
  public:
   SequenceNumber sequence_;
-  MemTable* mem_;
-  const Options* options_;
+  ColumnFamilyMemTables* cf_mems_;
+  bool recovery_;
+  uint64_t log_number_;
   DBImpl* db_;
-  const bool filter_deletes_;
+  const bool dont_filter_deletes_;
 
-  MemTableInserter(SequenceNumber sequence, MemTable* mem, const Options* opts,
-                   DB* db, const bool filter_deletes)
-    : sequence_(sequence),
-      mem_(mem),
-      options_(opts),
-      db_(reinterpret_cast<DBImpl*>(db)),
-      filter_deletes_(filter_deletes) {
-    assert(mem_);
-    if (filter_deletes_) {
-      assert(options_);
+  MemTableInserter(SequenceNumber sequence, ColumnFamilyMemTables* cf_mems,
+                   bool recovery, uint64_t log_number, DB* db,
+                   const bool dont_filter_deletes)
+      : sequence_(sequence),
+        cf_mems_(cf_mems),
+        recovery_(recovery),
+        log_number_(log_number),
+        db_(reinterpret_cast<DBImpl*>(db)),
+        dont_filter_deletes_(dont_filter_deletes) {
+    assert(cf_mems);
+    if (!dont_filter_deletes_) {
       assert(db_);
     }
   }
 
-  virtual void Put(const Slice& key, const Slice& value) {
-    if (!options_->inplace_update_support) {
-      mem_->Add(sequence_, kTypeValue, key, value);
-    } else if (options_->inplace_callback == nullptr) {
-      mem_->Update(sequence_, key, value);
-      RecordTick(options_->statistics.get(), NUMBER_KEYS_UPDATED);
+  bool SeekToColumnFamily(uint32_t column_family_id, Status* s) {
+    bool found = cf_mems_->Seek(column_family_id);
+    if (recovery_ && (!found || log_number_ < cf_mems_->GetLogNumber())) {
+      // if in recovery envoronment:
+      // * If column family was not found, it might mean that the WAL write
+      // batch references to the column family that was dropped after the
+      // insert. We don't want to fail the whole write batch in that case -- we
+      // just ignore the update.
+      // * If log_number_ < cf_mems_->GetLogNumber(), this means that column
+      // family already contains updates from this log. We can't apply updates
+      // twice because of update-in-place or merge workloads -- ignore the
+      // update
+      *s = Status::OK();
+      return false;
+    }
+    if (!found) {
+      assert(!recovery_);
+      // If the column family was not found in non-recovery enviornment
+      // (client's write code-path), we have to fail the write and return
+      // the failure status to the client.
+      *s = Status::InvalidArgument(
+          "Invalid column family specified in write batch");
+      return false;
+    }
+    return true;
+  }
+
+  virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+                       const Slice& value) {
+    Status seek_status;
+    if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+      ++sequence_;
+      return seek_status;
+    }
+    MemTable* mem = cf_mems_->GetMemTable();
+    const Options* options = cf_mems_->GetOptions();
+    if (!options->inplace_update_support) {
+      mem->Add(sequence_, kTypeValue, key, value);
+    } else if (options->inplace_callback == nullptr) {
+      mem->Update(sequence_, key, value);
+      RecordTick(options->statistics.get(), NUMBER_KEYS_UPDATED);
     } else {
-      if (mem_->UpdateCallback(sequence_, key, value, *options_)) {
+      if (mem->UpdateCallback(sequence_, key, value, *options)) {
       } else {
         // key not found in memtable. Do sst get, update, add
         SnapshotImpl read_from_snapshot;
@@ -212,21 +336,26 @@ class MemTableInserter : public WriteBatch::Handler {
 
         std::string prev_value;
         std::string merged_value;
-        Status s = db_->Get(ropts, key, &prev_value);
+
+        auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+        if (cf_handle == nullptr) {
+          cf_handle = db_->DefaultColumnFamily();
+        }
+        Status s = db_->Get(ropts, cf_handle, key, &prev_value);
+
         char* prev_buffer = const_cast<char*>(prev_value.c_str());
         uint32_t prev_size = prev_value.size();
-        auto status =
-          options_->inplace_callback(s.ok() ? prev_buffer: nullptr,
-                                     s.ok() ? &prev_size: nullptr,
-                                     value, &merged_value);
+        auto status = options->inplace_callback(s.ok() ? prev_buffer : nullptr,
+                                                s.ok() ? &prev_size : nullptr,
+                                                value, &merged_value);
         if (status == UpdateStatus::UPDATED_INPLACE) {
           // prev_value is updated in-place with final value.
-          mem_->Add(sequence_, kTypeValue, key, Slice(prev_buffer, prev_size));
-          RecordTick(options_->statistics.get(), NUMBER_KEYS_WRITTEN);
+          mem->Add(sequence_, kTypeValue, key, Slice(prev_buffer, prev_size));
+          RecordTick(options->statistics.get(), NUMBER_KEYS_WRITTEN);
         } else if (status == UpdateStatus::UPDATED) {
           // merged_value contains the final value.
-          mem_->Add(sequence_, kTypeValue, key, Slice(merged_value));
-          RecordTick(options_->statistics.get(), NUMBER_KEYS_WRITTEN);
+          mem->Add(sequence_, kTypeValue, key, Slice(merged_value));
+          RecordTick(options->statistics.get(), NUMBER_KEYS_WRITTEN);
         }
       }
     }
@@ -234,19 +363,28 @@ class MemTableInserter : public WriteBatch::Handler {
     // sequence number. Even if the update eventually fails and does not result
     // in memtable add/update.
     sequence_++;
+    return Status::OK();
   }
 
-  virtual void Merge(const Slice& key, const Slice& value) {
+  virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+                         const Slice& value) {
+    Status seek_status;
+    if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+      ++sequence_;
+      return seek_status;
+    }
+    MemTable* mem = cf_mems_->GetMemTable();
+    const Options* options = cf_mems_->GetOptions();
     bool perform_merge = false;
 
-    if (options_->max_successive_merges > 0 && db_ != nullptr) {
+    if (options->max_successive_merges > 0 && db_ != nullptr) {
       LookupKey lkey(key, sequence_);
 
       // Count the number of successive merges at the head
       // of the key in the memtable
-      size_t num_merges = mem_->CountSuccessiveMergeEntries(lkey);
+      size_t num_merges = mem->CountSuccessiveMergeEntries(lkey);
 
-      if (num_merges >= options_->max_successive_merges) {
+      if (num_merges >= options->max_successive_merges) {
         perform_merge = true;
       }
     }
@@ -262,62 +400,78 @@ class MemTableInserter : public WriteBatch::Handler {
       ReadOptions read_options;
       read_options.snapshot = &read_from_snapshot;
 
-      db_->Get(read_options, key, &get_value);
+      auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+      if (cf_handle == nullptr) {
+        cf_handle = db_->DefaultColumnFamily();
+      }
+      db_->Get(read_options, cf_handle, key, &get_value);
       Slice get_value_slice = Slice(get_value);
 
       // 2) Apply this merge
-      auto merge_operator = options_->merge_operator.get();
+      auto merge_operator = options->merge_operator.get();
       assert(merge_operator);
 
       std::deque<std::string> operands;
       operands.push_front(value.ToString());
       std::string new_value;
-      if (!merge_operator->FullMerge(key,
-                                     &get_value_slice,
-                                     operands,
-                                     &new_value,
-                                     options_->info_log.get())) {
+      if (!merge_operator->FullMerge(key, &get_value_slice, operands,
+                                     &new_value, options->info_log.get())) {
           // Failed to merge!
-          RecordTick(options_->statistics.get(), NUMBER_MERGE_FAILURES);
+        RecordTick(options->statistics.get(), NUMBER_MERGE_FAILURES);
 
           // Store the delta in memtable
           perform_merge = false;
       } else {
         // 3) Add value to memtable
-        mem_->Add(sequence_, kTypeValue, key, new_value);
+        mem->Add(sequence_, kTypeValue, key, new_value);
       }
     }
 
     if (!perform_merge) {
       // Add merge operator to memtable
-      mem_->Add(sequence_, kTypeMerge, key, value);
+      mem->Add(sequence_, kTypeMerge, key, value);
     }
 
     sequence_++;
+    return Status::OK();
   }
-  virtual void Delete(const Slice& key) {
-    if (filter_deletes_) {
+
+  virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
+    Status seek_status;
+    if (!SeekToColumnFamily(column_family_id, &seek_status)) {
+      ++sequence_;
+      return seek_status;
+    }
+    MemTable* mem = cf_mems_->GetMemTable();
+    const Options* options = cf_mems_->GetOptions();
+    if (!dont_filter_deletes_ && options->filter_deletes) {
       SnapshotImpl read_from_snapshot;
       read_from_snapshot.number_ = sequence_;
       ReadOptions ropts;
       ropts.snapshot = &read_from_snapshot;
       std::string value;
-      if (!db_->KeyMayExist(ropts, key, &value)) {
-        RecordTick(options_->statistics.get(), NUMBER_FILTERED_DELETES);
-        return;
+      auto cf_handle = cf_mems_->GetColumnFamilyHandle();
+      if (cf_handle == nullptr) {
+        cf_handle = db_->DefaultColumnFamily();
+      }
+      if (!db_->KeyMayExist(ropts, cf_handle, key, &value)) {
+        RecordTick(options->statistics.get(), NUMBER_FILTERED_DELETES);
+        return Status::OK();
       }
     }
-    mem_->Add(sequence_, kTypeDeletion, key, Slice());
+    mem->Add(sequence_, kTypeDeletion, key, Slice());
     sequence_++;
+    return Status::OK();
   }
 };
 }  // namespace
 
-Status WriteBatchInternal::InsertInto(const WriteBatch* b, MemTable* mem,
-                                      const Options* opts, DB* db,
-                                      const bool filter_deletes) {
-  MemTableInserter inserter(WriteBatchInternal::Sequence(b), mem, opts, db,
-                            filter_deletes);
+Status WriteBatchInternal::InsertInto(const WriteBatch* b,
+                                      ColumnFamilyMemTables* memtables,
+                                      bool recovery, uint64_t log_number,
+                                      DB* db, const bool dont_filter_deletes) {
+  MemTableInserter inserter(WriteBatchInternal::Sequence(b), memtables,
+                            recovery, log_number, db, dont_filter_deletes);
   return b->Iterate(&inserter);
 }
 

db/write_batch_internal.h

@@ -17,10 +17,66 @@ namespace rocksdb {
 
 class MemTable;
 
+class ColumnFamilyMemTables {
+ public:
+  virtual ~ColumnFamilyMemTables() {}
+  virtual bool Seek(uint32_t column_family_id) = 0;
+  // returns true if the update to memtable should be ignored
+  // (useful when recovering from log whose updates have already
+  // been processed)
+  virtual uint64_t GetLogNumber() const = 0;
+  virtual MemTable* GetMemTable() const = 0;
+  virtual const Options* GetOptions() const = 0;
+  virtual ColumnFamilyHandle* GetColumnFamilyHandle() = 0;
+};
+
+class ColumnFamilyMemTablesDefault : public ColumnFamilyMemTables {
+ public:
+  ColumnFamilyMemTablesDefault(MemTable* mem, const Options* options)
+      : ok_(false), mem_(mem), options_(options) {}
+
+  bool Seek(uint32_t column_family_id) override {
+    ok_ = (column_family_id == 0);
+    return ok_;
+  }
+
+  uint64_t GetLogNumber() const override { return 0; }
+
+  MemTable* GetMemTable() const override {
+    assert(ok_);
+    return mem_;
+  }
+
+  const Options* GetOptions() const override {
+    assert(ok_);
+    return options_;
+  }
+
+  ColumnFamilyHandle* GetColumnFamilyHandle() override { return nullptr; }
+
+ private:
+  bool ok_;
+  MemTable* mem_;
+  const Options* const options_;
+};
+
 // WriteBatchInternal provides static methods for manipulating a
 // WriteBatch that we don't want in the public WriteBatch interface.
 class WriteBatchInternal {
  public:
+  // WriteBatch methods with column_family_id instead of ColumnFamilyHandle*
+  static void Put(WriteBatch* batch, uint32_t column_family_id,
+                  const Slice& key, const Slice& value);
+
+  static void Put(WriteBatch* batch, uint32_t column_family_id,
+                  const SliceParts& key, const SliceParts& value);
+
+  static void Delete(WriteBatch* batch, uint32_t column_family_id,
+                     const Slice& key);
+
+  static void Merge(WriteBatch* batch, uint32_t column_family_id,
+                    const Slice& key, const Slice& value);
+
   // Return the number of entries in the batch.
   static int Count(const WriteBatch* batch);
 
@@ -45,11 +101,21 @@ class WriteBatchInternal {
   static void SetContents(WriteBatch* batch, const Slice& contents);
 
   // Inserts batch entries into memtable
-  // Drops deletes in batch if filter_del is set to true and
-  // db->KeyMayExist returns false
-  static Status InsertInto(const WriteBatch* batch, MemTable* memtable,
-                           const Options* opts, DB* db = nullptr,
-                           const bool filter_del = false);
+  // If dont_filter_deletes is false AND options.filter_deletes is true,
+  // then --> Drops deletes in batch if db->KeyMayExist returns false
+  // If recovery == true, this means InsertInto is executed on a recovery
+  // code-path. WriteBatch referencing a dropped column family can be
+  // found on a recovery code-path and should be ignored (recovery should not
+  // fail). Additionally, the memtable will be updated only if
+  // memtables->GetLogNumber() >= log_number
+  // However, if recovery == false, any WriteBatch referencing
+  // non-existing column family will return a failure. Also, log_number is
+  // ignored in that case
+  static Status InsertInto(const WriteBatch* batch,
+                           ColumnFamilyMemTables* memtables,
+                           bool recovery = false, uint64_t log_number = 0,
+                           DB* db = nullptr,
+                           const bool dont_filter_deletes = true);
 
   static void Append(WriteBatch* dst, const WriteBatch* src);
 };

db/write_batch_test.cc

@@ -11,6 +11,7 @@
 
 #include <memory>
 #include "db/memtable.h"
+#include "db/column_family.h"
 #include "db/write_batch_internal.h"
 #include "rocksdb/env.h"
 #include "rocksdb/memtablerep.h"
@@ -27,9 +28,10 @@ static std::string PrintContents(WriteBatch* b) {
   MemTable* mem = new MemTable(cmp, options);
   mem->Ref();
   std::string state;
-  Status s = WriteBatchInternal::InsertInto(b, mem, &options);
+  ColumnFamilyMemTablesDefault cf_mems_default(mem, &options);
+  Status s = WriteBatchInternal::InsertInto(b, &cf_mems_default);
   int count = 0;
-  Iterator* iter = mem->NewIterator();
+  Iterator* iter = mem->NewIterator(ReadOptions());
   for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
     ParsedInternalKey ikey;
     memset((void *)&ikey, 0, sizeof(ikey));
@@ -144,17 +146,37 @@ TEST(WriteBatchTest, Append) {
 namespace {
   struct TestHandler : public WriteBatch::Handler {
     std::string seen;
-    virtual void Put(const Slice& key, const Slice& value) {
-      seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
+    virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+                       const Slice& value) {
+      if (column_family_id == 0) {
+        seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
+      } else {
+        seen += "PutCF(" + std::to_string(column_family_id) + ", " +
+                key.ToString() + ", " + value.ToString() + ")";
+      }
+      return Status::OK();
     }
-    virtual void Merge(const Slice& key, const Slice& value) {
-      seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
+    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+                         const Slice& value) {
+      if (column_family_id == 0) {
+        seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
+      } else {
+        seen += "MergeCF(" + std::to_string(column_family_id) + ", " +
+                key.ToString() + ", " + value.ToString() + ")";
+      }
+      return Status::OK();
     }
     virtual void LogData(const Slice& blob) {
       seen += "LogData(" + blob.ToString() + ")";
     }
-    virtual void Delete(const Slice& key) {
-      seen += "Delete(" + key.ToString() + ")";
+    virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
+      if (column_family_id == 0) {
+        seen += "Delete(" + key.ToString() + ")";
+      } else {
+        seen += "DeleteCF(" + std::to_string(column_family_id) + ", " +
+                key.ToString() + ")";
+      }
+      return Status::OK();
     }
   };
 }
@@ -194,21 +216,23 @@ TEST(WriteBatchTest, Continue) {
 
   struct Handler : public TestHandler {
     int num_seen = 0;
-    virtual void Put(const Slice& key, const Slice& value) {
+    virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+                       const Slice& value) {
       ++num_seen;
-      TestHandler::Put(key, value);
+      return TestHandler::PutCF(column_family_id, key, value);
     }
-    virtual void Merge(const Slice& key, const Slice& value) {
+    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+                         const Slice& value) {
       ++num_seen;
-      TestHandler::Merge(key, value);
+      return TestHandler::MergeCF(column_family_id, key, value);
     }
     virtual void LogData(const Slice& blob) {
       ++num_seen;
       TestHandler::LogData(blob);
     }
-    virtual void Delete(const Slice& key) {
+    virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
       ++num_seen;
-      TestHandler::Delete(key);
+      return TestHandler::DeleteCF(column_family_id, key);
     }
     virtual bool Continue() override {
       return num_seen < 3;
@@ -256,6 +280,42 @@ TEST(WriteBatchTest, PutGatherSlices) {
   ASSERT_EQ(3, batch.Count());
 }
 
+namespace {
+class ColumnFamilyHandleImplDummy : public ColumnFamilyHandleImpl {
+ public:
+  explicit ColumnFamilyHandleImplDummy(int id)
+      : ColumnFamilyHandleImpl(nullptr, nullptr, nullptr), id_(id) {}
+  uint32_t GetID() const override { return id_; }
+
+ private:
+  uint32_t id_;
+};
+}  // namespace anonymous
+
+TEST(WriteBatchTest, ColumnFamiliesBatchTest) {
+  WriteBatch batch;
+  ColumnFamilyHandleImplDummy zero(0), two(2), three(3), eight(8);
+  batch.Put(&zero, Slice("foo"), Slice("bar"));
+  batch.Put(&two, Slice("twofoo"), Slice("bar2"));
+  batch.Put(&eight, Slice("eightfoo"), Slice("bar8"));
+  batch.Delete(&eight, Slice("eightfoo"));
+  batch.Merge(&three, Slice("threethree"), Slice("3three"));
+  batch.Put(&zero, Slice("foo"), Slice("bar"));
+  batch.Merge(Slice("omom"), Slice("nom"));
+
+  TestHandler handler;
+  batch.Iterate(&handler);
+  ASSERT_EQ(
+      "Put(foo, bar)"
+      "PutCF(2, twofoo, bar2)"
+      "PutCF(8, eightfoo, bar8)"
+      "DeleteCF(8, eightfoo)"
+      "MergeCF(3, threethree, 3three)"
+      "Put(foo, bar)"
+      "Merge(omom, nom)",
+      handler.seen);
+}
+
 }  // namespace rocksdb
 
 int main(int argc, char** argv) {

helpers/memenv/memenv.cc

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "helpers/memenv/memenv.h"
-
 #include "rocksdb/env.h"
 #include "rocksdb/status.h"
 #include "port/port.h"

helpers/memenv/memenv.h

@@ -1,19 +0,0 @@
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#ifndef STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
-#define STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
-namespace rocksdb {
-
-class Env;
-
-// Returns a new environment that stores its data in memory and delegates
-// all non-file-storage tasks to base_env. The caller must delete the result
-// when it is no longer needed.
-// *base_env must remain live while the result is in use.
-Env* NewMemEnv(Env* base_env);
-
-}  // namespace rocksdb
-
-#endif  // STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_

helpers/memenv/memenv_test.cc

@@ -2,8 +2,6 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "helpers/memenv/memenv.h"
-
 #include "db/db_impl.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"

include/rocksdb/c.h

@@ -83,6 +83,12 @@ extern rocksdb_t* rocksdb_open(
     const char* name,
     char** errptr);
 
+extern rocksdb_t* rocksdb_open_for_read_only(
+    const rocksdb_options_t* options,
+    const char* name,
+    unsigned char error_if_log_file_exist,
+    char** errptr);
+
 extern void rocksdb_close(rocksdb_t* db);
 
 extern void rocksdb_put(
@@ -463,13 +469,9 @@ extern void rocksdb_readoptions_set_verify_checksums(
     unsigned char);
 extern void rocksdb_readoptions_set_fill_cache(
     rocksdb_readoptions_t*, unsigned char);
-extern void rocksdb_readoptions_set_prefix_seek(
-    rocksdb_readoptions_t*, unsigned char);
 extern void rocksdb_readoptions_set_snapshot(
     rocksdb_readoptions_t*,
     const rocksdb_snapshot_t*);
-extern void rocksdb_readoptions_set_prefix(
-    rocksdb_readoptions_t*, const char* key, size_t keylen);
 extern void rocksdb_readoptions_set_read_tier(
     rocksdb_readoptions_t*, int);
 extern void rocksdb_readoptions_set_tailing(

include/rocksdb/cache.h

@@ -116,6 +116,12 @@ class Cache {
     // default implementation is noop
   };
 
+  // Apply callback to all entries in the cache
+  // If thread_safe is true, it will also lock the accesses. Otherwise, it will
+  // access the cache without the lock held
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) = 0;
+
  private:
   void LRU_Remove(Handle* e);
   void LRU_Append(Handle* e);

include/rocksdb/db.h

@@ -13,7 +13,9 @@
 #include <stdio.h>
 #include <memory>
 #include <vector>
+#include <string>
 #include <unordered_map>
+#include "rocksdb/version.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/options.h"
 #include "rocksdb/types.h"
@@ -23,9 +25,24 @@ namespace rocksdb {
 
 using std::unique_ptr;
 
-// Update Makefile if you change these
-static const int kMajorVersion = 2;
-static const int kMinorVersion = 0;
+class ColumnFamilyHandle {
+ public:
+  virtual ~ColumnFamilyHandle() {}
+};
+extern const std::string kDefaultColumnFamilyName;
+
+struct ColumnFamilyDescriptor {
+  std::string name;
+  ColumnFamilyOptions options;
+  ColumnFamilyDescriptor()
+      : name(kDefaultColumnFamilyName), options(ColumnFamilyOptions()) {}
+  ColumnFamilyDescriptor(const std::string& _name,
+                         const ColumnFamilyOptions& _options)
+      : name(_name), options(_options) {}
+};
+
+static const int kMajorVersion = __ROCKSDB_MAJOR__;
+static const int kMinorVersion = __ROCKSDB_MINOR__;
 
 struct Options;
 struct ReadOptions;
@@ -37,7 +54,8 @@ class Env;
 
 // Metadata associated with each SST file.
 struct LiveFileMetaData {
-  std::string name;        // Name of the file
+  std::string column_family_name;  // Name of the column family
+  std::string name;                // Name of the file
   int level;               // Level at which this file resides.
   size_t size;             // File size in bytes.
   std::string smallestkey; // Smallest user defined key in the file.
@@ -91,29 +109,88 @@ class DB {
       const std::string& name, DB** dbptr,
       bool error_if_log_file_exist = false);
 
+  // Open the database for read only with column families. When opening DB with
+  // read only, you can specify only a subset of column families in the
+  // database that should be opened. However, you always need to specify default
+  // column family. The default column family name is 'default' and it's stored
+  // in rocksdb::kDefaultColumnFamilyName
+  static Status OpenForReadOnly(
+      const DBOptions& db_options, const std::string& name,
+      const std::vector<ColumnFamilyDescriptor>& column_families,
+      std::vector<ColumnFamilyHandle*>* handles, DB** dbptr,
+      bool error_if_log_file_exist = false);
+
+  // Open DB with column families.
+  // db_options specify database specific options
+  // column_families is the vector of all column families in the databse,
+  // containing column family name and options. You need to open ALL column
+  // families in the database. To get the list of column families, you can use
+  // ListColumnFamilies(). Also, you can open only a subset of column families
+  // for read-only access.
+  // The default column family name is 'default' and it's stored
+  // in rocksdb::kDefaultColumnFamilyName.
+  // If everything is OK, handles will on return be the same size
+  // as column_families --- handles[i] will be a handle that you
+  // will use to operate on column family column_family[i]
+  static Status Open(const DBOptions& db_options, const std::string& name,
+                     const std::vector<ColumnFamilyDescriptor>& column_families,
+                     std::vector<ColumnFamilyHandle*>* handles, DB** dbptr);
+
+  // ListColumnFamilies will open the DB specified by argument name
+  // and return the list of all column families in that DB
+  // through column_families argument. The ordering of
+  // column families in column_families is unspecified.
+  static Status ListColumnFamilies(const DBOptions& db_options,
+                                   const std::string& name,
+                                   std::vector<std::string>* column_families);
+
   DB() { }
   virtual ~DB();
 
+  // Create a column_family and return the handle of column family
+  // through the argument handle.
+  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
+                                    const std::string& column_family_name,
+                                    ColumnFamilyHandle** handle);
+
+  // Drop a column family specified by column_family handle. This call
+  // only records a drop record in the manifest and prevents the column
+  // family from flushing and compacting.
+  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family);
+
   // Set the database entry for "key" to "value".
   // Returns OK on success, and a non-OK status on error.
   // Note: consider setting options.sync = true.
   virtual Status Put(const WriteOptions& options,
-                     const Slice& key,
+                     ColumnFamilyHandle* column_family, const Slice& key,
                      const Slice& value) = 0;
+  virtual Status Put(const WriteOptions& options, const Slice& key,
+                     const Slice& value) {
+    return Put(options, DefaultColumnFamily(), key, value);
+  }
 
   // Remove the database entry (if any) for "key".  Returns OK on
   // success, and a non-OK status on error.  It is not an error if "key"
   // did not exist in the database.
   // Note: consider setting options.sync = true.
-  virtual Status Delete(const WriteOptions& options, const Slice& key) = 0;
+  virtual Status Delete(const WriteOptions& options,
+                        ColumnFamilyHandle* column_family,
+                        const Slice& key) = 0;
+  virtual Status Delete(const WriteOptions& options, const Slice& key) {
+    return Delete(options, DefaultColumnFamily(), key);
+  }
 
   // Merge the database entry for "key" with "value".  Returns OK on success,
   // and a non-OK status on error. The semantics of this operation is
   // determined by the user provided merge_operator when opening DB.
   // Note: consider setting options.sync = true.
   virtual Status Merge(const WriteOptions& options,
-                       const Slice& key,
+                       ColumnFamilyHandle* column_family, const Slice& key,
                        const Slice& value) = 0;
+  virtual Status Merge(const WriteOptions& options, const Slice& key,
+                       const Slice& value) {
+    return Merge(options, DefaultColumnFamily(), key, value);
+  }
 
   // Apply the specified updates to the database.
   // Returns OK on success, non-OK on failure.
@@ -128,8 +205,11 @@ class DB {
   //
   // May return some other Status on an error.
   virtual Status Get(const ReadOptions& options,
-                     const Slice& key,
+                     ColumnFamilyHandle* column_family, const Slice& key,
                      std::string* value) = 0;
+  virtual Status Get(const ReadOptions& options, const Slice& key, std::string* value) {
+    return Get(options, DefaultColumnFamily(), key, value);
+  }
 
   // If keys[i] does not exist in the database, then the i'th returned
   // status will be one for which Status::IsNotFound() is true, and
@@ -141,9 +221,17 @@ class DB {
   // Similarly, the number of returned statuses will be the number of keys.
   // Note: keys will not be "de-duplicated". Duplicate keys will return
   // duplicate values in order.
+  virtual std::vector<Status> MultiGet(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_family,
+      const std::vector<Slice>& keys, std::vector<std::string>* values) = 0;
   virtual std::vector<Status> MultiGet(const ReadOptions& options,
                                        const std::vector<Slice>& keys,
-                                       std::vector<std::string>* values) = 0;
+                                       std::vector<std::string>* values) {
+    return MultiGet(options, std::vector<ColumnFamilyHandle*>(
+                                 keys.size(), DefaultColumnFamily()),
+                    keys, values);
+  }
 
   // If the key definitely does not exist in the database, then this method
   // returns false, else true. If the caller wants to obtain value when the key
@@ -153,14 +241,17 @@ class DB {
   // to make this lighter weight is to avoid doing any IOs.
   // Default implementation here returns true and sets 'value_found' to false
   virtual bool KeyMayExist(const ReadOptions& options,
-                           const Slice& key,
-                           std::string* value,
-                           bool* value_found = nullptr) {
+                           ColumnFamilyHandle* column_family, const Slice& key,
+                           std::string* value, bool* value_found = nullptr) {
     if (value_found != nullptr) {
       *value_found = false;
     }
     return true;
   }
+  virtual bool KeyMayExist(const ReadOptions& options, const Slice& key,
+                           std::string* value, bool* value_found = nullptr) {
+    return KeyMayExist(options, DefaultColumnFamily(), key, value, value_found);
+  }
 
   // Return a heap-allocated iterator over the contents of the database.
   // The result of NewIterator() is initially invalid (caller must
@@ -168,12 +259,26 @@ class DB {
   //
   // Caller should delete the iterator when it is no longer needed.
   // The returned iterator should be deleted before this db is deleted.
-  virtual Iterator* NewIterator(const ReadOptions& options) = 0;
+  virtual Iterator* NewIterator(const ReadOptions& options,
+                                ColumnFamilyHandle* column_family) = 0;
+  virtual Iterator* NewIterator(const ReadOptions& options) {
+    return NewIterator(options, DefaultColumnFamily());
+  }
+  // Returns iterators from a consistent database state across multiple
+  // column families. Iterators are heap allocated and need to be deleted
+  // before the db is deleted
+  virtual Status NewIterators(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_families,
+      std::vector<Iterator*>* iterators) = 0;
 
   // Return a handle to the current DB state.  Iterators created with
   // this handle will all observe a stable snapshot of the current DB
   // state.  The caller must call ReleaseSnapshot(result) when the
   // snapshot is no longer needed.
+  //
+  // nullptr will be returned if the DB fails to take a snapshot or does
+  // not support snapshot.
   virtual const Snapshot* GetSnapshot() = 0;
 
   // Release a previously acquired snapshot.  The caller must not
@@ -194,7 +299,11 @@ class DB {
   //     about the internal operation of the DB.
   //  "rocksdb.sstables" - returns a multi-line string that describes all
   //     of the sstables that make up the db contents.
-  virtual bool GetProperty(const Slice& property, std::string* value) = 0;
+  virtual bool GetProperty(ColumnFamilyHandle* column_family,
+                           const Slice& property, std::string* value) = 0;
+  virtual bool GetProperty(const Slice& property, std::string* value) {
+    return GetProperty(DefaultColumnFamily(), property, value);
+  }
 
   // For each i in [0,n-1], store in "sizes[i]", the approximate
   // file system space used by keys in "[range[i].start .. range[i].limit)".
@@ -204,8 +313,12 @@ class DB {
   // sizes will be one-tenth the size of the corresponding user data size.
   //
   // The results may not include the sizes of recently written data.
-  virtual void GetApproximateSizes(const Range* range, int n,
+  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
+                                   const Range* range, int n,
                                    uint64_t* sizes) = 0;
+  virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes) {
+    GetApproximateSizes(DefaultColumnFamily(), range, n, sizes);
+  }
 
   // Compact the underlying storage for the key range [*begin,*end].
   // The actual compaction interval might be superset of [*begin, *end].
@@ -224,19 +337,33 @@ class DB {
   // hosting all the files. In this case, client could set reduce_level
   // to true, to move the files back to the minimum level capable of holding
   // the data set or a given level (specified by non-negative target_level).
-  virtual Status CompactRange(const Slice* begin, const Slice* end,
+  virtual Status CompactRange(ColumnFamilyHandle* column_family,
+                              const Slice* begin, const Slice* end,
                               bool reduce_level = false,
                               int target_level = -1) = 0;
+  virtual Status CompactRange(const Slice* begin, const Slice* end,
+                              bool reduce_level = false,
+                              int target_level = -1) {
+    return CompactRange(DefaultColumnFamily(), begin, end, reduce_level,
+                        target_level);
+  }
 
   // Number of levels used for this DB.
-  virtual int NumberLevels() = 0;
+  virtual int NumberLevels(ColumnFamilyHandle* column_family) = 0;
+  virtual int NumberLevels() { return NumberLevels(DefaultColumnFamily()); }
 
   // Maximum level to which a new compacted memtable is pushed if it
   // does not create overlap.
-  virtual int MaxMemCompactionLevel() = 0;
+  virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family) = 0;
+  virtual int MaxMemCompactionLevel() {
+    return MaxMemCompactionLevel(DefaultColumnFamily());
+  }
 
   // Number of files in level-0 that would stop writes.
-  virtual int Level0StopWriteTrigger() = 0;
+  virtual int Level0StopWriteTrigger(ColumnFamilyHandle* column_family) = 0;
+  virtual int Level0StopWriteTrigger() {
+    return Level0StopWriteTrigger(DefaultColumnFamily());
+  }
 
   // Get DB name -- the exact same name that was provided as an argument to
   // DB::Open()
@@ -246,10 +373,23 @@ class DB {
   virtual Env* GetEnv() const = 0;
 
   // Get DB Options that we use
-  virtual const Options& GetOptions() const = 0;
+  virtual const Options& GetOptions(ColumnFamilyHandle* column_family)
+      const = 0;
+  virtual const Options& GetOptions() const {
+    return GetOptions(DefaultColumnFamily());
+  }
 
   // Flush all mem-table data.
-  virtual Status Flush(const FlushOptions& options) = 0;
+  virtual Status Flush(const FlushOptions& options,
+                       ColumnFamilyHandle* column_family) = 0;
+  virtual Status Flush(const FlushOptions& options) {
+    return Flush(options, DefaultColumnFamily());
+  }
+
+  // The sequence number of the most recent transaction.
+  virtual SequenceNumber GetLatestSequenceNumber() const = 0;
+
+#ifndef ROCKSDB_LITE
 
   // Prevent file deletions. Compactions will continue to occur,
   // but no obsolete files will be deleted. Calling this multiple
@@ -279,9 +419,12 @@ class DB {
   // Setting flush_memtable to true does Flush before recording the live files.
   // Setting flush_memtable to false is useful when we don't want to wait for
   // flush which may have to wait for compaction to complete taking an
-  // indeterminate time. But this will have to use GetSortedWalFiles after
-  // GetLiveFiles to compensate for memtables missed in this snapshot due to the
-  // absence of Flush, by WAL files to recover the database consistently later
+  // indeterminate time.
+  //
+  // In case you have multiple column families, even if flush_memtable is true,
+  // you still need to call GetSortedWalFiles after GetLiveFiles to compensate
+  // for new data that arrived to already-flushed column families while other
+  // column families were flushing
   virtual Status GetLiveFiles(std::vector<std::string>&,
                               uint64_t* manifest_file_size,
                               bool flush_memtable = true) = 0;
@@ -289,9 +432,6 @@ class DB {
   // Retrieve the sorted list of all wal files with earliest file first
   virtual Status GetSortedWalFiles(VectorLogPtr& files) = 0;
 
-  // The sequence number of the most recent transaction.
-  virtual SequenceNumber GetLatestSequenceNumber() const = 0;
-
   // Sets iter to an iterator that is positioned at a write-batch containing
   // seq_number. If the sequence number is non existent, it returns an iterator
   // at the first available seq_no after the requested seq_no
@@ -314,12 +454,23 @@ class DB {
   // and end key
   virtual void GetLiveFilesMetaData(std::vector<LiveFileMetaData>* metadata) {}
 
+#endif  // ROCKSDB_LITE
+
   // Sets the globally unique ID created at database creation time by invoking
   // Env::GenerateUniqueId(), in identity. Returns Status::OK if identity could
   // be set properly
   virtual Status GetDbIdentity(std::string& identity) = 0;
 
-  virtual Status GetPropertiesOfAllTables(TablePropertiesCollection* props) = 0;
+  // Returns default column family handle
+  virtual ColumnFamilyHandle* DefaultColumnFamily() const = 0;
+
+#ifndef ROCKSDB_LITE
+  virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
+                                          TablePropertiesCollection* props) = 0;
+  virtual Status GetPropertiesOfAllTables(TablePropertiesCollection* props) {
+    return GetPropertiesOfAllTables(DefaultColumnFamily(), props);
+  }
+#endif  // ROCKSDB_LITE
 
  private:
   // No copying allowed
@@ -331,11 +482,13 @@ class DB {
 // Be very careful using this method.
 Status DestroyDB(const std::string& name, const Options& options);
 
+#ifndef ROCKSDB_LITE
 // If a DB cannot be opened, you may attempt to call this method to
 // resurrect as much of the contents of the database as possible.
 // Some data may be lost, so be careful when calling this function
 // on a database that contains important information.
 Status RepairDB(const std::string& dbname, const Options& options);
+#endif
 
 }  // namespace rocksdb
 

include/rocksdb/env.h

@@ -34,7 +34,7 @@ class Slice;
 class WritableFile;
 class RandomRWFile;
 class Directory;
-struct Options;
+struct DBOptions;
 
 using std::unique_ptr;
 using std::shared_ptr;
@@ -47,7 +47,7 @@ struct EnvOptions {
   EnvOptions();
 
   // construct from Options
-  explicit EnvOptions(const Options& options);
+  explicit EnvOptions(const DBOptions& options);
 
   // If true, then allow caching of data in environment buffers
   bool use_os_buffer = true;
@@ -535,11 +535,11 @@ class Directory {
 };
 
 enum InfoLogLevel : unsigned char {
-  DEBUG = 0,
-  INFO,
-  WARN,
-  ERROR,
-  FATAL,
+  DEBUG_LEVEL = 0,
+  INFO_LEVEL,
+  WARN_LEVEL,
+  ERROR_LEVEL,
+  FATAL_LEVEL,
   NUM_INFO_LOG_LEVELS,
 };
 
@@ -547,7 +547,7 @@ enum InfoLogLevel : unsigned char {
 class Logger {
  public:
   enum { DO_NOT_SUPPORT_GET_LOG_FILE_SIZE = -1 };
-  explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO)
+  explicit Logger(const InfoLogLevel log_level = InfoLogLevel::INFO_LEVEL)
       : log_level_(log_level) {}
   virtual ~Logger();
 
@@ -565,7 +565,7 @@ class Logger {
       return;
     }
 
-    if (log_level == INFO) {
+    if (log_level == InfoLogLevel::INFO_LEVEL) {
       // Doesn't print log level if it is INFO level.
       // This is to avoid unexpected performance regression after we add
       // the feature of log level. All the logs before we add the feature
@@ -649,7 +649,8 @@ extern void Fatal(Logger* info_log, const char* format, ...);
 
 // A utility routine: write "data" to the named file.
 extern Status WriteStringToFile(Env* env, const Slice& data,
-                                const std::string& fname);
+                                const std::string& fname,
+                                bool should_sync = false);
 
 // A utility routine: read contents of named file into *data
 extern Status ReadFileToString(Env* env, const std::string& fname,
@@ -760,6 +761,12 @@ class EnvWrapper : public Env {
   Env* target_;
 };
 
+// Returns a new environment that stores its data in memory and delegates
+// all non-file-storage tasks to base_env. The caller must delete the result
+// when it is no longer needed.
+// *base_env must remain live while the result is in use.
+Env* NewMemEnv(Env* base_env);
+
 }  // namespace rocksdb
 
 #endif  // STORAGE_ROCKSDB_INCLUDE_ENV_H_

include/rocksdb/ldb_tool.h

@@ -2,8 +2,8 @@
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
-#ifndef STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
-#define STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
+#ifndef ROCKSDB_LITE
+#pragma once
 #include "rocksdb/options.h"
 
 namespace rocksdb {
@@ -15,4 +15,4 @@ class LDBTool {
 
 } // namespace rocksdb
 
-#endif // STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
+#endif  // ROCKSDB_LITE

include/rocksdb/memtablerep.h

@@ -148,17 +148,18 @@ class MemTableRep {
   // GetIterator().
   virtual Iterator* GetIterator(const Slice& user_key) { return GetIterator(); }
 
-  // Return an iterator over at least the keys with the specified prefix. The
-  // iterator may also allow access to other keys, but doesn't have to. Default:
-  // GetIterator().
-  virtual Iterator* GetPrefixIterator(const Slice& prefix) {
-    return GetIterator();
-  }
-
   // Return an iterator that has a special Seek semantics. The result of
   // a Seek might only include keys with the same prefix as the target key.
   virtual Iterator* GetDynamicPrefixIterator() { return GetIterator(); }
 
+  // Return true if the current MemTableRep supports merge operator.
+  // Default: true
+  virtual bool IsMergeOperatorSupported() const { return true; }
+
+  // Return true if the current MemTableRep supports snapshot
+  // Default: true
+  virtual bool IsSnapshotSupported() const { return true; }
+
  protected:
   // When *key is an internal key concatenated with the value, returns the
   // user key.
@@ -177,6 +178,16 @@ class MemTableRepFactory {
   virtual const char* Name() const = 0;
 };
 
+// This uses a skip list to store keys. It is the default.
+class SkipListFactory : public MemTableRepFactory {
+ public:
+  virtual MemTableRep* CreateMemTableRep(const MemTableRep::KeyComparator&,
+                                         Arena*,
+                                         const SliceTransform*) override;
+  virtual const char* Name() const override { return "SkipListFactory"; }
+};
+
+#ifndef ROCKSDB_LITE
 // This creates MemTableReps that are backed by an std::vector. On iteration,
 // the vector is sorted. This is useful for workloads where iteration is very
 // rare and writes are generally not issued after reads begin.
@@ -198,17 +209,6 @@ class VectorRepFactory : public MemTableRepFactory {
   }
 };
 
-// This uses a skip list to store keys. It is the default.
-class SkipListFactory : public MemTableRepFactory {
- public:
-  virtual MemTableRep* CreateMemTableRep(
-      const MemTableRep::KeyComparator&, Arena*,
-      const SliceTransform*) override;
-  virtual const char* Name() const override {
-    return "SkipListFactory";
-  }
-};
-
 // This class contains a fixed array of buckets, each
 // pointing to a skiplist (null if the bucket is empty).
 // bucket_count: number of fixed array buckets
@@ -223,8 +223,48 @@ extern MemTableRepFactory* NewHashSkipListRepFactory(
 // The factory is to create memtables with a hashed linked list:
 // it contains a fixed array of buckets, each pointing to a sorted single
 // linked list (null if the bucket is empty).
-// bucket_count: number of fixed array buckets
+// @bucket_count: number of fixed array buckets
+// @huge_page_tlb_size: if <=0, allocate the hash table bytes from malloc.
+//                      Otherwise from huge page TLB. The user needs to reserve
+//                      huge pages for it to be allocated, like:
+//                          sysctl -w vm.nr_hugepages=20
+//                      See linux doc Documentation/vm/hugetlbpage.txt
 extern MemTableRepFactory* NewHashLinkListRepFactory(
-    size_t bucket_count = 50000);
+    size_t bucket_count = 50000, size_t huge_page_tlb_size = 0);
 
+// This factory creates a cuckoo-hashing based mem-table representation.
+// Cuckoo-hash is a closed-hash strategy, in which all key/value pairs
+// are stored in the bucket array itself intead of in some data structures
+// external to the bucket array.  In addition, each key in cuckoo hash
+// has a constant number of possible buckets in the bucket array.  These
+// two properties together makes cuckoo hash more memory efficient and
+// a constant worst-case read time.  Cuckoo hash is best suitable for
+// point-lookup workload.
+//
+// When inserting a key / value, it first checks whether one of its possible
+// buckets is empty.  If so, the key / value will be inserted to that vacant
+// bucket.  Otherwise, one of the keys originally stored in one of these
+// possible buckets will be "kicked out" and move to one of its possible
+// buckets (and possibly kicks out another victim.)  In the current
+// implementation, such "kick-out" path is bounded.  If it cannot find a
+// "kick-out" path for a specific key, this key will be stored in a backup
+// structure, and the current memtable to be forced to immutable.
+//
+// Note that currently this mem-table representation does not support
+// snapshot (i.e., it only queries latest state) and iterators.  In addition,
+// MultiGet operation might also lose its atomicity due to the lack of
+// snapshot support.
+//
+// Parameters:
+//   write_buffer_size: the write buffer size in bytes.
+//   average_data_size: the average size of key + value in bytes.  This value
+//     together with write_buffer_size will be used to compute the number
+//     of buckets.
+//   hash_function_count: the number of hash functions that will be used by
+//     the cuckoo-hash.  The number also equals to the number of possible
+//     buckets each key will have.
+extern MemTableRepFactory* NewHashCuckooRepFactory(
+    size_t write_buffer_size, size_t average_data_size = 64,
+    unsigned int hash_function_count = 4);
+#endif  // ROCKSDB_LITE
 }  // namespace rocksdb

include/rocksdb/options.h

@@ -15,6 +15,7 @@
 #include <vector>
 #include <stdint.h>
 
+#include "rocksdb/version.h"
 #include "rocksdb/universal_compaction.h"
 
 namespace rocksdb {
@@ -62,8 +63,8 @@ struct CompressionOptions {
   int level;
   int strategy;
   CompressionOptions() : window_bits(-14), level(-1), strategy(0) {}
-  CompressionOptions(int wbits, int lev, int strategy)
-      : window_bits(wbits), level(lev), strategy(strategy) {}
+  CompressionOptions(int wbits, int _lev, int _strategy)
+      : window_bits(wbits), level(_lev), strategy(_strategy) {}
 };
 
 enum UpdateStatus {    // Return status For inplace update callback
@@ -72,8 +73,9 @@ enum UpdateStatus {    // Return status For inplace update callback
   UPDATED         = 2, // No inplace update. Merged value set
 };
 
-// Options to control the behavior of a database (passed to DB::Open)
-struct Options {
+struct Options;
+
+struct ColumnFamilyOptions {
   // -------------------
   // Parameters that affect behavior
 
@@ -130,38 +132,6 @@ struct Options {
   // Default: a factory that doesn't provide any object
   std::shared_ptr<CompactionFilterFactoryV2> compaction_filter_factory_v2;
 
-  // If true, the database will be created if it is missing.
-  // Default: false
-  bool create_if_missing;
-
-  // If true, an error is raised if the database already exists.
-  // Default: false
-  bool error_if_exists;
-
-  // If true, the implementation will do aggressive checking of the
-  // data it is processing and will stop early if it detects any
-  // errors.  This may have unforeseen ramifications: for example, a
-  // corruption of one DB entry may cause a large number of entries to
-  // become unreadable or for the entire DB to become unopenable.
-  // If any of the writes to the database fails (Put, Delete, Merge, Write),
-  // the database will switch to read-only mode and fail all other
-  // Write operations.
-  // Default: true
-  bool paranoid_checks;
-
-  // Use the specified object to interact with the environment,
-  // e.g. to read/write files, schedule background work, etc.
-  // Default: Env::Default()
-  Env* env;
-
-  // Any internal progress/error information generated by the db will
-  // be written to info_log if it is non-nullptr, or to a file stored
-  // in the same directory as the DB contents if info_log is nullptr.
-  // Default: nullptr
-  shared_ptr<Logger> info_log;
-
-  InfoLogLevel info_log_level;
-
   // -------------------
   // Parameters that affect performance
 
@@ -193,15 +163,6 @@ struct Options {
   // individual write buffers.  Default: 1
   int min_write_buffer_number_to_merge;
 
-  // Number of open files that can be used by the DB.  You may need to
-  // increase this if your database has a large working set. Value -1 means
-  // files opened are always kept open. You can estimate number of files based
-  // on target_file_size_base and target_file_size_multiplier for level-based
-  // compaction. For universal-style compaction, you can usually set it to -1.
-  //
-  // Default: 5000
-  int max_open_files;
-
   // Control over blocks (user data is stored in a set of blocks, and
   // a block is the unit of reading from disk).
 
@@ -297,6 +258,8 @@ struct Options {
 
   // Number of files to trigger level-0 compaction. A value <0 means that
   // level-0 compaction will not be triggered by number of files at all.
+  //
+  // Default: 4
   int level0_file_num_compaction_trigger;
 
   // Soft limit on number of level-0 files. We start slowing down writes at this
@@ -369,93 +332,12 @@ struct Options {
   // stop building a single file in a level->level+1 compaction.
   int max_grandparent_overlap_factor;
 
-  // If non-null, then we should collect metrics about database operations
-  // Statistics objects should not be shared between DB instances as
-  // it does not use any locks to prevent concurrent updates.
-  shared_ptr<Statistics> statistics;
-
-  // If true, then the contents of data files are not synced
-  // to stable storage. Their contents remain in the OS buffers till the
-  // OS decides to flush them. This option is good for bulk-loading
-  // of data. Once the bulk-loading is complete, please issue a
-  // sync to the OS to flush all dirty buffesrs to stable storage.
-  // Default: false
-  bool disableDataSync;
-
-  // If true, then every store to stable storage will issue a fsync.
-  // If false, then every store to stable storage will issue a fdatasync.
-  // This parameter should be set to true while storing data to
-  // filesystem like ext3 that can lose files after a reboot.
-  // Default: false
-  bool use_fsync;
-
-  // This number controls how often a new scribe log about
-  // db deploy stats is written out.
-  // -1 indicates no logging at all.
-  // Default value is 1800 (half an hour).
-  int db_stats_log_interval;
-
-  // This specifies the info LOG dir.
-  // If it is empty, the log files will be in the same dir as data.
-  // If it is non empty, the log files will be in the specified dir,
-  // and the db data dir's absolute path will be used as the log file
-  // name's prefix.
-  std::string db_log_dir;
-
-  // This specifies the absolute dir path for write-ahead logs (WAL).
-  // If it is empty, the log files will be in the same dir as data,
-  //   dbname is used as the data dir by default
-  // If it is non empty, the log files will be in kept the specified dir.
-  // When destroying the db,
-  //   all log files in wal_dir and the dir itself is deleted
-  std::string wal_dir;
-
   // Disable compaction triggered by seek.
   // With bloomfilter and fast storage, a miss on one level
   // is very cheap if the file handle is cached in table cache
   // (which is true if max_open_files is large).
   bool disable_seek_compaction;
 
-  // The periodicity when obsolete files get deleted. The default
-  // value is 6 hours. The files that get out of scope by compaction
-  // process will still get automatically delete on every compaction,
-  // regardless of this setting
-  uint64_t delete_obsolete_files_period_micros;
-
-  // Maximum number of concurrent background jobs, submitted to
-  // the default LOW priority thread pool
-  // Default: 1
-  int max_background_compactions;
-
-  // Maximum number of concurrent background memtable flush jobs, submitted to
-  // the HIGH priority thread pool.
-  // By default, all background jobs (major compaction and memtable flush) go
-  // to the LOW priority pool. If this option is set to a positive number,
-  // memtable flush jobs will be submitted to the HIGH priority pool.
-  // It is important when the same Env is shared by multiple db instances.
-  // Without a separate pool, long running major compaction jobs could
-  // potentially block memtable flush jobs of other db instances, leading to
-  // unnecessary Put stalls.
-  // Default: 1
-  int max_background_flushes;
-
-  // Specify the maximal size of the info log file. If the log file
-  // is larger than `max_log_file_size`, a new info log file will
-  // be created.
-  // If max_log_file_size == 0, all logs will be written to one
-  // log file.
-  size_t max_log_file_size;
-
-  // Time for the info log file to roll (in seconds).
-  // If specified with non-zero value, log file will be rolled
-  // if it has been active longer than `log_file_time_to_roll`.
-  // Default: 0 (disabled)
-  size_t log_file_time_to_roll;
-
-  // Maximal info log files to be kept.
-  // Default: 1000
-  size_t keep_log_file_num;
-
   // Puts are delayed 0-1 ms when any level has a compaction score that exceeds
   // soft_rate_limit. This is ignored when == 0.0.
   // CONSTRAINT: soft_rate_limit <= hard_rate_limit. If this constraint does not
@@ -473,32 +355,14 @@ struct Options {
   // Default: 1000
   unsigned int rate_limit_delay_max_milliseconds;
 
-  // manifest file is rolled over on reaching this limit.
-  // The older manifest file be deleted.
-  // The default value is MAX_INT so that roll-over does not take place.
-  uint64_t max_manifest_file_size;
-
   // Disable block cache. If this is set to true,
   // then no block cache should be used, and the block_cache should
   // point to a nullptr object.
   // Default: false
   bool no_block_cache;
 
-  // Number of shards used for table cache.
-  int table_cache_numshardbits;
-
-  // During data eviction of table's LRU cache, it would be inefficient
-  // to strictly follow LRU because this piece of memory will not really
-  // be released unless its refcount falls to zero. Instead, make two
-  // passes: the first pass will release items with refcount = 1,
-  // and if not enough space releases after scanning the number of
-  // elements specified by this parameter, we will remove items in LRU
-  // order.
-  int table_cache_remove_scan_count_limit;
-
-  // Size of one block in arena memory allocation.
-  //
-  // If <= 0, a proper value is automatically calculated (usually about 1/10 of
+  // size of one block in arena memory allocation.
+  // If <= 0, a proper value is automatically calculated (usually 1/10 of
   // writer_buffer_size).
   //
   // There are two additonal restriction of the The specified size:
@@ -512,71 +376,14 @@ struct Options {
   // Default: 0
   size_t arena_block_size;
 
-  // Create an Options object with default values for all fields.
-  Options();
-
-  void Dump(Logger* log) const;
-
-  // Set appropriate parameters for bulk loading.
-  // The reason that this is a function that returns "this" instead of a
-  // constructor is to enable chaining of multiple similar calls in the future.
-  //
-  // All data will be in level 0 without any automatic compaction.
-  // It's recommended to manually call CompactRange(NULL, NULL) before reading
-  // from the database, because otherwise the read can be very slow.
-  Options* PrepareForBulkLoad();
-
   // Disable automatic compactions. Manual compactions can still
-  // be issued on this database.
+  // be issued on this column family
   bool disable_auto_compactions;
 
-  // The following two fields affect how archived logs will be deleted.
-  // 1. If both set to 0, logs will be deleted asap and will not get into
-  //    the archive.
-  // 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
-  //    WAL files will be checked every 10 min and if total size is greater
-  //    then WAL_size_limit_MB, they will be deleted starting with the
-  //    earliest until size_limit is met. All empty files will be deleted.
-  // 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then
-  //    WAL files will be checked every WAL_ttl_secondsi / 2 and those that
-  //    are older than WAL_ttl_seconds will be deleted.
-  // 4. If both are not 0, WAL files will be checked every 10 min and both
-  //    checks will be performed with ttl being first.
-  uint64_t WAL_ttl_seconds;
-  uint64_t WAL_size_limit_MB;
-
-  // Number of bytes to preallocate (via fallocate) the manifest
-  // files.  Default is 4mb, which is reasonable to reduce random IO
-  // as well as prevent overallocation for mounts that preallocate
-  // large amounts of data (such as xfs's allocsize option).
-  size_t manifest_preallocation_size;
-
   // Purge duplicate/deleted keys when a memtable is flushed to storage.
   // Default: true
   bool purge_redundant_kvs_while_flush;
 
-  // Data being read from file storage may be buffered in the OS
-  // Default: true
-  bool allow_os_buffer;
-
-  // Allow the OS to mmap file for reading sst tables. Default: false
-  bool allow_mmap_reads;
-
-  // Allow the OS to mmap file for writing. Default: false
-  bool allow_mmap_writes;
-
-  // Disable child process inherit open files. Default: true
-  bool is_fd_close_on_exec;
-
-  // Skip log corruption error on recovery (If client is ok with
-  // losing most recent changes)
-  // Default: false
-  bool skip_log_error_on_recovery;
-
-  // if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
-  // Default: 3600 (1 hour)
-  unsigned int stats_dump_period_sec;
-
   // This is used to close a block before it reaches the configured
   // 'block_size'. If the percentage of free space in the current block is less
   // than this specified number and adding a new record to the block will
@@ -585,45 +392,17 @@ struct Options {
   // Default is 10.
   int block_size_deviation;
 
-  // If set true, will hint the underlying file system that the file
-  // access pattern is random, when a sst file is opened.
-  // Default: true
-  bool advise_random_on_open;
-
-  // Specify the file access pattern once a compaction is started.
-  // It will be applied to all input files of a compaction.
-  // Default: NORMAL
-  enum {
-    NONE,
-    NORMAL,
-    SEQUENTIAL,
-    WILLNEED
-  } access_hint_on_compaction_start;
-
-  // Use adaptive mutex, which spins in the user space before resorting
-  // to kernel. This could reduce context switch when the mutex is not
-  // heavily contended. However, if the mutex is hot, we could end up
-  // wasting spin time.
-  // Default: false
-  bool use_adaptive_mutex;
-
-  // Allows OS to incrementally sync files to disk while they are being
-  // written, asynchronously, in the background.
-  // Issue one request for every bytes_per_sync written. 0 turns it off.
-  // Default: 0
-  uint64_t bytes_per_sync;
-
   // The compaction style. Default: kCompactionStyleLevel
   CompactionStyle compaction_style;
 
-  // The options needed to support Universal Style compactions
-  CompactionOptionsUniversal compaction_options_universal;
-
   // If true, compaction will verify checksum on every read that happens
   // as part of compaction
   // Default: true
   bool verify_checksums_in_compaction;
 
+  // The options needed to support Universal Style compactions
+  CompactionOptionsUniversal compaction_options_universal;
+
   // Use KeyMayExist API to filter deletes when this is true.
   // If KeyMayExist returns false, i.e. the key definitely does not exist, then
   // the delete is a noop. KeyMayExist only incurs in-memory look up.
@@ -653,7 +432,7 @@ struct Options {
   // Default: emtpy vector -- no user-defined statistics collection will be
   // performed.
   typedef std::vector<std::shared_ptr<TablePropertiesCollector>>
-          TablePropertiesCollectors;
+      TablePropertiesCollectors;
   TablePropertiesCollectors table_properties_collectors;
 
   // Allows thread-safe inplace updates.
@@ -719,6 +498,14 @@ struct Options {
   // number of hash probes per key
   uint32_t memtable_prefix_bloom_probes;
 
+  // Page size for huge page TLB for bloom in memtable. If <=0, not allocate
+  // from huge page TLB but from malloc.
+  // Need to reserve huge pages for it to be allocated. For example:
+  //      sysctl -w vm.nr_hugepages=20
+  // See linux doc Documentation/vm/hugetlbpage.txt
+
+  size_t memtable_prefix_bloom_huge_page_tlb_size;
+
   // Control locality of bloom filter probes to improve cache miss rate.
   // This option only applies to memtable prefix bloom and plaintable
   // prefix bloom. It essentially limits the max number of cache lines each
@@ -750,9 +537,274 @@ struct Options {
   // Default: 2
   uint32_t min_partial_merge_operands;
 
+  // Create ColumnFamilyOptions with default values for all fields
+  ColumnFamilyOptions();
+  // Create ColumnFamilyOptions from Options
+  explicit ColumnFamilyOptions(const Options& options);
+
+  void Dump(Logger* log) const;
+};
+
+struct DBOptions {
+  // If true, the database will be created if it is missing.
+  // Default: false
+  bool create_if_missing;
+
+  // If true, an error is raised if the database already exists.
+  // Default: false
+  bool error_if_exists;
+
+  // If true, the implementation will do aggressive checking of the
+  // data it is processing and will stop early if it detects any
+  // errors.  This may have unforeseen ramifications: for example, a
+  // corruption of one DB entry may cause a large number of entries to
+  // become unreadable or for the entire DB to become unopenable.
+  // If any of the  writes to the database fails (Put, Delete, Merge, Write),
+  // the database will switch to read-only mode and fail all other
+  // Write operations.
+  // Default: true
+  bool paranoid_checks;
+
+  // Use the specified object to interact with the environment,
+  // e.g. to read/write files, schedule background work, etc.
+  // Default: Env::Default()
+  Env* env;
+
+  // Any internal progress/error information generated by the db will
+  // be written to info_log if it is non-nullptr, or to a file stored
+  // in the same directory as the DB contents if info_log is nullptr.
+  // Default: nullptr
+  shared_ptr<Logger> info_log;
+
+  InfoLogLevel info_log_level;
+
+  // Number of open files that can be used by the DB.  You may need to
+  // increase this if your database has a large working set. Value -1 means
+  // files opened are always kept open. You can estimate number of files based
+  // on target_file_size_base and target_file_size_multiplier for level-based
+  // compaction. For universal-style compaction, you can usually set it to -1.
+  // Default: 5000
+  int max_open_files;
+
+  // Once write-ahead logs exceed this size, we will start forcing the flush of
+  // column families whose memtables are backed by the oldest live WAL file
+  // (i.e. the ones that are causing all the space amplification). If set to 0
+  // (default), we will dynamically choose the WAL size limit to be
+  // [sum of all write_buffer_size * max_write_buffer_number] * 2
+  // Default: 0
+  uint64_t max_total_wal_size;
+
+  // If non-null, then we should collect metrics about database operations
+  // Statistics objects should not be shared between DB instances as
+  // it does not use any locks to prevent concurrent updates.
+  shared_ptr<Statistics> statistics;
+
+  // If true, then the contents of data files are not synced
+  // to stable storage. Their contents remain in the OS buffers till the
+  // OS decides to flush them. This option is good for bulk-loading
+  // of data. Once the bulk-loading is complete, please issue a
+  // sync to the OS to flush all dirty buffesrs to stable storage.
+  // Default: false
+  bool disableDataSync;
+
+  // If true, then every store to stable storage will issue a fsync.
+  // If false, then every store to stable storage will issue a fdatasync.
+  // This parameter should be set to true while storing data to
+  // filesystem like ext3 that can lose files after a reboot.
+  // Default: false
+  bool use_fsync;
+
+  // This number controls how often a new scribe log about
+  // db deploy stats is written out.
+  // -1 indicates no logging at all.
+  // Default value is 1800 (half an hour).
+  int db_stats_log_interval;
+
+  // This specifies the info LOG dir.
+  // If it is empty, the log files will be in the same dir as data.
+  // If it is non empty, the log files will be in the specified dir,
+  // and the db data dir's absolute path will be used as the log file
+  // name's prefix.
+  std::string db_log_dir;
+
+  // This specifies the absolute dir path for write-ahead logs (WAL).
+  // If it is empty, the log files will be in the same dir as data,
+  //   dbname is used as the data dir by default
+  // If it is non empty, the log files will be in kept the specified dir.
+  // When destroying the db,
+  //   all log files in wal_dir and the dir itself is deleted
+  std::string wal_dir;
+
+  // The periodicity when obsolete files get deleted. The default
+  // value is 6 hours. The files that get out of scope by compaction
+  // process will still get automatically delete on every compaction,
+  // regardless of this setting
+  uint64_t delete_obsolete_files_period_micros;
+
+  // Maximum number of concurrent background compaction jobs, submitted to
+  // the default LOW priority thread pool.
+  // If you're increasing this, also consider increasing number of threads in
+  // LOW priority thread pool. For more information, see
+  // Env::SetBackgroundThreads
+  // Default: 1
+  int max_background_compactions;
+
+  // Maximum number of concurrent background memtable flush jobs, submitted to
+  // the HIGH priority thread pool.
+  //
+  // By default, all background jobs (major compaction and memtable flush) go
+  // to the LOW priority pool. If this option is set to a positive number,
+  // memtable flush jobs will be submitted to the HIGH priority pool.
+  // It is important when the same Env is shared by multiple db instances.
+  // Without a separate pool, long running major compaction jobs could
+  // potentially block memtable flush jobs of other db instances, leading to
+  // unnecessary Put stalls.
+  //
+  // If you're increasing this, also consider increasing number of threads in
+  // HIGH priority thread pool. For more information, see
+  // Env::SetBackgroundThreads
+  // Default: 1
+  int max_background_flushes;
+
+  // Specify the maximal size of the info log file. If the log file
+  // is larger than `max_log_file_size`, a new info log file will
+  // be created.
+  // If max_log_file_size == 0, all logs will be written to one
+  // log file.
+  size_t max_log_file_size;
+
+  // Time for the info log file to roll (in seconds).
+  // If specified with non-zero value, log file will be rolled
+  // if it has been active longer than `log_file_time_to_roll`.
+  // Default: 0 (disabled)
+  size_t log_file_time_to_roll;
+
+  // Maximal info log files to be kept.
+  // Default: 1000
+  size_t keep_log_file_num;
+
+  // manifest file is rolled over on reaching this limit.
+  // The older manifest file be deleted.
+  // The default value is MAX_INT so that roll-over does not take place.
+  uint64_t max_manifest_file_size;
+
+  // Number of shards used for table cache.
+  int table_cache_numshardbits;
+
+  // During data eviction of table's LRU cache, it would be inefficient
+  // to strictly follow LRU because this piece of memory will not really
+  // be released unless its refcount falls to zero. Instead, make two
+  // passes: the first pass will release items with refcount = 1,
+  // and if not enough space releases after scanning the number of
+  // elements specified by this parameter, we will remove items in LRU
+  // order.
+  int table_cache_remove_scan_count_limit;
+
+  // The following two fields affect how archived logs will be deleted.
+  // 1. If both set to 0, logs will be deleted asap and will not get into
+  //    the archive.
+  // 2. If WAL_ttl_seconds is 0 and WAL_size_limit_MB is not 0,
+  //    WAL files will be checked every 10 min and if total size is greater
+  //    then WAL_size_limit_MB, they will be deleted starting with the
+  //    earliest until size_limit is met. All empty files will be deleted.
+  // 3. If WAL_ttl_seconds is not 0 and WAL_size_limit_MB is 0, then
+  //    WAL files will be checked every WAL_ttl_secondsi / 2 and those that
+  //    are older than WAL_ttl_seconds will be deleted.
+  // 4. If both are not 0, WAL files will be checked every 10 min and both
+  //    checks will be performed with ttl being first.
+  uint64_t WAL_ttl_seconds;
+  uint64_t WAL_size_limit_MB;
+
+  // Number of bytes to preallocate (via fallocate) the manifest
+  // files.  Default is 4mb, which is reasonable to reduce random IO
+  // as well as prevent overallocation for mounts that preallocate
+  // large amounts of data (such as xfs's allocsize option).
+  size_t manifest_preallocation_size;
+
+  // Data being read from file storage may be buffered in the OS
+  // Default: true
+  bool allow_os_buffer;
+
+  // Allow the OS to mmap file for reading sst tables. Default: false
+  bool allow_mmap_reads;
+
+  // Allow the OS to mmap file for writing. Default: false
+  bool allow_mmap_writes;
+
+  // Disable child process inherit open files. Default: true
+  bool is_fd_close_on_exec;
+
+  // Skip log corruption error on recovery (If client is ok with
+  // losing most recent changes)
+  // Default: false
+  bool skip_log_error_on_recovery;
+
+  // if not zero, dump rocksdb.stats to LOG every stats_dump_period_sec
+  // Default: 3600 (1 hour)
+  unsigned int stats_dump_period_sec;
+
+  // If set true, will hint the underlying file system that the file
+  // access pattern is random, when a sst file is opened.
+  // Default: true
+  bool advise_random_on_open;
+
+  // Specify the file access pattern once a compaction is started.
+  // It will be applied to all input files of a compaction.
+  // Default: NORMAL
+  enum {
+    NONE,
+    NORMAL,
+    SEQUENTIAL,
+    WILLNEED
+  } access_hint_on_compaction_start;
+
+  // Use adaptive mutex, which spins in the user space before resorting
+  // to kernel. This could reduce context switch when the mutex is not
+  // heavily contended. However, if the mutex is hot, we could end up
+  // wasting spin time.
+  // Default: false
+  bool use_adaptive_mutex;
+
+  // Allows OS to incrementally sync files to disk while they are being
+  // written, asynchronously, in the background.
+  // Issue one request for every bytes_per_sync written. 0 turns it off.
+  // Default: 0
+  uint64_t bytes_per_sync;
+
   // Allow RocksDB to use thread local storage to optimize performance.
   // Default: true
   bool allow_thread_local;
+
+  // Create DBOptions with default values for all fields
+  DBOptions();
+  // Create DBOptions from Options
+  explicit DBOptions(const Options& options);
+
+  void Dump(Logger* log) const;
+};
+
+// Options to control the behavior of a database (passed to DB::Open)
+struct Options : public DBOptions, public ColumnFamilyOptions {
+  // Create an Options object with default values for all fields.
+  Options() :
+    DBOptions(),
+    ColumnFamilyOptions() {}
+
+  Options(const DBOptions& db_options,
+          const ColumnFamilyOptions& column_family_options)
+      : DBOptions(db_options), ColumnFamilyOptions(column_family_options) {}
+
+  void Dump(Logger* log) const;
+
+  // Set appropriate parameters for bulk loading.
+  // The reason that this is a function that returns "this" instead of a
+  // constructor is to enable chaining of multiple similar calls in the future.
+  //
+
+  // All data will be in level 0 without any automatic compaction.
+  // It's recommended to manually call CompactRange(NULL, NULL) before reading
+  // from the database, because otherwise the read can be very slow.
+  Options* PrepareForBulkLoad();
 };
 
 //
@@ -782,7 +834,10 @@ struct ReadOptions {
 
   // If this option is set and memtable implementation allows, Seek
   // might only return keys with the same prefix as the seek-key
-  bool prefix_seek;
+  //
+  // ! DEPRECATED: prefix_seek is on by default when prefix_extractor
+  // is configured
+  // bool prefix_seek;
 
   // If "snapshot" is non-nullptr, read as of the supplied snapshot
   // (which must belong to the DB that is being read and which must
@@ -802,7 +857,9 @@ struct ReadOptions {
   // prefix, and SeekToLast() is not supported.  prefix filter with this
   // option will sometimes reduce the number of read IOPs.
   // Default: nullptr
-  const Slice* prefix;
+  //
+  // ! DEPRECATED
+  // const Slice* prefix;
 
   // Specify if this read request should process data that ALREADY
   // resides on a particular cache. If the required data is not
@@ -815,22 +872,19 @@ struct ReadOptions {
   // added data) and is optimized for sequential reads. It will return records
   // that were inserted into the database after the creation of the iterator.
   // Default: false
+  // Not supported in ROCKSDB_LITE mode!
   bool tailing;
 
   ReadOptions()
       : verify_checksums(true),
         fill_cache(true),
-        prefix_seek(false),
         snapshot(nullptr),
-        prefix(nullptr),
         read_tier(kReadAllTier),
         tailing(false) {}
   ReadOptions(bool cksum, bool cache)
       : verify_checksums(cksum),
         fill_cache(cache),
-        prefix_seek(false),
         snapshot(nullptr),
-        prefix(nullptr),
         read_tier(kReadAllTier),
         tailing(false) {}
 };

include/rocksdb/perf_context.h

@@ -64,7 +64,7 @@ struct PerfContext {
   uint64_t write_memtable_time;
 };
 
-#if defined(IOS_CROSS_COMPILE)
+#if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
 extern PerfContext perf_context;
 #else
 extern __thread PerfContext perf_context;

include/rocksdb/slice.h

@@ -105,8 +105,8 @@ class Slice {
 // A set of Slices that are virtually concatenated together.  'parts' points
 // to an array of Slices.  The number of elements in the array is 'num_parts'.
 struct SliceParts {
-  SliceParts(const Slice* parts, int num_parts) :
-      parts(parts), num_parts(num_parts) { }
+  SliceParts(const Slice* _parts, int _num_parts) :
+      parts(_parts), num_parts(_num_parts) { }
 
   const Slice* parts;
   int num_parts;

include/rocksdb/table.h

@@ -38,6 +38,12 @@ struct Options;
 
 using std::unique_ptr;
 
+enum ChecksumType : char {
+  kNoChecksum = 0x0,  // not yet supported. Will fail
+  kCRC32c = 0x1,
+  kxxHash = 0x2,
+};
+
 // For advanced user only
 struct BlockBasedTableOptions {
   // @flush_block_policy_factory creates the instances of flush block policy.
@@ -60,9 +66,18 @@ struct BlockBasedTableOptions {
     // A space efficient index block that is optimized for
     // binary-search-based index.
     kBinarySearch,
+
+    // The hash index, if enabled, will do the hash lookup when
+    // `Options.prefix_extractor` is provided.
+    kHashSearch,
   };
 
   IndexType index_type = kBinarySearch;
+
+  // Use the specified checksum type. Newly created table files will be
+  // protected with this checksum type. Old table files will still be readable,
+  // even though they have different checksum type.
+  ChecksumType checksum = kCRC32c;
 };
 
 // Table Properties that are specific to block-based table properties.
@@ -75,6 +90,7 @@ struct BlockBasedTablePropertyNames {
 extern TableFactory* NewBlockBasedTableFactory(
     const BlockBasedTableOptions& table_options = BlockBasedTableOptions());
 
+#ifndef ROCKSDB_LITE
 // -- Plain Table with prefix-only seek
 // For this factory, you need to set Options.prefix_extrator properly to make it
 // work. Look-up will starts with prefix hash lookup for key prefix. Inside the
@@ -91,12 +107,19 @@ extern TableFactory* NewBlockBasedTableFactory(
 //                    in the hash table
 // @index_sparseness: inside each prefix, need to build one index record for how
 //                    many keys for binary search inside each hash bucket.
+// @huge_page_tlb_size: if <=0, allocate hash indexes and blooms from malloc.
+//                      Otherwise from huge page TLB. The user needs to reserve
+//                      huge pages for it to be allocated, like:
+//                          sysctl -w vm.nr_hugepages=20
+//                      See linux doc Documentation/vm/hugetlbpage.txt
+
 const uint32_t kPlainTableVariableLength = 0;
 extern TableFactory* NewPlainTableFactory(uint32_t user_key_len =
                                               kPlainTableVariableLength,
                                           int bloom_bits_per_prefix = 10,
                                           double hash_table_ratio = 0.75,
-                                          size_t index_sparseness = 16);
+                                          size_t index_sparseness = 16,
+                                          size_t huge_page_tlb_size = 0);
 
 // -- Plain Table
 // This factory of plain table ignores Options.prefix_extractor and assumes no
@@ -110,9 +133,17 @@ extern TableFactory* NewPlainTableFactory(uint32_t user_key_len =
 //                  disable it by passing a zero.
 // @index_sparseness: need to build one index record for how many keys for
 //                    binary search.
+// @huge_page_tlb_size: if <=0, allocate hash indexes and blooms from malloc.
+//                      Otherwise from huge page TLB. The user needs to reserve
+//                      huge pages for it to be allocated, like:
+//                          sysctl -w vm.nr_hugepages=20
+//                      See linux doc Documentation/vm/hugetlbpage.txt
 extern TableFactory* NewTotalOrderPlainTableFactory(
     uint32_t user_key_len = kPlainTableVariableLength,
-    int bloom_bits_per_key = 0, size_t index_sparseness = 16);
+    int bloom_bits_per_key = 0, size_t index_sparseness = 16,
+    size_t huge_page_tlb_size = 0);
+
+#endif  // ROCKSDB_LITE
 
 // A base class for table factories.
 class TableFactory {

include/rocksdb/table_properties.h

@@ -23,7 +23,7 @@ namespace rocksdb {
 //      ++pos) {
 //   ...
 // }
-typedef std::map<std::string, std::string> UserCollectedProperties;
+typedef std::map<const std::string, std::string> UserCollectedProperties;
 
 // TableProperties contains a bunch of read-only properties of its associated
 // table.

include/rocksdb/version.h

@@ -0,0 +1,6 @@
+#pragma once
+
+// Also update Makefile if you change these
+#define __ROCKSDB_MAJOR__ 3
+#define __ROCKSDB_MINOR__ 0
+#define __ROCKSDB_PATCH__ 0

include/rocksdb/write_batch.h

@@ -31,6 +31,7 @@
 namespace rocksdb {
 
 class Slice;
+class ColumnFamilyHandle;
 struct SliceParts;
 
 class WriteBatch {
@@ -39,19 +40,32 @@ class WriteBatch {
   ~WriteBatch();
 
   // Store the mapping "key->value" in the database.
-  void Put(const Slice& key, const Slice& value);
+  void Put(ColumnFamilyHandle* column_family, const Slice& key,
+           const Slice& value);
+  void Put(const Slice& key, const Slice& value) {
+    Put(nullptr, key, value);
+  }
 
   // Variant of Put() that gathers output like writev(2).  The key and value
   // that will be written to the database are concatentations of arrays of
   // slices.
-  void Put(const SliceParts& key, const SliceParts& value);
+  void Put(ColumnFamilyHandle* column_family, const SliceParts& key,
+           const SliceParts& value);
+  void Put(const SliceParts& key, const SliceParts& value) {
+    Put(nullptr, key, value);
+  }
 
   // Merge "value" with the existing value of "key" in the database.
   // "key->merge(existing, value)"
-  void Merge(const Slice& key, const Slice& value);
+  void Merge(ColumnFamilyHandle* column_family, const Slice& key,
+             const Slice& value);
+  void Merge(const Slice& key, const Slice& value) {
+    Merge(nullptr, key, value);
+  }
 
   // If the database contains a mapping for "key", erase it.  Else do nothing.
-  void Delete(const Slice& key);
+  void Delete(ColumnFamilyHandle* column_family, const Slice& key);
+  void Delete(const Slice& key) { Delete(nullptr, key); }
 
   // Append a blob of arbitrary size to the records in this batch. The blob will
   // be stored in the transaction log but not in any other file. In particular,
@@ -72,14 +86,46 @@ class WriteBatch {
   class Handler {
    public:
     virtual ~Handler();
-    virtual void Put(const Slice& key, const Slice& value) = 0;
+    // default implementation will just call Put without column family for
+    // backwards compatibility. If the column family is not default,
+    // the function is noop
+    virtual Status PutCF(uint32_t column_family_id, const Slice& key,
+                         const Slice& value) {
+      if (column_family_id == 0) {
+        // Put() historically doesn't return status. We didn't want to be
+        // backwards incompatible so we didn't change the return status
+        // (this is a public API). We do an ordinary get and return Status::OK()
+        Put(key, value);
+        return Status::OK();
+      }
+      return Status::InvalidArgument(
+          "non-default column family and PutCF not implemented");
+    }
+    virtual void Put(const Slice& key, const Slice& value);
     // Merge and LogData are not pure virtual. Otherwise, we would break
     // existing clients of Handler on a source code level. The default
     // implementation of Merge simply throws a runtime exception.
+    virtual Status MergeCF(uint32_t column_family_id, const Slice& key,
+                           const Slice& value) {
+      if (column_family_id == 0) {
+        Merge(key, value);
+        return Status::OK();
+      }
+      return Status::InvalidArgument(
+          "non-default column family and MergeCF not implemented");
+    }
     virtual void Merge(const Slice& key, const Slice& value);
     // The default implementation of LogData does nothing.
     virtual void LogData(const Slice& blob);
-    virtual void Delete(const Slice& key) = 0;
+    virtual Status DeleteCF(uint32_t column_family_id, const Slice& key) {
+      if (column_family_id == 0) {
+        Delete(key);
+        return Status::OK();
+      }
+      return Status::InvalidArgument(
+          "non-default column family and DeleteCF not implemented");
+    }
+    virtual void Delete(const Slice& key);
     // Continue is called by WriteBatch::Iterate. If it returns false,
     // iteration is halted. Otherwise, it continues iterating. The default
     // implementation always returns true.

include/utilities/backupable_db.h

@@ -8,14 +8,18 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include "utilities/stackable_db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/status.h"
+#ifndef ROCKSDB_LITE
 
+#define __STDC_FORMAT_MACROS
+#include <inttypes.h>
 #include <string>
 #include <map>
 #include <vector>
 
+#include "utilities/stackable_db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+
 namespace rocksdb {
 
 struct BackupableDBOptions {
@@ -71,6 +75,14 @@ struct BackupableDBOptions {
   // Default: 0
   uint64_t restore_rate_limit;
 
+  // Only used if share_table_files is set to true. If true, will consider that
+  // backups can come from different databases, hence a sst is not uniquely
+  // identifed by its name, but by the triple (file name, crc32, file length)
+  // Default: false
+  // Note: this is an experimental option, and you'll need to set it manually
+  // *turn it on only if you know what you're doing*
+  bool share_files_with_checksum;
+
   void Dump(Logger* logger) const;
 
   explicit BackupableDBOptions(const std::string& _backup_dir,
@@ -89,7 +101,10 @@ struct BackupableDBOptions {
         destroy_old_data(_destroy_old_data),
         backup_log_files(_backup_log_files),
         backup_rate_limit(_backup_rate_limit),
-        restore_rate_limit(_restore_rate_limit) {}
+        restore_rate_limit(_restore_rate_limit),
+        share_files_with_checksum(false) {
+    assert(share_table_files || !share_files_with_checksum);
+  }
 };
 
 struct RestoreOptions {
@@ -116,6 +131,29 @@ struct BackupInfo {
       : backup_id(_backup_id), timestamp(_timestamp), size(_size) {}
 };
 
+class BackupEngineReadOnly {
+ public:
+  virtual ~BackupEngineReadOnly() {}
+
+  static BackupEngineReadOnly* NewReadOnlyBackupEngine(
+      Env* db_env, const BackupableDBOptions& options);
+
+  // You can GetBackupInfo safely, even with other BackupEngine performing
+  // backups on the same directory
+  virtual void GetBackupInfo(std::vector<BackupInfo>* backup_info) = 0;
+
+  // Restoring DB from backup is NOT safe when there is another BackupEngine
+  // running that might call DeleteBackup() or PurgeOldBackups(). It is caller's
+  // responsibility to synchronize the operation, i.e. don't delete the backup
+  // when you're restoring from it
+  virtual Status RestoreDBFromBackup(
+      BackupID backup_id, const std::string& db_dir, const std::string& wal_dir,
+      const RestoreOptions& restore_options = RestoreOptions()) = 0;
+  virtual Status RestoreDBFromLatestBackup(
+      const std::string& db_dir, const std::string& wal_dir,
+      const RestoreOptions& restore_options = RestoreOptions()) = 0;
+};
+
 // Please see the documentation in BackupableDB and RestoreBackupableDB
 class BackupEngine {
  public:
@@ -209,4 +247,5 @@ class RestoreBackupableDB {
   BackupEngine* backup_engine_;
 };
 
-} // rocksdb namespace
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

include/utilities/db_ttl.h

@@ -0,0 +1,68 @@
+//  Copyright (c) 2013, Facebook, Inc.  All rights reserved.
+//  This source code is licensed under the BSD-style license found in the
+//  LICENSE file in the root directory of this source tree. An additional grant
+//  of patent rights can be found in the PATENTS file in the same directory.
+
+#pragma once
+#ifndef ROCKSDB_LITE
+
+#include <string>
+#include <vector>
+
+#include "utilities/stackable_db.h"
+#include "rocksdb/db.h"
+
+namespace rocksdb {
+
+// Database with TTL support.
+//
+// USE-CASES:
+// This API should be used to open the db when key-values inserted are
+//  meant to be removed from the db in a non-strict 'ttl' amount of time
+//  Therefore, this guarantees that key-values inserted will remain in the
+//  db for >= ttl amount of time and the db will make efforts to remove the
+//  key-values as soon as possible after ttl seconds of their insertion.
+//
+// BEHAVIOUR:
+// TTL is accepted in seconds
+// (int32_t)Timestamp(creation) is suffixed to values in Put internally
+// Expired TTL values deleted in compaction only:(Timestamp+ttl<time_now)
+// Get/Iterator may return expired entries(compaction not run on them yet)
+// Different TTL may be used during different Opens
+// Example: Open1 at t=0 with ttl=4 and insert k1,k2, close at t=2
+//          Open2 at t=3 with ttl=5. Now k1,k2 should be deleted at t>=5
+// read_only=true opens in the usual read-only mode. Compactions will not be
+//  triggered(neither manual nor automatic), so no expired entries removed
+//
+// CONSTRAINTS:
+// Not specifying/passing or non-positive TTL behaves like TTL = infinity
+//
+// !!!WARNING!!!:
+// Calling DB::Open directly to re-open a db created by this API will get
+//  corrupt values(timestamp suffixed) and no ttl effect will be there
+//  during the second Open, so use this API consistently to open the db
+// Be careful when passing ttl with a small positive value because the
+//  whole database may be deleted in a small amount of time
+
+class DBWithTTL : public StackableDB {
+ public:
+  virtual Status CreateColumnFamilyWithTtl(
+      const ColumnFamilyOptions& options, const std::string& column_family_name,
+      ColumnFamilyHandle** handle, int ttl) = 0;
+
+  static Status Open(const Options& options, const std::string& dbname,
+                     DBWithTTL** dbptr, int32_t ttl = 0,
+                     bool read_only = false);
+
+  static Status Open(const DBOptions& db_options, const std::string& dbname,
+                     const std::vector<ColumnFamilyDescriptor>& column_families,
+                     std::vector<ColumnFamilyHandle*>* handles,
+                     DBWithTTL** dbptr, std::vector<int32_t> ttls,
+                     bool read_only = false);
+
+ protected:
+  explicit DBWithTTL(DB* db) : StackableDB(db) {}
+};
+
+}  // namespace rocksdb
+#endif  // ROCKSDB_LITE

include/utilities/geo_db.h

@@ -4,6 +4,7 @@
 //  of patent rights can be found in the PATENTS file in the same directory.
 //
 
+#ifndef ROCKSDB_LITE
 #pragma once
 #include <string>
 #include <vector>
@@ -101,3 +102,4 @@ class GeoDB : public StackableDB {
 };
 
 }  // namespace rocksdb
+#endif  // ROCKSDB_LITE

include/utilities/stackable_db.h

@@ -21,40 +21,59 @@ class StackableDB : public DB {
     return db_;
   }
 
+  virtual Status CreateColumnFamily(const ColumnFamilyOptions& options,
+                                    const std::string& column_family_name,
+                                    ColumnFamilyHandle** handle) {
+    return db_->CreateColumnFamily(options, column_family_name, handle);
+  }
+
+  virtual Status DropColumnFamily(ColumnFamilyHandle* column_family) {
+    return db_->DropColumnFamily(column_family);
+  }
+
+  using DB::Put;
   virtual Status Put(const WriteOptions& options,
-                     const Slice& key,
+                     ColumnFamilyHandle* column_family, const Slice& key,
                      const Slice& val) override {
-    return db_->Put(options, key, val);
+    return db_->Put(options, column_family, key, val);
   }
 
+  using DB::Get;
   virtual Status Get(const ReadOptions& options,
-                     const Slice& key,
+                     ColumnFamilyHandle* column_family, const Slice& key,
                      std::string* value) override {
-    return db_->Get(options, key, value);
+    return db_->Get(options, column_family, key, value);
   }
 
-  virtual std::vector<Status> MultiGet(const ReadOptions& options,
-                                       const std::vector<Slice>& keys,
-                                       std::vector<std::string>* values)
-    override {
-      return db_->MultiGet(options, keys, values);
+  using DB::MultiGet;
+  virtual std::vector<Status> MultiGet(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_family,
+      const std::vector<Slice>& keys,
+      std::vector<std::string>* values) override {
+    return db_->MultiGet(options, column_family, keys, values);
   }
 
+  using DB::KeyMayExist;
   virtual bool KeyMayExist(const ReadOptions& options,
-                           const Slice& key,
+                           ColumnFamilyHandle* column_family, const Slice& key,
                            std::string* value,
                            bool* value_found = nullptr) override {
-    return db_->KeyMayExist(options, key, value, value_found);
+    return db_->KeyMayExist(options, column_family, key, value, value_found);
   }
 
-  virtual Status Delete(const WriteOptions& wopts, const Slice& key) override {
-    return db_->Delete(wopts, key);
+  using DB::Delete;
+  virtual Status Delete(const WriteOptions& wopts,
+                        ColumnFamilyHandle* column_family,
+                        const Slice& key) override {
+    return db_->Delete(wopts, column_family, key);
   }
 
+  using DB::Merge;
   virtual Status Merge(const WriteOptions& options,
-                       const Slice& key,
+                       ColumnFamilyHandle* column_family, const Slice& key,
                        const Slice& value) override {
-    return db_->Merge(options, key, value);
+    return db_->Merge(options, column_family, key, value);
   }
 
 
@@ -63,10 +82,20 @@ class StackableDB : public DB {
       return db_->Write(opts, updates);
   }
 
-  virtual Iterator* NewIterator(const ReadOptions& opts) override {
-    return db_->NewIterator(opts);
+  using DB::NewIterator;
+  virtual Iterator* NewIterator(const ReadOptions& opts,
+                                ColumnFamilyHandle* column_family) override {
+    return db_->NewIterator(opts, column_family);
   }
 
+  virtual Status NewIterators(
+      const ReadOptions& options,
+      const std::vector<ColumnFamilyHandle*>& column_families,
+      std::vector<Iterator*>* iterators) {
+    return db_->NewIterators(options, column_families, iterators);
+  }
+
+
   virtual const Snapshot* GetSnapshot() override {
     return db_->GetSnapshot();
   }
@@ -75,32 +104,43 @@ class StackableDB : public DB {
     return db_->ReleaseSnapshot(snapshot);
   }
 
-  virtual bool GetProperty(const Slice& property, std::string* value)
-    override {
-      return db_->GetProperty(property, value);
+  using DB::GetProperty;
+  virtual bool GetProperty(ColumnFamilyHandle* column_family,
+                           const Slice& property, std::string* value) override {
+      return db_->GetProperty(column_family, property, value);
   }
 
-  virtual void GetApproximateSizes(const Range* r, int n, uint64_t* sizes)
-    override {
-      return db_->GetApproximateSizes(r, n, sizes);
+  using DB::GetApproximateSizes;
+  virtual void GetApproximateSizes(ColumnFamilyHandle* column_family,
+                                   const Range* r, int n,
+                                   uint64_t* sizes) override {
+      return db_->GetApproximateSizes(column_family, r, n, sizes);
   }
 
-  virtual Status CompactRange(const Slice* begin, const Slice* end,
+  using DB::CompactRange;
+  virtual Status CompactRange(ColumnFamilyHandle* column_family,
+                              const Slice* begin, const Slice* end,
                               bool reduce_level = false,
                               int target_level = -1) override {
-    return db_->CompactRange(begin, end, reduce_level, target_level);
+    return db_->CompactRange(column_family, begin, end, reduce_level,
+                             target_level);
   }
 
-  virtual int NumberLevels() override {
-    return db_->NumberLevels();
+  using DB::NumberLevels;
+  virtual int NumberLevels(ColumnFamilyHandle* column_family) override {
+    return db_->NumberLevels(column_family);
   }
 
-  virtual int MaxMemCompactionLevel() override {
-    return db_->MaxMemCompactionLevel();
+  using DB::MaxMemCompactionLevel;
+  virtual int MaxMemCompactionLevel(ColumnFamilyHandle* column_family)
+      override {
+    return db_->MaxMemCompactionLevel(column_family);
   }
 
-  virtual int Level0StopWriteTrigger() override {
-    return db_->Level0StopWriteTrigger();
+  using DB::Level0StopWriteTrigger;
+  virtual int Level0StopWriteTrigger(ColumnFamilyHandle* column_family)
+      override {
+    return db_->Level0StopWriteTrigger(column_family);
   }
 
   virtual const std::string& GetName() const override {
@@ -111,12 +151,16 @@ class StackableDB : public DB {
     return db_->GetEnv();
   }
 
-  virtual const Options& GetOptions() const override {
-    return db_->GetOptions();
+  using DB::GetOptions;
+  virtual const Options& GetOptions(ColumnFamilyHandle* column_family) const
+      override {
+    return db_->GetOptions(column_family);
   }
 
-  virtual Status Flush(const FlushOptions& fopts) override {
-    return db_->Flush(fopts);
+  using DB::Flush;
+  virtual Status Flush(const FlushOptions& fopts,
+                       ColumnFamilyHandle* column_family) override {
+    return db_->Flush(fopts, column_family);
   }
 
   virtual Status DisableFileDeletions() override {
@@ -148,8 +192,10 @@ class StackableDB : public DB {
     return db_->GetDbIdentity(identity);
   }
 
-  virtual Status GetPropertiesOfAllTables(TablePropertiesCollection* props) {
-    return db_->GetPropertiesOfAllTables(props);
+  using DB::GetPropertiesOfAllTables;
+  virtual Status GetPropertiesOfAllTables(ColumnFamilyHandle* column_family,
+                                          TablePropertiesCollection* props) {
+    return db_->GetPropertiesOfAllTables(column_family, props);
   }
 
   virtual Status GetUpdatesSince(
@@ -158,6 +204,10 @@ class StackableDB : public DB {
     return db_->GetUpdatesSince(seq_number, iter, read_options);
   }
 
+  virtual ColumnFamilyHandle* DefaultColumnFamily() const override {
+    return db_->DefaultColumnFamily();
+  }
+
  protected:
   DB* db_;
 };

include/utilities/utility_db.h

@@ -3,48 +3,28 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #pragma once
-#include "stackable_db.h"
+#ifndef ROCKSDB_LITE
+#include <vector>
+#include <string>
+
+#include "utilities/stackable_db.h"
+#include "utilities/db_ttl.h"
+#include "rocksdb/db.h"
 
 namespace rocksdb {
 
-// This class contains APIs to open rocksdb with specific support eg. TTL
+// Please don't use this class. It's deprecated
 class UtilityDB {
-
-  public:
-    // Open the database with TTL support.
-    //
-    // USE-CASES:
-    // This API should be used to open the db when key-values inserted are
-    //  meant to be removed from the db in a non-strict 'ttl' amount of time
-    //  Therefore, this guarantees that key-values inserted will remain in the
-    //  db for >= ttl amount of time and the db will make efforts to remove the
-    //  key-values as soon as possible after ttl seconds of their insertion.
-    //
-    // BEHAVIOUR:
-    // TTL is accepted in seconds
-    // (int32_t)Timestamp(creation) is suffixed to values in Put internally
-    // Expired TTL values deleted in compaction only:(Timestamp+ttl<time_now)
-    // Get/Iterator may return expired entries(compaction not run on them yet)
-    // Different TTL may be used during different Opens
-    // Example: Open1 at t=0 with ttl=4 and insert k1,k2, close at t=2
-    //          Open2 at t=3 with ttl=5. Now k1,k2 should be deleted at t>=5
-    // read_only=true opens in the usual read-only mode. Compactions will not be
-    //  triggered(neither manual nor automatic), so no expired entries removed
-    //
-    // CONSTRAINTS:
-    // Not specifying/passing or non-positive TTL behaves like TTL = infinity
-    //
-    // !!!WARNING!!!:
-    // Calling DB::Open directly to re-open a db created by this API will get
-    //  corrupt values(timestamp suffixed) and no ttl effect will be there
-    //  during the second Open, so use this API consistently to open the db
-    // Be careful when passing ttl with a small positive value because the
-    //  whole database may be deleted in a small amount of time
-    static Status OpenTtlDB(const Options& options,
-                            const std::string& name,
-                            StackableDB** dbptr,
-                            int32_t ttl = 0,
-                            bool read_only = false);
+ public:
+  // This function is here only for backwards compatibility. Please use the
+  // functions defined in DBWithTTl (utilities/db_ttl.h)
+  // (deprecated)
+  __attribute__((deprecated)) static Status OpenTtlDB(const Options& options,
+                                                      const std::string& name,
+                                                      StackableDB** dbptr,
+                                                      int32_t ttl = 0,
+                                                      bool read_only = false);
 };
 
 } //  namespace rocksdb
+#endif  // ROCKSDB_LITE

java/Makefile

@@ -1,4 +1,4 @@
-NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions
+NATIVE_JAVA_CLASSES = org.rocksdb.RocksDB org.rocksdb.Options org.rocksdb.WriteBatch org.rocksdb.WriteBatchInternal org.rocksdb.WriteBatchTest org.rocksdb.WriteOptions org.rocksdb.BackupableDB org.rocksdb.BackupableDBOptions org.rocksdb.Statistics org.rocksdb.Iterator org.rocksdb.VectorMemTableConfig org.rocksdb.SkipListMemTableConfig org.rocksdb.HashLinkedListMemTableConfig org.rocksdb.HashSkipListMemTableConfig org.rocksdb.PlainTableConfig org.rocksdb.ReadOptions org.rocksdb.Filter org.rocksdb.BloomFilter
 NATIVE_INCLUDE = ./include
 ROCKSDB_JAR = rocksdbjni.jar
 
@@ -8,17 +8,24 @@ clean:
 	rm -f $(ROCKSDB_JAR)
 
 java:
-	javac org/rocksdb/*.java
-	jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class
+	javac org/rocksdb/util/*.java org/rocksdb/*.java
+	jar -cf $(ROCKSDB_JAR) org/rocksdb/*.class org/rocksdb/util/*.class
 	javah -d $(NATIVE_INCLUDE) -jni $(NATIVE_JAVA_CLASSES)
 
-sample:
+sample: java
 	javac -cp $(ROCKSDB_JAR) RocksDBSample.java
 	@rm -rf /tmp/rocksdbjni
 	@rm -rf /tmp/rocksdbjni_not_found
-	java -ea -Djava.library.path=.:../ -cp ".:./*" RocksDBSample /tmp/rocksdbjni
+	java -ea -Djava.library.path=.:../ -cp ".:./*" -Xcheck:jni RocksDBSample /tmp/rocksdbjni
 	@rm -rf /tmp/rocksdbjni
 	@rm -rf /tmp/rocksdbjni_not_found
 
-test:
+test: java
+	javac org/rocksdb/test/*.java
 	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.WriteBatchTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.BackupableDBTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.OptionsTest
+	java -ea -Djava.library.path=.:../ -cp "$(ROCKSDB_JAR):.:./*" org.rocksdb.test.ReadOptionsTest
+
+db_bench: java
+	javac org/rocksdb/benchmark/*.java

java/RocksDBSample.java

@@ -3,14 +3,17 @@
 // LICENSE file in the root directory of this source tree. An additional grant
 // of patent rights can be found in the PATENTS file in the same directory.
 
-import java.util.*;
-import java.lang.*;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.ArrayList;
 import org.rocksdb.*;
+import org.rocksdb.util.SizeUnit;
 import java.io.IOException;
 
 public class RocksDBSample {
   static {
-    System.loadLibrary("rocksdbjni");
+    RocksDB.loadLibrary();
   }
 
   public static void main(String[] args) {
@@ -32,7 +35,47 @@ public class RocksDBSample {
       assert(db == null);
     }
 
-    options.setCreateIfMissing(true);
+    Filter filter = new BloomFilter(10);
+    options.setCreateIfMissing(true)
+        .createStatistics()
+        .setWriteBufferSize(8 * SizeUnit.KB)
+        .setMaxWriteBufferNumber(3)
+        .setDisableSeekCompaction(true)
+        .setBlockSize(64 * SizeUnit.KB)
+        .setMaxBackgroundCompactions(10)
+        .setFilter(filter);
+    Statistics stats = options.statisticsPtr();
+
+    assert(options.createIfMissing() == true);
+    assert(options.writeBufferSize() == 8 * SizeUnit.KB);
+    assert(options.maxWriteBufferNumber() == 3);
+    assert(options.disableSeekCompaction() == true);
+    assert(options.blockSize() == 64 * SizeUnit.KB);
+    assert(options.maxBackgroundCompactions() == 10);
+
+    assert(options.memTableFactoryName().equals("SkipListFactory"));
+    options.setMemTableConfig(
+        new HashSkipListMemTableConfig()
+            .setHeight(4)
+            .setBranchingFactor(4)
+            .setBucketCount(2000000));
+    assert(options.memTableFactoryName().equals("HashSkipListRepFactory"));
+
+    options.setMemTableConfig(
+        new HashLinkedListMemTableConfig()
+            .setBucketCount(100000));
+    assert(options.memTableFactoryName().equals("HashLinkedListRepFactory"));
+
+    options.setMemTableConfig(
+        new VectorMemTableConfig().setReservedSize(10000));
+    assert(options.memTableFactoryName().equals("VectorRepFactory"));
+
+    options.setMemTableConfig(new SkipListMemTableConfig());
+    assert(options.memTableFactoryName().equals("SkipListFactory"));
+
+    options.setTableFormatConfig(new PlainTableConfig());
+    assert(options.tableFactoryName().equals("PlainTable"));
+
     try {
       db = RocksDB.open(options, db_path_not_found);
       db.put("hello".getBytes(), "world".getBytes());
@@ -43,13 +86,14 @@ public class RocksDBSample {
       assert(db == null);
       assert(false);
     }
-
     // be sure to release the c++ pointer
-    options.dispose();
     db.close();
 
+    ReadOptions readOptions = new ReadOptions();
+    readOptions.setFillCache(false);
+
     try {
-      db = RocksDB.open(db_path);
+      db = RocksDB.open(options, db_path);
       db.put("hello".getBytes(), "world".getBytes());
       byte[] value = db.get("hello".getBytes());
       System.out.format("Get('hello') = %s\n",
@@ -74,6 +118,8 @@ public class RocksDBSample {
       assert(value != null);
       value = db.get("world".getBytes());
       assert(value == null);
+      value = db.get(readOptions, "world".getBytes());
+      assert(value == null);
 
       byte[] testKey = "asdf".getBytes();
       byte[] testValue =
@@ -83,6 +129,10 @@ public class RocksDBSample {
       assert(testResult != null);
       assert(Arrays.equals(testValue, testResult));
       assert(new String(testValue).equals(new String(testResult)));
+      testResult = db.get(readOptions, testKey);
+      assert(testResult != null);
+      assert(Arrays.equals(testValue, testResult));
+      assert(new String(testValue).equals(new String(testResult)));
 
       byte[] insufficientArray = new byte[10];
       byte[] enoughArray = new byte[50];
@@ -94,6 +144,13 @@ public class RocksDBSample {
       len = db.get(testKey, enoughArray);
       assert(len == testValue.length);
 
+      len = db.get(readOptions, testKey, insufficientArray);
+      assert(len > insufficientArray.length);
+      len = db.get(readOptions, "asdfjkl;".getBytes(), enoughArray);
+      assert(len == RocksDB.NOT_FOUND);
+      len = db.get(readOptions, testKey, enoughArray);
+      assert(len == testValue.length);
+
       db.remove(testKey);
       len = db.get(testKey, enoughArray);
       assert(len == RocksDB.NOT_FOUND);
@@ -108,11 +165,89 @@ public class RocksDBSample {
       assert(new String(testValue).equals(
           new String(enoughArray, 0, len)));
       writeOpts.dispose();
+
+      try {
+        for (TickerType statsType : TickerType.values()) {
+          stats.getTickerCount(statsType);
+        }
+        System.out.println("getTickerCount() passed.");
+      } catch (Exception e) {
+        System.out.println("Failed in call to getTickerCount()");
+        assert(false); //Should never reach here.
+      }
+
+      try {
+        for (HistogramType histogramType : HistogramType.values()) {
+          HistogramData data = stats.geHistogramData(histogramType);
+        }
+        System.out.println("geHistogramData() passed.");
+      } catch (Exception e) {
+        System.out.println("Failed in call to geHistogramData()");
+        assert(false); //Should never reach here.
+      }
+
+      Iterator iterator = db.newIterator();
+
+      boolean seekToFirstPassed = false;
+      for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) {
+        iterator.status();
+        assert(iterator.key() != null);
+        assert(iterator.value() != null);
+        seekToFirstPassed = true;
+      }
+      if(seekToFirstPassed) {
+        System.out.println("iterator seekToFirst tests passed.");
+      }
+
+      boolean seekToLastPassed = false;
+      for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
+        iterator.status();
+        assert(iterator.key() != null);
+        assert(iterator.value() != null);
+        seekToLastPassed = true;
+      }
+
+      if(seekToLastPassed) {
+        System.out.println("iterator seekToLastPassed tests passed.");
+      }
+
+      iterator.seekToFirst();
+      iterator.seek(iterator.key());
+      assert(iterator.key() != null);
+      assert(iterator.value() != null);
+
+      System.out.println("iterator seek test passed.");
+
+      iterator.dispose();
+      System.out.println("iterator tests passed.");
+
+      iterator = db.newIterator();
+      List<byte[]> keys = new ArrayList<byte[]>();
+      for (iterator.seekToLast(); iterator.isValid(); iterator.prev()) {
+        keys.add(iterator.key());
+      }
+      iterator.dispose();
+
+      Map<byte[], byte[]> values = db.multiGet(keys);
+      assert(values.size() == keys.size());
+      for(byte[] value1 : values.values()) {
+        assert(value1 != null);
+      }
+
+      values = db.multiGet(new ReadOptions(), keys);
+      assert(values.size() == keys.size());
+      for(byte[] value1 : values.values()) {
+        assert(value1 != null);
+      }
     } catch (RocksDBException e) {
       System.err.println(e);
     }
     if (db != null) {
       db.close();
     }
+    // be sure to dispose c++ pointers
+    options.dispose();
+    readOptions.dispose();
+    filter.dispose();
   }
 }

java/jdb_bench.sh

@@ -0,0 +1 @@
+java -server -d64 -XX:NewSize=4m -XX:+AggressiveOpts -Djava.library.path=.:../ -cp "rocksdbjni.jar:.:./*" org.rocksdb.benchmark.DbBenchmark $@

java/org/rocksdb/BackupableDB.java

@@ -0,0 +1,80 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * A subclass of RocksDB which supports backup-related operations.
+ *
+ * @see BackupableDBOptions
+ */
+public class BackupableDB extends RocksDB {
+  /**
+   * Open a BackupableDB under the specified path.
+   * Note that the backup path should be set properly in the
+   * input BackupableDBOptions.
+   *
+   * @param opt options for db.
+   * @param bopt backup related options.
+   * @param the db path for storing data.  The path for storing
+   *     backup should be specified in the BackupableDBOptions.
+   * @return reference to the opened BackupableDB.
+   */
+  public static BackupableDB open(
+      Options opt, BackupableDBOptions bopt, String db_path)
+      throws RocksDBException {
+    // since BackupableDB c++ will handle the life cycle of
+    // the returned RocksDB of RocksDB.open(), here we store
+    // it as a BackupableDB member variable to avoid GC.
+    BackupableDB bdb = new BackupableDB(RocksDB.open(opt, db_path));
+    bdb.open(bdb.db_.nativeHandle_, bopt.nativeHandle_);
+
+    return bdb;
+  }
+
+  /**
+   * Captures the state of the database in the latest backup.
+   * Note that this function is not thread-safe.
+   *
+   * @param flushBeforeBackup if true, then all data will be flushed
+   *     before creating backup.
+   */
+  public void createNewBackup(boolean flushBeforeBackup) {
+    createNewBackup(nativeHandle_, flushBeforeBackup);
+  }
+
+
+  /**
+   * Close the BackupableDB instance and release resource.
+   *
+   * Internally, BackupableDB owns the rocksdb::DB pointer to its
+   * associated RocksDB.  The release of that RocksDB pointer is
+   * handled in the destructor of the c++ rocksdb::BackupableDB and
+   * should be transparent to Java developers.
+   */
+  @Override public synchronized void close() {
+    if (isInitialized()) {
+      super.close();
+    }
+  }
+
+  /**
+   * A protected construction that will be used in the static factory
+   * method BackupableDB.open().
+   */
+  protected BackupableDB(RocksDB db) {
+    super();
+    db_ = db;
+  }
+
+  @Override protected void finalize() {
+    close();
+  }
+
+  protected native void open(long rocksDBHandle, long backupDBOptionsHandle);
+  protected native void createNewBackup(long handle, boolean flag);
+
+  private final RocksDB db_;
+}

java/org/rocksdb/BackupableDBOptions.java

@@ -0,0 +1,44 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * BackupableDBOptions to control the behavior of a backupable database.
+ * It will be used during the creation of a BackupableDB.
+ *
+ * Note that dispose() must be called before an Options instance
+ * become out-of-scope to release the allocated memory in c++.
+ */
+public class BackupableDBOptions extends RocksObject {
+  public BackupableDBOptions(String path) {
+    super();
+    newBackupableDBOptions(path);
+  }
+
+  /**
+   * Returns the path to the BackupableDB directory.
+   *
+   * @return the path to the BackupableDB directory.
+   */
+  public String backupDir() {
+    assert(isInitialized());
+    return backupDir(nativeHandle_);
+  }
+
+  /**
+   * Release the memory allocated for the current instance
+   * in the c++ side.
+   */
+  @Override public synchronized void dispose() {
+    if (isInitialized()) {
+      dispose(nativeHandle_);
+    }
+  }
+
+  private native void newBackupableDBOptions(String path);
+  private native String backupDir(long handle);
+  private native void dispose(long handle);
+}

java/org/rocksdb/BloomFilter.java

@@ -0,0 +1,37 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * This class creates a new filter policy that uses a bloom filter
+ * with approximately the specified number of bits per key.
+ * A good value for bitsPerKey is 10, which yields a filter
+ * with ~ 1% false positive rate.
+ *
+ * Default value of bits per key is 10.
+ */
+public class BloomFilter extends Filter {
+  private static final int DEFAULT_BITS_PER_KEY = 10;
+  private final int bitsPerKey_;
+
+  public BloomFilter() {
+    this(DEFAULT_BITS_PER_KEY);
+  }
+
+  public BloomFilter(int bitsPerKey) {
+    super();
+    bitsPerKey_ = bitsPerKey;
+
+    createNewFilter();
+  }
+
+  @Override
+  protected void createNewFilter() {
+    createNewFilter0(bitsPerKey_);
+  }
+
+  private native void createNewFilter0(int bitsKeyKey);
+}

java/org/rocksdb/Filter.java

@@ -0,0 +1,32 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * Filters are stored in rocksdb and are consulted automatically
+ * by rocksdb to decide whether or not to read some
+ * information from disk. In many cases, a filter can cut down the
+ * number of disk seeks form a handful to a single disk seek per
+ * DB::Get() call.
+ */
+public abstract class Filter extends RocksObject {
+  protected abstract void createNewFilter();
+
+  /**
+   * Deletes underlying C++ filter pointer.
+   *
+   * Note that this function should be called only after all
+   * RocksDB instances referencing the filter are closed.
+   * Otherwise an undefined behavior will occur.
+   */
+  @Override public synchronized void dispose() {
+    if (isInitialized()) {
+      dispose0(nativeHandle_);
+    }
+  }
+
+  private native void dispose0(long handle);
+}

java/org/rocksdb/HashLinkedListMemTableConfig.java

@@ -0,0 +1,52 @@
+package org.rocksdb;
+
+/**
+ * The config for hash linked list memtable representation
+ * Such memtable contains a fix-sized array of buckets, where
+ * each bucket points to a sorted singly-linked
+ * list (or null if the bucket is empty).
+ *
+ * Note that since this mem-table representation relies on the
+ * key prefix, it is required to invoke one of the usePrefixExtractor
+ * functions to specify how to extract key prefix given a key.
+ * If proper prefix-extractor is not set, then RocksDB will
+ * use the default memtable representation (SkipList) instead
+ * and post a warning in the LOG.
+ */
+public class HashLinkedListMemTableConfig extends MemTableConfig {
+  public static final long DEFAULT_BUCKET_COUNT = 50000;
+
+  public HashLinkedListMemTableConfig() {
+    bucketCount_ = DEFAULT_BUCKET_COUNT;
+  }
+
+  /**
+   * Set the number of buckets in the fixed-size array used
+   * in the hash linked-list mem-table.
+   *
+   * @param count the number of hash buckets.
+   * @return the reference to the current HashLinkedListMemTableConfig.
+   */
+  public HashLinkedListMemTableConfig setBucketCount(long count) {
+    bucketCount_ = count;
+    return this;
+  }
+
+  /**
+   * Returns the number of buckets that will be used in the memtable
+   * created based on this config.
+   *
+   * @return the number of buckets
+   */
+  public long bucketCount() {
+    return bucketCount_;
+  }
+
+  @Override protected long newMemTableFactoryHandle() {
+    return newMemTableFactoryHandle(bucketCount_);
+  }
+
+  private native long newMemTableFactoryHandle(long bucketCount);
+
+  private long bucketCount_;
+}

java/org/rocksdb/HashSkipListMemTableConfig.java

@@ -0,0 +1,97 @@
+package org.rocksdb;
+
+/**
+ * The config for hash skip-list mem-table representation.
+ * Such mem-table representation contains a fix-sized array of
+ * buckets, where each bucket points to a skiplist (or null if the
+ * bucket is empty).
+ *
+ * Note that since this mem-table representation relies on the
+ * key prefix, it is required to invoke one of the usePrefixExtractor
+ * functions to specify how to extract key prefix given a key.
+ * If proper prefix-extractor is not set, then RocksDB will
+ * use the default memtable representation (SkipList) instead
+ * and post a warning in the LOG.
+ */
+public class HashSkipListMemTableConfig extends MemTableConfig {
+  public static final int DEFAULT_BUCKET_COUNT = 1000000;
+  public static final int DEFAULT_BRANCHING_FACTOR = 4;
+  public static final int DEFAULT_HEIGHT = 4;
+
+  public HashSkipListMemTableConfig() {
+    bucketCount_ = DEFAULT_BUCKET_COUNT;
+    branchingFactor_ = DEFAULT_BRANCHING_FACTOR;
+    height_ = DEFAULT_HEIGHT;
+  }
+
+  /**
+   * Set the number of hash buckets used in the hash skiplist memtable.
+   * Default = 1000000.
+   *
+   * @param count the number of hash buckets used in the hash
+   *    skiplist memtable.
+   * @return the reference to the current HashSkipListMemTableConfig.
+   */
+  public HashSkipListMemTableConfig setBucketCount(long count) {
+    bucketCount_ = count;
+    return this;
+  }
+
+  /**
+   * @return the number of hash buckets
+   */
+  public long bucketCount() {
+    return bucketCount_;
+  }
+
+  /**
+   * Set the height of the skip list.  Default = 4.
+   *
+   * @return the reference to the current HashSkipListMemTableConfig.
+   */
+  public HashSkipListMemTableConfig setHeight(int height) {
+    height_ = height;
+    return this;
+  }
+
+  /**
+   * @return the height of the skip list.
+   */
+  public int height() {
+    return height_;
+  }
+
+  /**
+   * Set the branching factor used in the hash skip-list memtable.
+   * This factor controls the probabilistic size ratio between adjacent
+   * links in the skip list.
+   *
+   * @param bf the probabilistic size ratio between adjacent link
+   *     lists in the skip list.
+   * @return the reference to the current HashSkipListMemTableConfig.
+   */
+  public HashSkipListMemTableConfig setBranchingFactor(int bf) {
+    branchingFactor_ = bf;
+    return this;
+  }
+
+  /**
+   * @return branching factor, the probabilistic size ratio between
+   *     adjacent links in the skip list.
+   */
+  public int branchingFactor() {
+    return branchingFactor_;
+  }
+
+  @Override protected long newMemTableFactoryHandle() {
+    return newMemTableFactoryHandle(
+        bucketCount_, height_, branchingFactor_);
+  }
+
+  private native long newMemTableFactoryHandle(
+      long bucketCount, int height, int branchingFactor);
+
+  private long bucketCount_;
+  private int branchingFactor_;
+  private int height_;
+}

java/org/rocksdb/HistogramData.java

@@ -0,0 +1,43 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+public class HistogramData {
+  private final double median_;
+  private final double percentile95_;
+  private final double percentile99_;
+  private final double average_;
+  private final double standardDeviation_;
+
+  public HistogramData(double median, double percentile95,
+      double percentile99, double average, double standardDeviation) {
+    median_ = median;
+    percentile95_ = percentile95;
+    percentile99_ = percentile99;
+    average_ = average;
+    standardDeviation_ = standardDeviation;
+  }
+
+  public double getMedian() {
+    return median_;
+  }
+
+  public double getPercentile95() {
+    return percentile95_;
+  }
+
+  public double getPercentile99() {
+    return percentile99_;
+  }
+
+  public double getAverage() {
+    return average_;
+  }
+
+  public double getStandardDeviation() {
+    return standardDeviation_;
+  }
+}

java/org/rocksdb/HistogramType.java

@@ -0,0 +1,39 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+public enum HistogramType {
+  DB_GET(0),
+  DB_WRITE(1),
+  COMPACTION_TIME(2),
+  TABLE_SYNC_MICROS(3),
+  COMPACTION_OUTFILE_SYNC_MICROS(4),
+  WAL_FILE_SYNC_MICROS(5),
+  MANIFEST_FILE_SYNC_MICROS(6),
+  // TIME SPENT IN IO DURING TABLE OPEN
+  TABLE_OPEN_IO_MICROS(7),
+  DB_MULTIGET(8),
+  READ_BLOCK_COMPACTION_MICROS(9),
+  READ_BLOCK_GET_MICROS(10),
+  WRITE_RAW_BLOCK_MICROS(11),
+
+  STALL_L0_SLOWDOWN_COUNT(12),
+  STALL_MEMTABLE_COMPACTION_COUNT(13),
+  STALL_L0_NUM_FILES_COUNT(14),
+  HARD_RATE_LIMIT_DELAY_COUNT(15),
+  SOFT_RATE_LIMIT_DELAY_COUNT(16),
+  NUM_FILES_IN_SINGLE_COMPACTION(17);
+
+  private final int value_;
+
+  private HistogramType(int value) {
+    value_ = value;
+  }
+
+  public int getValue() {
+    return value_;
+  }
+}

java/org/rocksdb/Iterator.java

@@ -0,0 +1,138 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+
+package org.rocksdb;
+
+/**
+ * An iterator yields a sequence of key/value pairs from a source.
+ * The following class defines the interface.  Multiple implementations
+ * are provided by this library.  In particular, iterators are provided
+ * to access the contents of a Table or a DB.
+ *
+ * Multiple threads can invoke const methods on an Iterator without
+ * external synchronization, but if any of the threads may call a
+ * non-const method, all threads accessing the same Iterator must use
+ * external synchronization.
+ */
+public class Iterator extends RocksObject {
+  public Iterator(long nativeHandle) {
+    super();
+    nativeHandle_ = nativeHandle;
+  }
+
+  /**
+   * An iterator is either positioned at a key/value pair, or
+   * not valid.  This method returns true iff the iterator is valid.
+   * @return true if iterator is valid.
+   */
+  public boolean isValid() {
+    assert(isInitialized());
+    return isValid0(nativeHandle_);
+  }
+
+  /**
+   * Position at the first key in the source.  The iterator is Valid()
+   * after this call iff the source is not empty.
+   */
+  public void seekToFirst() {
+    assert(isInitialized());
+    seekToFirst0(nativeHandle_);
+  }
+
+  /**
+   * Position at the last key in the source.  The iterator is
+   * Valid() after this call iff the source is not empty.
+   */
+  public void seekToLast() {
+    assert(isInitialized());
+    seekToLast0(nativeHandle_);
+  }
+
+  /**
+   * Moves to the next entry in the source.  After this call, Valid() is
+   * true iff the iterator was not positioned at the last entry in the source.
+   * REQUIRES: Valid()
+   */
+  public void next() {
+    assert(isInitialized());
+    next0(nativeHandle_);
+  }
+
+  /**
+   * Moves to the previous entry in the source.  After this call, Valid() is
+   * true iff the iterator was not positioned at the first entry in source.
+   * REQUIRES: Valid()
+   */
+  public void prev() {
+    assert(isInitialized());
+    prev0(nativeHandle_);
+  }
+
+  /**
+   * Return the key for the current entry.  The underlying storage for
+   * the returned slice is valid only until the next modification of
+   * the iterator.
+   * REQUIRES: Valid()
+   * @return key for the current entry.
+   */
+  public byte[] key() {
+    assert(isInitialized());
+    return key0(nativeHandle_);
+  }
+
+  /**
+   * Return the value for the current entry.  The underlying storage for
+   * the returned slice is valid only until the next modification of
+   * the iterator.
+   * REQUIRES: !AtEnd() && !AtStart()
+   * @return value for the current entry.
+   */
+  public byte[] value() {
+    assert(isInitialized());
+    return value0(nativeHandle_);
+  }
+
+  /**
+   * Position at the first key in the source that at or past target
+   * The iterator is Valid() after this call iff the source contains
+   * an entry that comes at or past target.
+   */
+  public void seek(byte[] target) {
+    assert(isInitialized());
+    seek0(nativeHandle_, target, target.length);
+  }
+
+  /**
+   * If an error has occurred, return it.  Else return an ok status.
+   * If non-blocking IO is requested and this operation cannot be
+   * satisfied without doing some IO, then this returns Status::Incomplete().
+   *
+   */
+  public void status() throws RocksDBException {
+    assert(isInitialized());
+    status0(nativeHandle_);
+  }
+
+  /**
+   * Deletes underlying C++ iterator pointer.
+   */
+  @Override public synchronized void dispose() {
+    if(isInitialized()) {
+      dispose(nativeHandle_);
+      nativeHandle_ = 0;
+    }
+  }
+
+  private native boolean isValid0(long handle);
+  private native void dispose(long handle);
+  private native void seekToFirst0(long handle);
+  private native void seekToLast0(long handle);
+  private native void next0(long handle);
+  private native void prev0(long handle);
+  private native byte[] key0(long handle);
+  private native byte[] value0(long handle);
+  private native void seek0(long handle, byte[] target, int targetLen);
+  private native void status0(long handle);
+}

java/org/rocksdb/MemTableConfig.java

@@ -0,0 +1,27 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+package org.rocksdb;
+
+/**
+ * MemTableConfig is used to config the internal mem-table of a RocksDB.
+ * It is required for each memtable to have one such sub-class to allow
+ * Java developers to use it.
+ *
+ * To make a RocksDB to use a specific MemTable format, its associated
+ * MemTableConfig should be properly set and passed into Options
+ * via Options.setMemTableFactory() and open the db using that Options.
+ *
+ * @see Options
+ */
+public abstract class MemTableConfig {
+  /**
+   * This function should only be called by Options.setMemTableConfig(),
+   * which will create a c++ shared-pointer to the c++ MemTableRepFactory
+   * that associated with the Java MemTableConfig.
+   *
+   * @see Options.setMemTableFactory()
+   */
+  abstract protected long newMemTableFactoryHandle();
+}

java/org/rocksdb/PlainTableConfig.java

@@ -0,0 +1,123 @@
+// Copyright (c) 2014, Facebook, Inc.  All rights reserved.
+// This source code is licensed under the BSD-style license found in the
+// LICENSE file in the root directory of this source tree. An additional grant
+// of patent rights can be found in the PATENTS file in the same directory.
+package org.rocksdb;
+
+/**
+ * The config for plain table sst format.
+ *
+ * PlainTable is a RocksDB's SST file format optimized for low query latency
+ * on pure-memory or really low-latency media.  It also support prefix
+ * hash feature.
+ */
+public class PlainTableConfig extends TableFormatConfig {
+  public static final int VARIABLE_LENGTH = 0;
+  public static final int DEFAULT_BLOOM_BITS_PER_KEY = 10;
+  public static final double DEFAULT_HASH_TABLE_RATIO = 0.75;
+  public static final int DEFAULT_INDEX_SPARSENESS = 16;
+
+  public PlainTableConfig() {
+    keySize_ = VARIABLE_LENGTH;
+    bloomBitsPerKey_ = DEFAULT_BLOOM_BITS_PER_KEY;
+    hashTableRatio_ = DEFAULT_HASH_TABLE_RATIO;
+    indexSparseness_ = DEFAULT_INDEX_SPARSENESS;
+  }
+
+  /**
+   * Set the length of the user key. If it is set to be VARIABLE_LENGTH,
+   * then it indicates the user keys are variable-lengthed.  Otherwise,
+   * all the keys need to have the same length in byte.
+   * DEFAULT: VARIABLE_LENGTH
+   *
+   * @param keySize the length of the user key.
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setKeySize(int keySize) {
+    keySize_ = keySize;
+    return this;
+  }
+
+  /**
+   * @return the specified size of the user key.  If VARIABLE_LENGTH,
+   *     then it indicates variable-length key.
+   */
+  public int keySize() {
+    return keySize_;
+  }
+
+  /**
+   * Set the number of bits per key used by the internal bloom filter
+   * in the plain table sst format.
+   *
+   * @param bitsPerKey the number of bits per key for bloom filer.
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setBloomBitsPerKey(int bitsPerKey) {
+    bloomBitsPerKey_ = bitsPerKey;
+    return this;
+  }
+
+  /**
+   * @return the number of bits per key used for the bloom filter.
+   */
+  public int bloomBitsPerKey() {
+    return bloomBitsPerKey_;
+  }
+
+  /**
+   * hashTableRatio is the desired utilization of the hash table used
+   * for prefix hashing.  The ideal ratio would be the number of
+   * prefixes / the number of hash buckets.  If this value is set to
+   * zero, then hash table will not be used.
+   *
+   * @param ratio the hash table ratio.
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setHashTableRatio(double ratio) {
+    hashTableRatio_ = ratio;
+    return this;
+  }
+
+  /**
+   * @return the hash table ratio.
+   */
+  public double hashTableRatio() {
+    return hashTableRatio_;
+  }
+
+  /**
+   * Index sparseness determines the index interval for keys inside the
+   * same prefix.  This number is equal to the maximum number of linear
+   * search required after hash and binary search.  If it's set to 0,
+   * then each key will be indexed.
+   *
+   * @param sparseness the index sparseness.
+   * @return the reference to the current config.
+   */
+  public PlainTableConfig setIndexSparseness(int sparseness) {
+    indexSparseness_ = sparseness;
+    return this;
+  }
+
+  /**
+   * @return the index sparseness.
+   */
+  public int indexSparseness() {
+    return indexSparseness_;
+  }
+
+  @Override protected long newTableFactoryHandle() {
+    return newTableFactoryHandle(keySize_, bloomBitsPerKey_,
+        hashTableRatio_, indexSparseness_);
+  }
+
+  private native long newTableFactoryHandle(
+      int keySize, int bloomBitsPerKey,
+      double hashTableRatio, int indexSparseness);
+
+  private int keySize_;
+  private int bloomBitsPerKey_;
+  private double hashTableRatio_;
+  private int indexSparseness_;
+}