Updating readme file for version 2.3

Summary:

Test Plan:

Reviewers:

CC:

Task ID: #

Blame Rev:

.gitignore

@@ -21,3 +21,4 @@ util/build_version.cc
 build_tools/VALGRIND_LOGS/
 coverage/COVERAGE_REPORT
 util/build_version.cc.tmp
+.gdbhistory

Makefile

@@ -14,7 +14,7 @@ OPT += -O2 -fno-omit-frame-pointer -momit-leaf-frame-pointer
 #-----------------------------------------------
 
 # detect what platform we're building on
-$(shell (cd build_tools/; ROCKSDB_ROOT=.. ./build_detect_platform ../build_config.mk))
+$(shell (export ROCKSDB_ROOT=$(CURDIR); $(CURDIR)/build_tools/build_detect_platform $(CURDIR)/build_config.mk))
 # this file is generated by the previous line to set build flags and sources
 include build_config.mk
 
@@ -64,7 +64,8 @@ TESTS = \
 	ttl_test \
 	version_edit_test \
 	version_set_test \
-	write_batch_test
+	write_batch_test\
+	deletefile_test
 
 TOOLS = \
         sst_dump \
@@ -121,7 +122,7 @@ release:
 
 coverage:
 	$(MAKE) clean
-	COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" $(MAKE) all check
+	COVERAGEFLAGS="-fprofile-arcs -ftest-coverage" LDFLAGS+="-lgcov" $(MAKE) all check
 	(cd coverage; ./coverage_test.sh)
 	# Delete intermediate files
 	find . -type f -regex ".*\.\(\(gcda\)\|\(gcno\)\)" | xargs --no-run-if-empty rm
@@ -266,6 +267,9 @@ write_batch_test: db/write_batch_test.o $(LIBOBJECTS) $(TESTHARNESS)
 merge_test: db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(CXX) db/merge_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS) $(COVERAGEFLAGS)
 
+deletefile_test: db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(CXX) db/deletefile_test.o $(LIBOBJECTS) $(TESTHARNESS) $(EXEC_LDFLAGS) -o $@ $(LDFLAGS)
+
 $(MEMENVLIBRARY) : $(MEMENVOBJECTS)
 	rm -f $@
 	$(AR) -rs $@ $(MEMENVOBJECTS)

README.fb

@@ -1,3 +1,3 @@
 * Detailed instructions on how to compile using fbcode and jemalloc
 
-* Latest release is 2.1.fb
+* Latest release is 2.3.fb

build_tools/build_detect_platform

@@ -31,9 +31,9 @@ fi
 # Default to fbcode gcc on internal fb machines
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
     if [ -z "$USE_CLANG" ]; then
-        source ./fbcode.gcc471.sh
+        source $PWD/build_tools/fbcode.gcc471.sh
     else
-        source ./fbcode.clang31.sh
+        source $PWD/build_tools/fbcode.clang31.sh
     fi
 fi
 
@@ -65,7 +65,7 @@ PLATFORM_SHARED_CFLAGS="-fPIC"
 PLATFORM_SHARED_VERSIONED=true
 
 # generic port files (working on all platform by #ifdef) go directly in /port
-GENERIC_PORT_FILES=`cd $ROCKSDB_ROOT; find port -name '*.cc' | tr "\n" " "`
+GENERIC_PORT_FILES=`find $ROCKSDB_ROOT/port -name '*.cc' | tr "\n" " "`
 
 # On GCC, we pick libc's memcmp over GCC's memcmp via -fno-builtin-memcmp
 case "$TARGET_OS" in
@@ -82,7 +82,7 @@ case "$TARGET_OS" in
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
         fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
         ;;
     SunOS)
@@ -127,7 +127,7 @@ case "$TARGET_OS" in
         exit 1
 esac
 
-./build_detect_version
+$PWD/build_tools/build_detect_version
 
 # We want to make a list of all cc files within util, db, table, and helpers
 # except for the test and benchmark files. By default, find will output a list

build_tools/build_detect_version

@@ -19,15 +19,15 @@ if [ "$?" = 0 ]; then
   awk '
   BEGIN {
     print "#include \"build_version.h\"\n"
-  } 
+  }
   { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:" $0"\";" }
   ' >  ${VFILE}
 else
   echo "git not found" |
   awk '
     BEGIN {
-      print "#include \"build_version.h\"" 
-    } 
+      print "#include \"build_version.h\""
+    }
     { print "const char* leveldb_build_git_sha = \"leveldb_build_git_sha:git not found\";" }
     ' > ${VFILE}
 fi

coverage/coverage_test.sh

@@ -55,12 +55,19 @@ then
   exit 0
 fi
 
+LCOV_VERSION=$(lcov -v | grep 1.1 || true)
+if [ $LCOV_VERSION ]
+then
+  echo "Not supported lcov version. Expect lcov 1.1."
+  exit 0
+fi
+
 (cd $ROOT; lcov --no-external \
      --capture  \
      --directory $PWD \
      --gcov-tool $GCOV \
-     --output-file $COVERAGE_DIR/coverage.info &>/dev/null)
+     --output-file $COVERAGE_DIR/coverage.info)
 
-genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR &>/dev/null
+genhtml $COVERAGE_DIR/coverage.info -o $COVERAGE_DIR
 
 echo "HTML Coverage report is generated in $COVERAGE_DIR"

db/builder.cc

@@ -9,9 +9,9 @@
 #include "db/merge_helper.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
 #include "util/stop_watch.h"
 
 namespace leveldb {

db/builder.h

@@ -5,9 +5,9 @@
 #ifndef STORAGE_LEVELDB_DB_BUILDER_H_
 #define STORAGE_LEVELDB_DB_BUILDER_H_
 
-#include "leveldb/comparator.h"
-#include "leveldb/status.h"
-#include "leveldb/types.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/status.h"
+#include "rocksdb/types.h"
 
 namespace leveldb {
 

db/c.cc

@@ -2,19 +2,19 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "leveldb/c.h"
+#include "rocksdb/c.h"
 
 #include <stdlib.h>
 #include <unistd.h>
-#include "leveldb/cache.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-#include "leveldb/status.h"
-#include "leveldb/write_batch.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
+#include "rocksdb/write_batch.h"
 
 using leveldb::Cache;
 using leveldb::Comparator;

db/c_test.c

@@ -2,7 +2,7 @@
    Use of this source code is governed by a BSD-style license that can be
    found in the LICENSE file. See the AUTHORS file for names of contributors. */
 
-#include "leveldb/c.h"
+#include "rocksdb/c.h"
 
 #include <stddef.h>
 #include <stdio.h>

db/corruption_test.cc

@@ -2,15 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <sys/types.h>
-#include "leveldb/cache.h"
-#include "leveldb/env.h"
-#include "leveldb/write_batch.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/env.h"
+#include "rocksdb/write_batch.h"
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "db/log_format.h"

db/db_bench.cc

@@ -9,12 +9,12 @@
 #include "db/db_impl.h"
 #include "db/version_set.h"
 #include "db/db_statistics.h"
-#include "leveldb/options.h"
-#include "leveldb/cache.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/write_batch.h"
-#include "leveldb/statistics.h"
+#include "rocksdb/options.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/write_batch.h"
+#include "rocksdb/statistics.h"
 #include "port/port.h"
 #include "util/bit_set.h"
 #include "util/crc32c.h"
@@ -41,8 +41,9 @@
 //      readrandom    -- read N times in random order
 //      readmissing   -- read N missing keys in random order
 //      readhot       -- read N times in random order from 1% section of DB
-//      readwhilewriting -- 1 writer, N threads doing random reads
-//      readrandomwriterandom - N threads doing random-read, random-write
+//      readwhilewriting      -- 1 writer, N threads doing random reads
+//      readrandomwriterandom -- N threads doing random-read, random-write
+//      prefixscanrandom      -- prefix scan N times in random order
 //      updaterandom  -- N threads doing read-modify-write for random keys
 //      appendrandom  -- N threads doing read-modify-write with growing values
 //      mergerandom   -- same as updaterandom/appendrandom using merge operator
@@ -82,12 +83,12 @@ static const char* FLAGS_benchmarks =
 // the maximum size of key in bytes
 static const int MAX_KEY_SIZE = 128;
 // Number of key/values to place in database
-static long FLAGS_num = 1000000;
+static long long FLAGS_num = 1000000;
 
 // Number of distinct keys to use. Used in RandomWithVerify to read/write
 // on fewer keys so that gets are more likely to find the key and puts
 // are more likely to update the same key
-static long FLAGS_numdistinct = 1000;
+static long long FLAGS_numdistinct = 1000;
 
 // Number of read operations to do.  If negative, do FLAGS_num reads.
 static long FLAGS_reads = -1;
@@ -95,6 +96,13 @@ static long FLAGS_reads = -1;
 // When ==1 reads use ::Get, when >1 reads use an iterator
 static long FLAGS_read_range = 1;
 
+// Whether to place prefixes in blooms
+static bool FLAGS_use_prefix_blooms = false;
+
+// Whether to set ReadOptions.prefix for prefixscanrandom.  If this
+// true, use_prefix_blooms must also be true.
+static bool FLAGS_use_prefix_api = false;
+
 // Seed base for random number generators. When 0 it is deterministic.
 static long FLAGS_seed = 0;
 
@@ -278,7 +286,7 @@ static leveldb::Env* FLAGS_env = leveldb::Env::Default();
 
 // Stats are reported every N operations when this is greater
 // than zero. When 0 the interval grows over time.
-static int FLAGS_stats_interval = 0;
+static long long FLAGS_stats_interval = 0;
 
 // Reports additional stats per interval when this is greater
 // than 0.
@@ -353,6 +361,18 @@ static auto FLAGS_bytes_per_sync =
 // On true, deletes use bloom-filter and drop the delete if key not present
 static bool FLAGS_filter_deletes = false;
 
+// Control the prefix size for PrefixHashRep
+static bool FLAGS_prefix_size = 0;
+
+enum RepFactory {
+  kSkipList,
+  kPrefixHash,
+  kUnsorted,
+  kVectorRep
+};
+
+static enum RepFactory FLAGS_rep_factory;
+
 // The merge operator to use with the database.
 // If a new merge operator is specified, be sure to use fresh database
 // The possible merge operators are defined in utilities/merge_operators.h
@@ -418,9 +438,9 @@ class Stats {
   double start_;
   double finish_;
   double seconds_;
-  long done_;
-  long last_report_done_;
-  int next_report_;
+  long long done_;
+  long long last_report_done_;
+  long long next_report_;
   int64_t bytes_;
   double last_op_finish_;
   double last_report_finish_;
@@ -497,12 +517,13 @@ class Stats {
         else if (next_report_ < 100000) next_report_ += 10000;
         else if (next_report_ < 500000) next_report_ += 50000;
         else                            next_report_ += 100000;
-        fprintf(stderr, "... finished %ld ops%30s\r", done_, "");
+        fprintf(stderr, "... finished %lld ops%30s\r", done_, "");
         fflush(stderr);
       } else {
         double now = FLAGS_env->NowMicros();
         fprintf(stderr,
-                "%s ... thread %d: (%ld,%ld) ops and (%.1f,%.1f) ops/second in (%.6f,%.6f) seconds\n",
+                "%s ... thread %d: (%lld,%lld) ops and "
+                "(%.1f,%.1f) ops/second in (%.6f,%.6f) seconds\n",
                 FLAGS_env->TimeToString((uint64_t) now/1000000).c_str(),
                 id_,
                 done_ - last_report_done_, done_,
@@ -584,7 +605,7 @@ struct SharedState {
 // Per-thread state for concurrent executions of the same benchmark.
 struct ThreadState {
   int tid;             // 0..n-1 when running in n threads
-  Random rand;         // Has different seeds for different threads
+  Random64 rand;         // Has different seeds for different threads
   Stats stats;
   SharedState* shared;
 
@@ -596,7 +617,7 @@ struct ThreadState {
 
 class Duration {
  public:
-  Duration(int max_seconds, long max_ops) {
+  Duration(int max_seconds, long long max_ops) {
     max_seconds_ = max_seconds;
     max_ops_= max_ops;
     ops_ = 0;
@@ -622,8 +643,8 @@ class Duration {
 
  private:
   int max_seconds_;
-  long max_ops_;
-  long ops_;
+  long long max_ops_;
+  long long ops_;
   double start_at_;
 };
 
@@ -631,15 +652,16 @@ class Benchmark {
  private:
   shared_ptr<Cache> cache_;
   const FilterPolicy* filter_policy_;
+  const SliceTransform* prefix_extractor_;
   DB* db_;
-  long num_;
+  long long num_;
   int value_size_;
   int key_size_;
   int entries_per_batch_;
   WriteOptions write_options_;
-  long reads_;
-  long writes_;
-  long readwrites_;
+  long long reads_;
+  long long writes_;
+  long long readwrites_;
   int heap_counter_;
   char keyFormat_[100]; // this string will contain the format of key. e.g "%016d"
   void PrintHeader() {
@@ -648,7 +670,7 @@ class Benchmark {
     fprintf(stdout, "Values:     %d bytes each (%d bytes after compression)\n",
             FLAGS_value_size,
             static_cast<int>(FLAGS_value_size * FLAGS_compression_ratio + 0.5));
-    fprintf(stdout, "Entries:    %ld\n", num_);
+    fprintf(stdout, "Entries:    %lld\n", num_);
     fprintf(stdout, "RawSize:    %.1f MB (estimated)\n",
             ((static_cast<int64_t>(FLAGS_key_size + FLAGS_value_size) * num_)
              / 1048576.0));
@@ -672,6 +694,21 @@ class Benchmark {
         break;
     }
 
+    switch (FLAGS_rep_factory) {
+      case kPrefixHash:
+        fprintf(stdout, "Memtablerep: prefix_hash\n");
+        break;
+      case kSkipList:
+        fprintf(stdout, "Memtablerep: skip_list\n");
+        break;
+      case kUnsorted:
+        fprintf(stdout, "Memtablerep: unsorted\n");
+        break;
+      case kVectorRep:
+        fprintf(stdout, "Memtablerep: vector\n");
+        break;
+    }
+
     PrintWarnings();
     fprintf(stdout, "------------------------------------------------\n");
   }
@@ -773,6 +810,7 @@ class Benchmark {
     filter_policy_(FLAGS_bloom_bits >= 0
                    ? NewBloomFilterPolicy(FLAGS_bloom_bits)
                    : nullptr),
+    prefix_extractor_(NewFixedPrefixTransform(FLAGS_key_size-1)),
     db_(nullptr),
     num_(FLAGS_num),
     value_size_(FLAGS_value_size),
@@ -799,16 +837,17 @@ class Benchmark {
   ~Benchmark() {
     delete db_;
     delete filter_policy_;
+    delete prefix_extractor_;
   }
 
-  //this function will construct string format for key. e.g "%016d"
+  //this function will construct string format for key. e.g "%016lld"
   void ConstructStrFormatForKey(char* str, int keySize) {
     str[0] = '%';
     str[1] = '0';
-    sprintf(str+2, "%dd%s", keySize, "%s");
+    sprintf(str+2, "%dlld%s", keySize, "%s");
   }
 
-  unique_ptr<char []> GenerateKeyFromInt(int v, const char* suffix = "") {
+  unique_ptr<char []> GenerateKeyFromInt(long long v, const char* suffix = "") {
     unique_ptr<char []> keyInStr(new char[MAX_KEY_SIZE]);
     snprintf(keyInStr.get(), MAX_KEY_SIZE, keyFormat_, v, suffix);
     return keyInStr;
@@ -894,6 +933,8 @@ class Benchmark {
       } else if (name == Slice("readrandomsmall")) {
         reads_ /= 1000;
         method = &Benchmark::ReadRandom;
+      } else if (name == Slice("prefixscanrandom")) {
+        method = &Benchmark::PrefixScanRandom;
       } else if (name == Slice("deleteseq")) {
         method = &Benchmark::DeleteSeq;
       } else if (name == Slice("deleterandom")) {
@@ -1146,6 +1187,8 @@ class Benchmark {
       FLAGS_compaction_universal_min_merge_width;
     options.block_size = FLAGS_block_size;
     options.filter_policy = filter_policy_;
+    options.prefix_extractor = FLAGS_use_prefix_blooms ? prefix_extractor_
+                                                       : nullptr;
     options.max_open_files = FLAGS_open_files;
     options.statistics = dbstats;
     options.env = FLAGS_env;
@@ -1158,6 +1201,31 @@ class Benchmark {
     options.max_bytes_for_level_multiplier =
         FLAGS_max_bytes_for_level_multiplier;
     options.filter_deletes = FLAGS_filter_deletes;
+    if ((FLAGS_prefix_size == 0) == (FLAGS_rep_factory == kPrefixHash)) {
+      fprintf(stderr,
+            "prefix_size should be non-zero iff memtablerep == prefix_hash\n");
+      exit(1);
+    }
+    switch (FLAGS_rep_factory) {
+      case kPrefixHash:
+        options.memtable_factory.reset(
+          new PrefixHashRepFactory(NewFixedPrefixTransform(FLAGS_prefix_size))
+        );
+        break;
+      case kUnsorted:
+        options.memtable_factory.reset(
+          new UnsortedRepFactory
+        );
+        break;
+      case kSkipList:
+        // no need to do anything
+        break;
+      case kVectorRep:
+        options.memtable_factory.reset(
+          new VectorRepFactory
+        );
+        break;
+    }
     if (FLAGS_max_bytes_for_level_multiplier_additional.size() > 0) {
       if (FLAGS_max_bytes_for_level_multiplier_additional.size() !=
           (unsigned int)FLAGS_num_levels) {
@@ -1261,7 +1329,7 @@ class Benchmark {
 
     if (num_ != FLAGS_num) {
       char msg[100];
-      snprintf(msg, sizeof(msg), "(%ld ops)", num_);
+      snprintf(msg, sizeof(msg), "(%lld ops)", num_);
       thread->stats.AddMessage(msg);
     }
 
@@ -1273,7 +1341,7 @@ class Benchmark {
     while (!duration.Done(entries_per_batch_)) {
       batch.Clear();
       for (int j = 0; j < entries_per_batch_; j++) {
-        int k = 0;
+        long long k = 0;
         switch(write_mode) {
           case SEQUENTIAL:
             k = i +j;
@@ -1283,7 +1351,7 @@ class Benchmark {
             break;
           case UNIQUE_RANDOM:
             {
-              int t = thread->rand.Next() % FLAGS_num;
+              const long long t = thread->rand.Next() % FLAGS_num;
               if (!bit_set->test(t)) {
                 // best case
                 k = t;
@@ -1328,7 +1396,7 @@ class Benchmark {
 
   void ReadSequential(ThreadState* thread) {
     Iterator* iter = db_->NewIterator(ReadOptions(FLAGS_verify_checksum, true));
-    long i = 0;
+    long long i = 0;
     int64_t bytes = 0;
     for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
       bytes += iter->key().size() + iter->value().size();
@@ -1341,7 +1409,7 @@ class Benchmark {
 
   void ReadReverse(ThreadState* thread) {
     Iterator* iter = db_->NewIterator(ReadOptions(FLAGS_verify_checksum, true));
-    long i = 0;
+    long long i = 0;
     int64_t bytes = 0;
     for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
       bytes += iter->key().size() + iter->value().size();
@@ -1355,14 +1423,14 @@ class Benchmark {
   // Calls MultiGet over a list of keys from a random distribution.
   // Returns the total number of keys found.
   long MultiGetRandom(ReadOptions& options, int num_keys,
-                     Random& rand, int range, const char* suffix) {
+                     Random64& rand, long long range, const char* suffix) {
     assert(num_keys > 0);
     std::vector<Slice> keys(num_keys);
     std::vector<std::string> values(num_keys);
     std::vector<unique_ptr<char []> > gen_keys(num_keys);
 
     int i;
-    long k;
+    long long k;
 
     // Fill the keys vector
     for(i=0; i<num_keys; ++i) {
@@ -1404,7 +1472,7 @@ class Benchmark {
     ReadOptions options(FLAGS_verify_checksum, true);
     Duration duration(FLAGS_duration, reads_);
 
-    long found = 0;
+    long long found = 0;
 
     if (FLAGS_use_multiget) {   // MultiGet
       const long& kpg = FLAGS_keys_per_multiget;  // keys per multiget group
@@ -1421,7 +1489,7 @@ class Benchmark {
       Iterator* iter = db_->NewIterator(options);
       std::string value;
       while (!duration.Done(1)) {
-        const int k = thread->rand.Next() % FLAGS_num;
+        const long long k = thread->rand.Next() % FLAGS_num;
         unique_ptr<char []> key = GenerateKeyFromInt(k);
         if (FLAGS_use_snapshot) {
           options.snapshot = db_->GetSnapshot();
@@ -1463,7 +1531,42 @@ class Benchmark {
     }
 
     char msg[100];
-    snprintf(msg, sizeof(msg), "(%ld of %ld found)", found, reads_);
+    snprintf(msg, sizeof(msg), "(%lld of %lld found)", found, reads_);
+    thread->stats.AddMessage(msg);
+  }
+
+  void PrefixScanRandom(ThreadState* thread) {
+    if (FLAGS_use_prefix_api) {
+      assert(FLAGS_use_prefix_blooms);
+      assert(FLAGS_bloom_bits >= 1);
+    }
+
+    ReadOptions options(FLAGS_verify_checksum, true);
+    Duration duration(FLAGS_duration, reads_);
+
+    long long found = 0;
+
+    while (!duration.Done(1)) {
+      std::string value;
+      const int k = thread->rand.Next() % FLAGS_num;
+      unique_ptr<char []> key = GenerateKeyFromInt(k);
+      Slice skey(key.get());
+      Slice prefix = prefix_extractor_->Transform(skey);
+      options.prefix = FLAGS_use_prefix_api ? &prefix : nullptr;
+
+      Iterator* iter = db_->NewIterator(options);
+      for (iter->Seek(skey);
+           iter->Valid() && iter->key().starts_with(prefix);
+           iter->Next()) {
+        found++;
+      }
+      delete iter;
+
+      thread->stats.FinishedSingleOp(db_);
+    }
+
+    char msg[100];
+    snprintf(msg, sizeof(msg), "(%lld of %lld found)", found, reads_);
     thread->stats.AddMessage(msg);
   }
 
@@ -1496,7 +1599,7 @@ class Benchmark {
       std::string value;
       Status s;
       while (!duration.Done(1)) {
-        const int k = thread->rand.Next() % FLAGS_num;
+        const long long k = thread->rand.Next() % FLAGS_num;
         unique_ptr<char []> key = GenerateKeyFromInt(k, ".");
         s = db_->Get(options, key.get(), &value);
         assert(!s.ok() && s.IsNotFound());
@@ -1508,12 +1611,12 @@ class Benchmark {
   void ReadHot(ThreadState* thread) {
     Duration duration(FLAGS_duration, reads_);
     ReadOptions options(FLAGS_verify_checksum, true);
-    const long range = (FLAGS_num + 99) / 100;
-    long found = 0;
+    const long long range = (FLAGS_num + 99) / 100;
+    long long found = 0;
 
     if (FLAGS_use_multiget) {
-      const long& kpg = FLAGS_keys_per_multiget;  // keys per multiget group
-      long keys_left = reads_;
+      const long long kpg = FLAGS_keys_per_multiget;  // keys per multiget group
+      long long keys_left = reads_;
 
       // Recalculate number of keys per group, and call MultiGet until done
       long num_keys;
@@ -1525,7 +1628,7 @@ class Benchmark {
     } else {
       std::string value;
       while (!duration.Done(1)) {
-        const int k = thread->rand.Next() % range;
+        const long long k = thread->rand.Next() % range;
         unique_ptr<char []> key = GenerateKeyFromInt(k);
         if (db_->Get(options, key.get(), &value).ok()){
           ++found;
@@ -1535,7 +1638,7 @@ class Benchmark {
     }
 
     char msg[100];
-    snprintf(msg, sizeof(msg), "(%ld of %ld found)", found, reads_);
+    snprintf(msg, sizeof(msg), "(%lld of %lld found)", found, reads_);
     thread->stats.AddMessage(msg);
   }
 
@@ -1543,10 +1646,10 @@ class Benchmark {
     Duration duration(FLAGS_duration, reads_);
     ReadOptions options(FLAGS_verify_checksum, true);
     std::string value;
-    long found = 0;
+    long long found = 0;
     while (!duration.Done(1)) {
       Iterator* iter = db_->NewIterator(options);
-      const int k = thread->rand.Next() % FLAGS_num;
+      const long long k = thread->rand.Next() % FLAGS_num;
       unique_ptr<char []> key = GenerateKeyFromInt(k);
       iter->Seek(key.get());
       if (iter->Valid() && iter->key() == key.get()) found++;
@@ -1554,7 +1657,7 @@ class Benchmark {
       thread->stats.FinishedSingleOp(db_);
     }
     char msg[100];
-    snprintf(msg, sizeof(msg), "(%ld of %ld found)", found, num_);
+    snprintf(msg, sizeof(msg), "(%lld of %lld found)", found, num_);
     thread->stats.AddMessage(msg);
   }
 
@@ -1566,7 +1669,7 @@ class Benchmark {
     while (!duration.Done(entries_per_batch_)) {
       batch.Clear();
       for (int j = 0; j < entries_per_batch_; j++) {
-        const int k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
+        const long long k = seq ? i+j : (thread->rand.Next() % FLAGS_num);
         unique_ptr<char []> key = GenerateKeyFromInt(k);
         batch.Delete(key.get());
         thread->stats.FinishedSingleOp(db_);
@@ -1616,7 +1719,7 @@ class Benchmark {
           }
         }
 
-        const int k = thread->rand.Next() % FLAGS_num;
+        const long long k = thread->rand.Next() % FLAGS_num;
         unique_ptr<char []> key = GenerateKeyFromInt(k);
         Status s = db_->Put(write_options_, key.get(), gen.Generate(value_size_));
         if (!s.ok()) {
@@ -1730,17 +1833,17 @@ class Benchmark {
     ReadOptions options(FLAGS_verify_checksum, true);
     RandomGenerator gen;
     std::string value;
-    long found = 0;
+    long long found = 0;
     int get_weight = 0;
     int put_weight = 0;
     int delete_weight = 0;
-    long gets_done = 0;
-    long puts_done = 0;
-    long deletes_done = 0;
+    long long gets_done = 0;
+    long long puts_done = 0;
+    long long deletes_done = 0;
 
     // the number of iterations is the larger of read_ or write_
-    for (long i = 0; i < readwrites_; i++) {
-      const int k = thread->rand.Next() % (FLAGS_numdistinct);
+    for (long long i = 0; i < readwrites_; i++) {
+      const long long k = thread->rand.Next() % (FLAGS_numdistinct);
       unique_ptr<char []> key = GenerateKeyFromInt(k);
       if (get_weight == 0 && put_weight == 0 && delete_weight == 0) {
         // one batch completed, reinitialize for next batch
@@ -1783,7 +1886,8 @@ class Benchmark {
       thread->stats.FinishedSingleOp(db_);
     }
     char msg[100];
-    snprintf(msg, sizeof(msg), "( get:%ld put:%ld del:%ld total:%ld found:%ld)",
+    snprintf(msg, sizeof(msg),
+             "( get:%lld put:%lld del:%lld total:%lld found:%lld)",
              gets_done, puts_done, deletes_done, readwrites_, found);
     thread->stats.AddMessage(msg);
   }
@@ -1800,16 +1904,16 @@ class Benchmark {
     ReadOptions options(FLAGS_verify_checksum, true);
     RandomGenerator gen;
     std::string value;
-    long found = 0;
+    long long found = 0;
     int get_weight = 0;
     int put_weight = 0;
-    long reads_done = 0;
-    long writes_done = 0;
+    long long reads_done = 0;
+    long long writes_done = 0;
     Duration duration(FLAGS_duration, readwrites_);
 
     // the number of iterations is the larger of read_ or write_
     while (!duration.Done(1)) {
-      const int k = thread->rand.Next() % FLAGS_num;
+      const long long k = thread->rand.Next() % FLAGS_num;
       unique_ptr<char []> key = GenerateKeyFromInt(k);
       if (get_weight == 0 && put_weight == 0) {
         // one batch completed, reinitialize for next batch
@@ -1824,7 +1928,7 @@ class Benchmark {
 
         if (FLAGS_get_approx) {
           char key2[100];
-          snprintf(key2, sizeof(key2), "%016d", k + 1);
+          snprintf(key2, sizeof(key2), "%016lld", k + 1);
           Slice skey2(key2);
           Slice skey(key2);
           Range range(skey, skey2);
@@ -1863,7 +1967,8 @@ class Benchmark {
       thread->stats.FinishedSingleOp(db_);
     }
     char msg[100];
-    snprintf(msg, sizeof(msg), "( reads:%ld writes:%ld total:%ld found:%ld)",
+    snprintf(msg, sizeof(msg),
+             "( reads:%lld writes:%lld total:%lld found:%lld)",
              reads_done, writes_done, readwrites_, found);
     thread->stats.AddMessage(msg);
   }
@@ -1920,7 +2025,7 @@ class Benchmark {
 
       // Now do the puts
       int i;
-      long k;
+      long long k;
       for(i=0; i<num_put_keys; ++i) {
         k = thread->rand.Next() % FLAGS_num;
         unique_ptr<char []> key = GenerateKeyFromInt(k);
@@ -1938,7 +2043,7 @@ class Benchmark {
     }
     char msg[100];
     snprintf(msg, sizeof(msg),
-             "( reads:%ld writes:%ld total:%ld multiget_ops:%ld found:%ld)",
+             "( reads:%ld writes:%ld total:%lld multiget_ops:%ld found:%ld)",
              reads_done, writes_done, readwrites_, multigets_done, found);
     thread->stats.AddMessage(msg);
   }
@@ -1949,12 +2054,12 @@ class Benchmark {
     ReadOptions options(FLAGS_verify_checksum, true);
     RandomGenerator gen;
     std::string value;
-    long found = 0;
+    long long found = 0;
     Duration duration(FLAGS_duration, readwrites_);
 
     // the number of iterations is the larger of read_ or write_
     while (!duration.Done(1)) {
-      const int k = thread->rand.Next() % FLAGS_num;
+      const long long k = thread->rand.Next() % FLAGS_num;
       unique_ptr<char []> key = GenerateKeyFromInt(k);
 
       if (FLAGS_use_snapshot) {
@@ -1963,7 +2068,7 @@ class Benchmark {
 
       if (FLAGS_get_approx) {
         char key2[100];
-        snprintf(key2, sizeof(key2), "%016d", k + 1);
+        snprintf(key2, sizeof(key2), "%016lld", k + 1);
         Slice skey2(key2);
         Slice skey(key2);
         Range range(skey, skey2);
@@ -1987,7 +2092,8 @@ class Benchmark {
       thread->stats.FinishedSingleOp(db_);
     }
     char msg[100];
-    snprintf(msg, sizeof(msg), "( updates:%ld found:%ld)", readwrites_, found);
+    snprintf(msg, sizeof(msg),
+             "( updates:%lld found:%lld)", readwrites_, found);
     thread->stats.AddMessage(msg);
   }
 
@@ -2003,7 +2109,7 @@ class Benchmark {
     // The number of iterations is the larger of read_ or write_
     Duration duration(FLAGS_duration, readwrites_);
     while (!duration.Done(1)) {
-      const int k = thread->rand.Next() % FLAGS_num;
+      const long long k = thread->rand.Next() % FLAGS_num;
       unique_ptr<char []> key = GenerateKeyFromInt(k);
 
       if (FLAGS_use_snapshot) {
@@ -2012,7 +2118,7 @@ class Benchmark {
 
       if (FLAGS_get_approx) {
         char key2[100];
-        snprintf(key2, sizeof(key2), "%016d", k + 1);
+        snprintf(key2, sizeof(key2), "%016lld", k + 1);
         Slice skey2(key2);
         Slice skey(key2);
         Range range(skey, skey2);
@@ -2049,7 +2155,7 @@ class Benchmark {
       thread->stats.FinishedSingleOp(db_);
     }
     char msg[100];
-    snprintf(msg, sizeof(msg), "( updates:%ld found:%ld)", readwrites_, found);
+    snprintf(msg, sizeof(msg), "( updates:%lld found:%ld)", readwrites_, found);
     thread->stats.AddMessage(msg);
   }
 
@@ -2066,7 +2172,7 @@ class Benchmark {
     // The number of iterations is the larger of read_ or write_
     Duration duration(FLAGS_duration, readwrites_);
     while (!duration.Done(1)) {
-      const int k = thread->rand.Next() % FLAGS_num;
+      const long long k = thread->rand.Next() % FLAGS_num;
       unique_ptr<char []> key = GenerateKeyFromInt(k);
 
       Status s = db_->Merge(write_options_, key.get(),
@@ -2081,7 +2187,7 @@ class Benchmark {
 
     // Print some statistics
     char msg[100];
-    snprintf(msg, sizeof(msg), "( updates:%ld)", readwrites_);
+    snprintf(msg, sizeof(msg), "( updates:%lld)", readwrites_);
     thread->stats.AddMessage(msg);
   }
 
@@ -2148,6 +2254,7 @@ int main(int argc, char** argv) {
     double d;
     int n;
     long l;
+    long long ll;
     char junk;
     char buf[2048];
     char str[512];
@@ -2162,14 +2269,21 @@ int main(int argc, char** argv) {
     } else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
                (n == 0 || n == 1)) {
       FLAGS_use_existing_db = n;
-    } else if (sscanf(argv[i], "--num=%ld%c", &l, &junk) == 1) {
-      FLAGS_num = l;
-    } else if (sscanf(argv[i], "--numdistinct=%ld%c", &l, &junk) == 1) {
-      FLAGS_numdistinct = l;
-    } else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
-      FLAGS_reads = n;
+    } else if (sscanf(argv[i], "--num=%lld%c", &ll, &junk) == 1) {
+      FLAGS_num = ll;
+    } else if (sscanf(argv[i], "--numdistinct=%lld%c", &ll, &junk) == 1) {
+      FLAGS_numdistinct = ll;
+    } else if (sscanf(argv[i], "--reads=%lld%c", &ll, &junk) == 1) {
+      FLAGS_reads = ll;
     } else if (sscanf(argv[i], "--read_range=%d%c", &n, &junk) == 1) {
       FLAGS_read_range = n;
+
+    } else if (sscanf(argv[i], "--use_prefix_blooms=%d%c", &n, &junk) == 1 &&
+               (n == 0 || n == 1)) {
+      FLAGS_use_prefix_blooms = n;
+    } else if (sscanf(argv[i], "--use_prefix_api=%d%c", &n, &junk) == 1 &&
+               (n == 0 || n == 1)) {
+      FLAGS_use_prefix_api = n;
     } else if (sscanf(argv[i], "--duration=%d%c", &n, &junk) == 1) {
       FLAGS_duration = n;
     } else if (sscanf(argv[i], "--seed=%ld%c", &l, &junk) == 1) {
@@ -2244,8 +2358,8 @@ int main(int argc, char** argv) {
         dbstats = leveldb::CreateDBStatistics();
         FLAGS_statistics = true;
       }
-    } else if (sscanf(argv[i], "--writes=%d%c", &n, &junk) == 1) {
-      FLAGS_writes = n;
+    } else if (sscanf(argv[i], "--writes=%lld%c", &ll, &junk) == 1) {
+      FLAGS_writes = ll;
     } else if (sscanf(argv[i], "--writes_per_second=%d%c", &n, &junk) == 1) {
       FLAGS_writes_per_second = n;
     } else if (sscanf(argv[i], "--sync=%d%c", &n, &junk) == 1 &&
@@ -2319,6 +2433,19 @@ int main(int argc, char** argv) {
       else {
         fprintf(stdout, "Cannot parse %s\n", argv[i]);
       }
+    } else if (strncmp(argv[i], "--memtablerep=", 14) == 0) {
+      const char* ctype = argv[i] + 14;
+      if (!strcasecmp(ctype, "skip_list"))
+        FLAGS_rep_factory = kSkipList;
+      else if (!strcasecmp(ctype, "prefix_hash"))
+        FLAGS_rep_factory = kPrefixHash;
+      else if (!strcasecmp(ctype, "unsorted"))
+        FLAGS_rep_factory = kUnsorted;
+      else if (!strcasecmp(ctype, "vector"))
+        FLAGS_rep_factory = kVectorRep;
+      else {
+        fprintf(stdout, "Cannot parse %s\n", argv[i]);
+      }
     } else if (sscanf(argv[i], "--min_level_to_compress=%d%c", &n, &junk) == 1
         && n >= 0) {
       FLAGS_min_level_to_compress = n;
@@ -2328,12 +2455,14 @@ int main(int argc, char** argv) {
     } else if (sscanf(argv[i], "--delete_obsolete_files_period_micros=%ld%c",
                       &l, &junk) == 1) {
       FLAGS_delete_obsolete_files_period_micros = l;
-    } else if (sscanf(argv[i], "--stats_interval=%d%c", &n, &junk) == 1 &&
-               n >= 0 && n < 2000000000) {
-      FLAGS_stats_interval = n;
+    } else if (sscanf(argv[i], "--stats_interval=%lld%c", &ll, &junk) == 1) {
+      FLAGS_stats_interval = ll;
     } else if (sscanf(argv[i], "--stats_per_interval=%d%c", &n, &junk) == 1
         && (n == 0 || n == 1)) {
       FLAGS_stats_per_interval = n;
+    } else if (sscanf(argv[i], "--prefix_size=%d%c", &n, &junk) == 1 &&
+               n >= 0 && n < 2000000000) {
+      FLAGS_prefix_size = n;
     } else if (sscanf(argv[i], "--soft_rate_limit=%lf%c", &d, &junk) == 1 &&
                d > 0.0) {
       FLAGS_soft_rate_limit = d;

db/db_filesnapshot.cc

@@ -8,8 +8,8 @@
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "db/version_set.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
 
@@ -48,18 +48,17 @@ Status DBImpl::GetLiveFiles(std::vector<std::string>& ret,
   std::set<uint64_t> live;
   versions_->AddLiveFilesCurrentVersion(&live);
 
-  ret.resize(live.size() + 2); //*.sst + CURRENT + MANIFEST
+  ret.clear();
+  ret.reserve(live.size() + 2); //*.sst + CURRENT + MANIFEST
 
   // create names of the live files. The names are not absolute
   // paths, instead they are relative to dbname_;
-  std::set<uint64_t>::iterator it = live.begin();
-  for (unsigned int i = 0; i < live.size(); i++, it++) {
-    ret[i] = TableFileName("", *it);
+  for (auto live_file : live) {
+    ret.push_back(TableFileName("", live_file));
   }
 
-  ret[live.size()] = CurrentFileName("");
-  ret[live.size()+1] = DescriptorFileName("",
-                                          versions_->ManifestFileNumber());
+  ret.push_back(CurrentFileName(""));
+  ret.push_back(DescriptorFileName("", versions_->ManifestFileNumber()));
 
   // find length of manifest file while holding the mutex lock
   *manifest_file_size = versions_->ManifestFileSize();
@@ -71,7 +70,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
   // First get sorted files in archive dir, then append sorted files from main
   // dir to maintain sorted order
 
-  //  list wal files in archive dir.
+  // list wal files in archive dir.
   Status s;
   std::string archivedir = ArchivalDirectory(dbname_);
   if (env_->FileExists(archivedir)) {
@@ -81,11 +80,7 @@ Status DBImpl::GetSortedWalFiles(VectorLogPtr& files) {
     }
   }
   // list wal files in main db dir.
-  s = AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
-  if (!s.ok()) {
-    return s;
-  }
-  return s;
+  return AppendSortedWalsOfType(dbname_, files, kAliveLogFile);
 }
 
 Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
@@ -93,14 +88,17 @@ Status DBImpl::DeleteWalFiles(const VectorLogPtr& files) {
   std::string archivedir = ArchivalDirectory(dbname_);
   std::string files_not_deleted;
   for (const auto& wal : files) {
-    /* Try deleting in archive dir. If fails, try deleting in main db dir.
-     * This is efficient because all except for very few wal files will be in
-     * archive. Checking for WalType is not much helpful because alive wal could
-       be archived now.
+    /* Try deleting in the dir that pathname points to for the logfile.
+       This may fail if we try to delete a log file which was live when captured
+       but is archived now. Try deleting it from archive also
      */
-    if (!env_->DeleteFile(archivedir + "/" + wal->Filename()).ok() &&
-        !env_->DeleteFile(dbname_ + "/" + wal->Filename()).ok()) {
-      files_not_deleted.append(wal->Filename());
+    Status st = env_->DeleteFile(dbname_ + "/" + wal->PathName());
+    if (!st.ok()) {
+      if (wal->Type() == kAliveLogFile &&
+          env_->DeleteFile(LogFileName(archivedir, wal->LogNumber())).ok()) {
+        continue;
+      }
+      files_not_deleted.append(wal->PathName());
     }
   }
   if (!files_not_deleted.empty()) {

db/db_impl.cc

@@ -28,13 +28,13 @@
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
 #include "db/transaction_log_impl.h"
-#include "leveldb/compaction_filter.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/merge_operator.h"
-#include "leveldb/statistics.h"
-#include "leveldb/status.h"
-#include "leveldb/table_builder.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table_builder.h"
 #include "port/port.h"
 #include "table/block.h"
 #include "table/merger.h"
@@ -163,6 +163,20 @@ Options SanitizeOptions(const std::string& dbname,
       result.compaction_filter_factory->CreateCompactionFilter().get()) {
     Log(result.info_log, "Both filter and factory specified. Using filter");
   }
+  if (result.prefix_extractor) {
+    // If a prefix extractor has been supplied and a PrefixHashRepFactory is
+    // being used, make sure that the latter uses the former as its transform
+    // function.
+    auto factory = dynamic_cast<PrefixHashRepFactory*>(
+      result.memtable_factory.get());
+    if (factory != nullptr &&
+        factory->GetTransform() != result.prefix_extractor) {
+      Log(result.info_log, "A prefix hash representation factory was supplied "
+          "whose prefix extractor does not match options.prefix_extractor. "
+          "Falling back to skip list representation factory");
+      result.memtable_factory = std::make_shared<SkipListFactory>();
+    }
+  }
   return result;
 }
 
@@ -932,7 +946,7 @@ Status DBImpl::CompactMemTable(bool* madeProgress) {
 }
 
 void DBImpl::CompactRange(const Slice* begin, const Slice* end,
-                          bool reduce_level) {
+                          bool reduce_level, int target_level) {
   int max_level_with_files = 1;
   {
     MutexLock l(&mutex_);
@@ -949,7 +963,7 @@ void DBImpl::CompactRange(const Slice* begin, const Slice* end,
   }
 
   if (reduce_level) {
-    ReFitLevel(max_level_with_files);
+    ReFitLevel(max_level_with_files, target_level);
   }
 }
 
@@ -969,7 +983,7 @@ int DBImpl::FindMinimumEmptyLevelFitting(int level) {
   return minimum_level;
 }
 
-void DBImpl::ReFitLevel(int level) {
+void DBImpl::ReFitLevel(int level, int target_level) {
   assert(level < NumberLevels());
 
   MutexLock l(&mutex_);
@@ -991,7 +1005,10 @@ void DBImpl::ReFitLevel(int level) {
   }
 
   // move to a smaller level
-  int to_level = FindMinimumEmptyLevelFitting(level);
+  int to_level = target_level;
+  if (target_level < 0) {
+    to_level = FindMinimumEmptyLevelFitting(level);
+  }
 
   assert(to_level <= level);
 
@@ -1196,6 +1213,12 @@ Status DBImpl::AppendSortedWalsOfType(const std::string& path,
     return status;
   }
   log_files.reserve(log_files.size() + all_files.size());
+  VectorLogPtr::iterator pos_start;
+  if (!log_files.empty()) {
+    pos_start = log_files.end() - 1;
+  } else {
+    pos_start = log_files.begin();
+  }
   for (const auto& f : all_files) {
     uint64_t number;
     FileType type;
@@ -1221,7 +1244,7 @@ Status DBImpl::AppendSortedWalsOfType(const std::string& path,
     }
   }
   CompareLogByPointer compare_log_files;
-  std::sort(log_files.begin(), log_files.end(), compare_log_files);
+  std::sort(pos_start, log_files.end(), compare_log_files);
   return status;
 }
 
@@ -2097,7 +2120,8 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
   VersionSet::LevelSummaryStorage tmp;
   Log(options_.info_log,
       "compacted to: %s, %.1f MB/sec, level %d, files in(%d, %d) out(%d) "
-      "MB in(%.1f, %.1f) out(%.1f), amplify(%.1f) %s\n",
+      "MB in(%.1f, %.1f) out(%.1f), read-write-amplify(%.1f) "
+      "write-amplify(%.1f) %s\n",
       versions_->LevelSummary(&tmp),
       (stats.bytes_readn + stats.bytes_readnp1 + stats.bytes_written) /
           (double) stats.micros,
@@ -2106,8 +2130,9 @@ Status DBImpl::DoCompactionWork(CompactionState* compact) {
       stats.bytes_readn / 1048576.0,
       stats.bytes_readnp1 / 1048576.0,
       stats.bytes_written / 1048576.0,
-      (stats.bytes_written + stats.bytes_readnp1) /
+      (stats.bytes_written + stats.bytes_readnp1 + stats.bytes_readn) /
           (double) stats.bytes_readn,
+      stats.bytes_written / (double) stats.bytes_readn,
       status.ToString().c_str());
 
   return status;
@@ -2141,7 +2166,8 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
   // Collect together all needed child iterators for mem
   std::vector<Iterator*> list;
   mem_->Ref();
-  list.push_back(mem_->NewIterator());
+  list.push_back(mem_->NewIterator(options.prefix));
+
   cleanup->mem.push_back(mem_);
 
   // Collect together all needed child iterators for imm_
@@ -2150,7 +2176,7 @@ Iterator* DBImpl::NewInternalIterator(const ReadOptions& options,
   for (unsigned int i = 0; i < immutables.size(); i++) {
     MemTable* m = immutables[i];
     m->Ref();
-    list.push_back(m->NewIterator());
+    list.push_back(m->NewIterator(options.prefix));
     cleanup->mem.push_back(m);
   }
 
@@ -2187,13 +2213,12 @@ Status DBImpl::Get(const ReadOptions& options,
 Status DBImpl::GetImpl(const ReadOptions& options,
                        const Slice& key,
                        std::string* value,
-                       const bool no_io,
                        bool* value_found) {
   Status s;
 
   StopWatch sw(env_, options_.statistics, DB_GET);
   SequenceNumber snapshot;
-  MutexLock l(&mutex_);
+  mutex_.Lock();
   if (options.snapshot != nullptr) {
     snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
   } else {
@@ -2238,6 +2263,9 @@ Status DBImpl::GetImpl(const ReadOptions& options,
   mem->Unref();
   imm.UnrefAll();
   current->Unref();
+  mutex_.Unlock();
+
+  // Note, tickers are atomic now - no lock protection needed any more.
   RecordTick(options_.statistics, NUMBER_KEYS_READ);
   RecordTick(options_.statistics, BYTES_READ, value->size());
   return s;
@@ -2249,7 +2277,7 @@ std::vector<Status> DBImpl::MultiGet(const ReadOptions& options,
 
   StopWatch sw(env_, options_.statistics, DB_MULTIGET);
   SequenceNumber snapshot;
-  MutexLock l(&mutex_);
+  mutex_.Lock();
   if (options.snapshot != nullptr) {
     snapshot = reinterpret_cast<const SnapshotImpl*>(options.snapshot)->number_;
   } else {
@@ -2313,6 +2341,8 @@ std::vector<Status> DBImpl::MultiGet(const ReadOptions& options,
   mem->Unref();
   imm.UnrefAll();
   current->Unref();
+  mutex_.Unlock();
+
   RecordTick(options_.statistics, NUMBER_MULTIGET_CALLS);
   RecordTick(options_.statistics, NUMBER_MULTIGET_KEYS_READ, numKeys);
   RecordTick(options_.statistics, NUMBER_MULTIGET_BYTES_READ, bytesRead);
@@ -2327,7 +2357,9 @@ bool DBImpl::KeyMayExist(const ReadOptions& options,
   if (value_found != nullptr) {
     *value_found = true; // falsify later if key-may-exist but can't fetch value
   }
-  return GetImpl(options, key, value, true, value_found).ok();
+  ReadOptions roptions = options;
+  roptions.read_tier = kBlockCacheTier; // read from block cache only
+  return GetImpl(roptions, key, value, value_found).ok();
 }
 
 Iterator* DBImpl::NewIterator(const ReadOptions& options) {
@@ -2397,22 +2429,28 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
   uint64_t last_sequence = versions_->LastSequence();
   Writer* last_writer = &w;
   if (status.ok() && my_batch != nullptr) {  // nullptr batch is for compactions
+    // TODO: BuildBatchGroup physically concatenate/copy all write batches into
+    // a new one. Mem copy is done with the lock held. Ideally, we only need
+    // the lock to obtain the last_writer and the references to all batches.
+    // Creation (copy) of the merged batch could have been done outside of the
+    // lock protected region.
     WriteBatch* updates = BuildBatchGroup(&last_writer);
-    const SequenceNumber current_sequence = last_sequence + 1;
-    WriteBatchInternal::SetSequence(updates, current_sequence);
-    int my_batch_count = WriteBatchInternal::Count(updates);
-    last_sequence += my_batch_count;
-    // Record statistics
-    RecordTick(options_.statistics, NUMBER_KEYS_WRITTEN, my_batch_count);
-    RecordTick(options_.statistics,
-               BYTES_WRITTEN,
-               WriteBatchInternal::ByteSize(updates));
+
     // Add to log and apply to memtable.  We can release the lock
     // during this phase since &w is currently responsible for logging
     // and protects against concurrent loggers and concurrent writes
     // into mem_.
     {
       mutex_.Unlock();
+      const SequenceNumber current_sequence = last_sequence + 1;
+      WriteBatchInternal::SetSequence(updates, current_sequence);
+      int my_batch_count = WriteBatchInternal::Count(updates);
+      last_sequence += my_batch_count;
+      // Record statistics
+      RecordTick(options_.statistics, NUMBER_KEYS_WRITTEN, my_batch_count);
+      RecordTick(options_.statistics,
+                 BYTES_WRITTEN,
+                 WriteBatchInternal::ByteSize(updates));
       if (options.disableWAL) {
         flush_on_destroy_ = true;
       }
@@ -2439,7 +2477,7 @@ Status DBImpl::Write(const WriteOptions& options, WriteBatch* my_batch) {
           // have succeeded in memtable but Status reports error for all writes.
           throw std::runtime_error("In memory WriteBatch corruption!");
         }
-        RecordTick(options_.statistics, SEQUENCE_NUMBER, my_batch_count);
+        SetTickerCount(options_.statistics, SEQUENCE_NUMBER, last_sequence);
         versions_->SetLastSequence(last_sequence);
         last_flushed_sequence_ = current_sequence;
       }
@@ -2707,6 +2745,9 @@ Status DBImpl::MakeRoomForWrite(bool force) {
       log_.reset(new log::Writer(std::move(lfile)));
       mem_->SetLogNumber(logfile_number_);
       imm_.Add(mem_);
+      if (force) {
+        imm_.FlushRequested();
+      }
       mem_ = new MemTable(internal_comparator_, mem_rep_factory_,
         NumberLevels(), options_);
       mem_->Ref();
@@ -2773,7 +2814,7 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
     // Pardon the long line but I think it is easier to read this way.
     snprintf(buf, sizeof(buf),
              "                               Compactions\n"
-             "Level  Files Size(MB) Score Time(sec)  Read(MB) Write(MB)    Rn(MB)  Rnp1(MB)  Wnew(MB) Amplify Read(MB/s) Write(MB/s)      Rn     Rnp1     Wnp1     NewW    Count  Ln-stall Stall-cnt\n"
+             "Level  Files Size(MB) Score Time(sec)  Read(MB) Write(MB)    Rn(MB)  Rnp1(MB)  Wnew(MB) RW-Amplify Read(MB/s) Write(MB/s)      Rn     Rnp1     Wnp1     NewW    Count  Ln-stall Stall-cnt\n"
              "--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------\n"
              );
     value->append(buf);
@@ -2786,7 +2827,9 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
                             stats_[level].bytes_readnp1;
         double amplify = (stats_[level].bytes_readn == 0)
             ? 0.0
-            : (stats_[level].bytes_written + stats_[level].bytes_readnp1) /
+            : (stats_[level].bytes_written +
+               stats_[level].bytes_readnp1 +
+               stats_[level].bytes_readn) /
                 (double) stats_[level].bytes_readn;
 
         total_bytes_read += bytes_read;
@@ -2794,12 +2837,12 @@ bool DBImpl::GetProperty(const Slice& property, std::string* value) {
 
         snprintf(
             buf, sizeof(buf),
-            "%3d %8d %8.0f %5.1f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %7.1f %9.1f %11.1f %8d %8d %8d %8d %8d %9.1f %9lu\n",
+            "%3d %8d %8.0f %5.1f %9.0f %9.0f %9.0f %9.0f %9.0f %9.0f %10.1f %9.1f %11.1f %8d %8d %8d %8d %8d %9.1f %9lu\n",
             level,
             files,
             versions_->NumLevelBytes(level) / 1048576.0,
             versions_->NumLevelBytes(level) /
-            versions_->MaxBytesForLevel(level),
+                versions_->MaxBytesForLevel(level),
             stats_[level].micros / 1e6,
             bytes_read / 1048576.0,
             stats_[level].bytes_written / 1048576.0,
@@ -2951,6 +2994,71 @@ inline void DBImpl::DelayLoggingAndReset() {
   }
 }
 
+Status DBImpl::DeleteFile(std::string name) {
+  uint64_t number;
+  FileType type;
+  if (!ParseFileName(name, &number, &type) ||
+      (type != kTableFile)) {
+    Log(options_.info_log, "DeleteFile #%lld FAILED. Invalid file name\n",
+        static_cast<unsigned long long>(number));
+    return Status::InvalidArgument("Invalid file name");
+  }
+
+  int level;
+  FileMetaData metadata;
+  int maxlevel = NumberLevels();
+  VersionEdit edit(maxlevel);
+  DeletionState deletion_state;
+  Status status;
+  {
+    MutexLock l(&mutex_);
+    status = versions_->GetMetadataForFile(number, &level, &metadata);
+    if (!status.ok()) {
+      Log(options_.info_log, "DeleteFile #%lld FAILED. File not found\n",
+          static_cast<unsigned long long>(number));
+      return Status::InvalidArgument("File not found");
+    }
+    assert((level > 0) && (level < maxlevel));
+
+    // If the file is being compacted no need to delete.
+    if (metadata.being_compacted) {
+      Log(options_.info_log,
+          "DeleteFile #%lld Skipped. File about to be compacted\n",
+          static_cast<unsigned long long>(number));
+      return Status::OK();
+    }
+
+    // Only the files in the last level can be deleted externally.
+    // This is to make sure that any deletion tombstones are not
+    // lost. Check that the level passed is the last level.
+    for (int i = level + 1; i < maxlevel; i++) {
+      if (versions_->NumLevelFiles(i) != 0) {
+        Log(options_.info_log,
+            "DeleteFile #%lld FAILED. File not in last level\n",
+            static_cast<unsigned long long>(number));
+        return Status::InvalidArgument("File not in last level");
+      }
+    }
+    edit.DeleteFile(level, number);
+    status = versions_->LogAndApply(&edit, &mutex_);
+    if (status.ok()) {
+      FindObsoleteFiles(deletion_state);
+    }
+  } // lock released here
+
+  if (status.ok()) {
+    // remove files outside the db-lock
+    PurgeObsoleteFiles(deletion_state);
+    EvictObsoleteFiles(deletion_state);
+  }
+  return status;
+}
+
+void DBImpl::GetLiveFilesMetaData(std::vector<LiveFileMetaData> *metadata) {
+  MutexLock l(&mutex_);
+  return versions_->GetLiveFilesMetaData(metadata);
+}
+
 // Default implementations of convenience methods that subclasses of DB
 // can call if they wish
 Status DB::Put(const WriteOptions& opt, const Slice& key, const Slice& value) {
@@ -3011,6 +3119,20 @@ Status DB::Open(const Options& options, const std::string& dbname, DB** dbptr) {
     }
   }
   impl->mutex_.Unlock();
+
+  if (options.compaction_style == kCompactionStyleUniversal) {
+    std::string property;
+    int num_files;
+    for (int i = 1; i < impl->NumberLevels(); i++) {
+      num_files = impl->versions_->NumLevelFiles(i);
+      if (num_files > 0) {
+        s = Status::InvalidArgument("Not all files are at level 0. Cannot "
+          "open with universal compaction style.");
+        break;
+      }
+    }
+  }
+
   if (s.ok()) {
     *dbptr = impl;
   } else {

db/db_impl.h

@@ -12,10 +12,10 @@
 #include "db/dbformat.h"
 #include "db/log_writer.h"
 #include "db/snapshot.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/memtablerep.h"
-#include "leveldb/transaction_log.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/transaction_log.h"
 #include "port/port.h"
 #include "util/stats_logger.h"
 #include "memtablelist.h"
@@ -64,7 +64,7 @@ class DBImpl : public DB {
   virtual bool GetProperty(const Slice& property, std::string* value);
   virtual void GetApproximateSizes(const Range* range, int n, uint64_t* sizes);
   virtual void CompactRange(const Slice* begin, const Slice* end,
-                            bool reduce_level = false);
+                            bool reduce_level = false, int target_level = -1);
   virtual int NumberLevels();
   virtual int MaxMemCompactionLevel();
   virtual int Level0StopWriteTrigger();
@@ -78,6 +78,10 @@ class DBImpl : public DB {
   virtual SequenceNumber GetLatestSequenceNumber();
   virtual Status GetUpdatesSince(SequenceNumber seq_number,
                                  unique_ptr<TransactionLogIterator>* iter);
+  virtual Status DeleteFile(std::string name);
+
+  virtual void GetLiveFilesMetaData(
+    std::vector<LiveFileMetaData> *metadata);
 
   // Extra methods (for testing) that are not in the public DB interface
 
@@ -237,9 +241,10 @@ class DBImpl : public DB {
   // input level. Return the input level, if such level could not be found.
   int FindMinimumEmptyLevelFitting(int level);
 
-  // Move the files in the input level to the minimum level that could hold
-  // the data set.
-  void ReFitLevel(int level);
+  // Move the files in the input level to the target level.
+  // If target_level < 0, automatically calculate the minimum level that could
+  // hold the data set.
+  void ReFitLevel(int level, int target_level = -1);
 
   // Constant after construction
   const InternalFilterPolicy internal_filter_policy_;
@@ -420,7 +425,6 @@ class DBImpl : public DB {
   Status GetImpl(const ReadOptions& options,
                  const Slice& key,
                  std::string* value,
-                 const bool no_io = false,
                  bool* value_found = nullptr);
 };
 

db/db_impl_readonly.cc

@@ -21,10 +21,10 @@
 #include "db/table_cache.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/status.h"
-#include "leveldb/table_builder.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table_builder.h"
 #include "port/port.h"
 #include "table/block.h"
 #include "table/merger.h"

db/db_impl_readonly.h

@@ -12,8 +12,8 @@
 #include "db/dbformat.h"
 #include "db/log_writer.h"
 #include "db/snapshot.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
 #include "port/port.h"
 #include "util/stats_logger.h"
 
@@ -51,7 +51,7 @@ public:
    return Status::NotSupported("Not supported operation in read only mode.");
  }
  virtual void CompactRange(const Slice* begin, const Slice* end,
-                           bool reduce_level = false) {
+                           bool reduce_level = false, int target_level = -1) {
  }
  virtual Status DisableFileDeletions() {
    return Status::NotSupported("Not supported operation in read only mode.");

db/db_iter.cc

@@ -8,10 +8,10 @@
 
 #include "db/filename.h"
 #include "db/dbformat.h"
-#include "leveldb/env.h"
-#include "leveldb/options.h"
-#include "leveldb/iterator.h"
-#include "leveldb/merge_operator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/merge_operator.h"
 #include "port/port.h"
 #include "util/logging.h"
 #include "util/mutexlock.h"
@@ -65,6 +65,7 @@ class DBIter: public Iterator {
         current_entry_is_merged_(false),
         statistics_(options.statistics) {
     RecordTick(statistics_, NO_ITERATORS, 1);
+    max_skip_ = options.max_sequential_skip_in_iterations;
   }
   virtual ~DBIter() {
     RecordTick(statistics_, NO_ITERATORS, -1);
@@ -129,6 +130,7 @@ class DBIter: public Iterator {
   bool valid_;
   bool current_entry_is_merged_;
   std::shared_ptr<Statistics> statistics_;
+  uint64_t max_skip_;
 
   // No copying allowed
   DBIter(const DBIter&);
@@ -188,12 +190,13 @@ void DBIter::FindNextUserEntry(bool skipping) {
   assert(iter_->Valid());
   assert(direction_ == kForward);
   current_entry_is_merged_ = false;
+  uint64_t num_skipped = 0;
   do {
     ParsedInternalKey ikey;
     if (ParseKey(&ikey) && ikey.sequence <= sequence_) {
       if (skipping &&
           user_comparator_->Compare(ikey.user_key, saved_key_) <= 0) {
-        // skip this entry
+        num_skipped++; // skip this entry
       } else {
         skipping = false;
         switch (ikey.type) {
@@ -202,6 +205,7 @@ void DBIter::FindNextUserEntry(bool skipping) {
             // they are hidden by this deletion.
             SaveKey(ikey.user_key, &saved_key_);
             skipping = true;
+            num_skipped = 0;
             break;
           case kTypeValue:
             valid_ = true;
@@ -220,7 +224,20 @@ void DBIter::FindNextUserEntry(bool skipping) {
         }
       }
     }
-    iter_->Next();
+    // If we have sequentially iterated via numerous keys and still not
+    // found the next user-key, then it is better to seek so that we can
+    // avoid too many key comparisons. We seek to the last occurence of
+    // our current key by looking for sequence number 0.
+    if (skipping && num_skipped > max_skip_) {
+      num_skipped = 0;
+      std::string last_key;
+      AppendInternalKey(&last_key,
+        ParsedInternalKey(Slice(saved_key_), 0, kValueTypeForSeek));
+      iter_->Seek(last_key);
+      RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+    } else {
+      iter_->Next();
+    }
   } while (iter_->Valid());
   valid_ = false;
 }
@@ -342,6 +359,7 @@ void DBIter::Prev() {
 
 void DBIter::FindPrevUserEntry() {
   assert(direction_ == kReverse);
+  uint64_t num_skipped = 0;
 
   ValueType value_type = kTypeDeletion;
   if (iter_->Valid()) {
@@ -367,7 +385,22 @@ void DBIter::FindPrevUserEntry() {
           saved_value_.assign(raw_value.data(), raw_value.size());
         }
       }
-      iter_->Prev();
+      num_skipped++;
+      // If we have sequentially iterated via numerous keys and still not
+      // found the prev user-key, then it is better to seek so that we can
+      // avoid too many key comparisons. We seek to the first occurence of
+      // our current key by looking for max sequence number.
+      if (num_skipped > max_skip_) {
+        num_skipped = 0;
+        std::string last_key;
+        AppendInternalKey(&last_key,
+          ParsedInternalKey(Slice(saved_key_), kMaxSequenceNumber,
+                            kValueTypeForSeek));
+        iter_->Seek(last_key);
+        RecordTick(statistics_, NUMBER_OF_RESEEKS_IN_ITERATION);
+      } else {
+        iter_->Prev();
+      }
     } while (iter_->Valid());
   }
 

db/db_iter.h

@@ -6,7 +6,7 @@
 #define STORAGE_LEVELDB_DB_DB_ITER_H_
 
 #include <stdint.h>
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 #include "db/dbformat.h"
 
 namespace leveldb {

db/db_statistics.h

@@ -10,7 +10,7 @@
 #include <vector>
 #include <memory>
 
-#include "leveldb/statistics.h"
+#include "rocksdb/statistics.h"
 #include "util/histogram.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
@@ -62,5 +62,3 @@ std::shared_ptr<Statistics> CreateDBStatistics() {
 } // namespace leveldb
 
 #endif // LEVELDB_STORAGE_DB_DB_STATISTICS_H_
-
-

db/db_stats_logger.cc

@@ -7,8 +7,8 @@
 #include <stdint.h>
 #include <stdio.h>
 #include "db/version_set.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
 

db/db_test.cc

@@ -5,15 +5,16 @@
 #include <algorithm>
 #include <set>
 
-#include "leveldb/db.h"
-#include "leveldb/filter_policy.h"
+#include "rocksdb/db.h"
+#include "rocksdb/filter_policy.h"
 #include "db/db_impl.h"
 #include "db/filename.h"
 #include "db/version_set.h"
 #include "db/write_batch_internal.h"
-#include "leveldb/cache.h"
-#include "leveldb/compaction_filter.h"
-#include "leveldb/env.h"
+#include "db/db_statistics.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/compaction_filter.h"
+#include "rocksdb/env.h"
 #include "table/table.h"
 #include "util/hash.h"
 #include "util/logging.h"
@@ -68,6 +69,7 @@ class AtomicCounter {
     count_ = 0;
   }
 };
+
 }
 
 // Special Env used to delay background operations
@@ -207,9 +209,12 @@ class DBTest {
  private:
   const FilterPolicy* filter_policy_;
 
+ protected:
   // Sequence of option configurations to try
   enum OptionConfig {
     kDefault,
+    kVectorRep,
+    kUnsortedRep,
     kMergePut,
     kFilter,
     kUncompressed,
@@ -219,6 +224,7 @@ class DBTest {
     kCompactOnFlush,
     kPerfOptions,
     kDeletesFilterFirst,
+    kPrefixHashRep,
     kUniversalCompaction,
     kEnd
   };
@@ -293,6 +299,10 @@ class DBTest {
   Options CurrentOptions() {
     Options options;
     switch (option_config_) {
+      case kPrefixHashRep:
+        options.memtable_factory.reset(new
+          PrefixHashRepFactory(NewFixedPrefixTransform(1)));
+        break;
       case kMergePut:
         options.merge_operator = MergeOperators::CreatePutOperator();
         break;
@@ -321,6 +331,12 @@ class DBTest {
       case kDeletesFilterFirst:
         options.filter_deletes = true;
         break;
+      case kUnsortedRep:
+        options.memtable_factory.reset(new UnsortedRepFactory);
+        break;
+      case kVectorRep:
+        options.memtable_factory.reset(new VectorRepFactory);
+        break;
       case kUniversalCompaction:
         options.compaction_style = kCompactionStyleUniversal;
         break;
@@ -815,6 +831,7 @@ TEST(DBTest, KeyMayExist) {
     std::string value;
     Options options = CurrentOptions();
     options.filter_policy = NewBloomFilterPolicy(20);
+    options.statistics = leveldb::CreateDBStatistics();
     Reopen(&options);
 
     ASSERT_TRUE(!db_->KeyMayExist(ropts, "a", &value));
@@ -827,25 +844,114 @@ TEST(DBTest, KeyMayExist) {
 
     dbfull()->Flush(FlushOptions());
     value.clear();
-    value_found = false;
+
+    long numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
+    long cache_miss =
+      options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
     ASSERT_TRUE(db_->KeyMayExist(ropts, "a", &value, &value_found));
-    ASSERT_TRUE(value_found);
-    ASSERT_EQ("b", value);
+    ASSERT_TRUE(!value_found);
+    // assert that no new files were opened and no new blocks were
+    // read into block cache.
+    ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
+    ASSERT_EQ(cache_miss,
+              options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
 
     ASSERT_OK(db_->Delete(WriteOptions(), "a"));
+
+    numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
+    cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
     ASSERT_TRUE(!db_->KeyMayExist(ropts, "a", &value));
+    ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
+    ASSERT_EQ(cache_miss,
+              options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
 
     dbfull()->Flush(FlushOptions());
     dbfull()->CompactRange(nullptr, nullptr);
+
+    numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
+    cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
     ASSERT_TRUE(!db_->KeyMayExist(ropts, "a", &value));
+    ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
+    ASSERT_EQ(cache_miss,
+              options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
 
     ASSERT_OK(db_->Delete(WriteOptions(), "c"));
+
+    numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
+    cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
     ASSERT_TRUE(!db_->KeyMayExist(ropts, "c", &value));
+    ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
+    ASSERT_EQ(cache_miss,
+              options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
 
     delete options.filter_policy;
   } while (ChangeOptions());
 }
 
+TEST(DBTest, NonBlockingIteration) {
+  do {
+    ReadOptions non_blocking_opts, regular_opts;
+    Options options = CurrentOptions();
+    options.statistics = leveldb::CreateDBStatistics();
+    non_blocking_opts.read_tier = kBlockCacheTier;
+    Reopen(&options);
+
+    // write one kv to the database.
+    ASSERT_OK(db_->Put(WriteOptions(), "a", "b"));
+
+    // scan using non-blocking iterator. We should find it because
+    // it is in memtable.
+    Iterator* iter = db_->NewIterator(non_blocking_opts);
+    int count = 0;
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+      ASSERT_TRUE(iter->status().ok());
+      count++;
+    }
+    ASSERT_EQ(count, 1);
+    delete iter;
+
+    // flush memtable to storage. Now, the key should not be in the
+    // memtable neither in the block cache.
+    dbfull()->Flush(FlushOptions());
+
+    // verify that a non-blocking iterator does not find any
+    // kvs. Neither does it do any IOs to storage.
+    long numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
+    long cache_miss =
+      options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
+    iter = db_->NewIterator(non_blocking_opts);
+    count = 0;
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+      count++;
+    }
+    ASSERT_EQ(count, 0);
+    ASSERT_TRUE(iter->status().IsIncomplete());
+    ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
+    ASSERT_EQ(cache_miss,
+              options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
+    delete iter;
+
+    // read in the specified block via a regular get
+    ASSERT_EQ(Get("a"), "b");
+
+    // verify that we can find it via a non-blocking scan
+    numopen = options.statistics.get()->getTickerCount(NO_FILE_OPENS);
+    cache_miss = options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS);
+    iter = db_->NewIterator(non_blocking_opts);
+    count = 0;
+    for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+      ASSERT_TRUE(iter->status().ok());
+      count++;
+    }
+    ASSERT_EQ(count, 1);
+    ASSERT_EQ(numopen, options.statistics.get()->getTickerCount(NO_FILE_OPENS));
+    ASSERT_EQ(cache_miss,
+              options.statistics.get()->getTickerCount(BLOCK_CACHE_MISS));
+    delete iter;
+
+  } while (ChangeOptions());
+}
+
 // A delete is skipped for key if KeyMayExist(key) returns False
 // Tests Writebatch consistency and proper delete behaviour
 TEST(DBTest, FilterDeletes) {
@@ -1028,6 +1134,95 @@ TEST(DBTest, IterMulti) {
   } while (ChangeCompactOptions());
 }
 
+// Check that we can skip over a run of user keys
+// by using reseek rather than sequential scan
+TEST(DBTest, IterReseek) {
+  Options options = CurrentOptions();
+  options.max_sequential_skip_in_iterations = 3;
+  options.create_if_missing = true;
+  options.statistics = leveldb::CreateDBStatistics();
+  DestroyAndReopen(&options);
+
+  // insert two keys with same userkey and verify that
+  // reseek is not invoked. For each of these test cases,
+  // verify that we can find the next key "b".
+  ASSERT_OK(Put("a",  "one"));
+  ASSERT_OK(Put("a",  "two"));
+  ASSERT_OK(Put("b",  "bone"));
+  Iterator* iter = db_->NewIterator(ReadOptions());
+  iter->SeekToFirst();
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), 0);
+  ASSERT_EQ(IterStatus(iter), "a->two");
+  iter->Next();
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), 0);
+  ASSERT_EQ(IterStatus(iter), "b->bone");
+  delete iter;
+
+  // insert a total of three keys with same userkey and verify
+  // that reseek is still not invoked.
+  ASSERT_OK(Put("a",  "three"));
+  iter = db_->NewIterator(ReadOptions());
+  iter->SeekToFirst();
+  ASSERT_EQ(IterStatus(iter), "a->three");
+  iter->Next();
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), 0);
+  ASSERT_EQ(IterStatus(iter), "b->bone");
+  delete iter;
+
+  // insert a total of four keys with same userkey and verify
+  // that reseek is invoked.
+  ASSERT_OK(Put("a",  "four"));
+  iter = db_->NewIterator(ReadOptions());
+  iter->SeekToFirst();
+  ASSERT_EQ(IterStatus(iter), "a->four");
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), 0);
+  iter->Next();
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), 1);
+  ASSERT_EQ(IterStatus(iter), "b->bone");
+  delete iter;
+
+  // Testing reverse iterator
+  // At this point, we have three versions of "a" and one version of "b".
+  // The reseek statistics is already at 1.
+  int num_reseeks = (int)options.statistics.get()->getTickerCount(
+                 NUMBER_OF_RESEEKS_IN_ITERATION);
+
+  // Insert another version of b and assert that reseek is not invoked
+  ASSERT_OK(Put("b",  "btwo"));
+  iter = db_->NewIterator(ReadOptions());
+  iter->SeekToLast();
+  ASSERT_EQ(IterStatus(iter), "b->btwo");
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks);
+  iter->Prev();
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks+1);
+  ASSERT_EQ(IterStatus(iter), "a->four");
+  delete iter;
+
+  // insert two more versions of b. This makes a total of 4 versions
+  // of b and 4 versions of a.
+  ASSERT_OK(Put("b",  "bthree"));
+  ASSERT_OK(Put("b",  "bfour"));
+  iter = db_->NewIterator(ReadOptions());
+  iter->SeekToLast();
+  ASSERT_EQ(IterStatus(iter), "b->bfour");
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks + 2);
+  iter->Prev();
+
+  // the previous Prev call should have invoked reseek
+  ASSERT_EQ(options.statistics.get()->getTickerCount(
+            NUMBER_OF_RESEEKS_IN_ITERATION), num_reseeks + 3);
+  ASSERT_EQ(IterStatus(iter), "a->four");
+  delete iter;
+}
+
 TEST(DBTest, IterSmallAndLargeMix) {
   do {
     ASSERT_OK(Put("a", "va"));
@@ -1171,6 +1366,24 @@ TEST(DBTest, CheckLock) {
   } while (ChangeCompactOptions());
 }
 
+TEST(DBTest, FlushMultipleMemtable) {
+  do {
+    Options options = CurrentOptions();
+    WriteOptions writeOpt = WriteOptions();
+    writeOpt.disableWAL = true;
+    options.max_write_buffer_number = 4;
+    options.min_write_buffer_number_to_merge = 3;
+    Reopen(&options);
+    ASSERT_OK(dbfull()->Put(writeOpt, "foo", "v1"));
+    dbfull()->Flush(FlushOptions());
+    ASSERT_OK(dbfull()->Put(writeOpt, "bar", "v1"));
+
+    ASSERT_EQ("v1", Get("foo"));
+    ASSERT_EQ("v1", Get("bar"));
+    dbfull()->Flush(FlushOptions());
+  } while (ChangeCompactOptions());
+}
+
 TEST(DBTest, FLUSH) {
   do {
     Options options = CurrentOptions();
@@ -1515,6 +1728,100 @@ TEST(DBTest, UniversalCompactionTrigger) {
   }
 }
 
+TEST(DBTest, ConvertCompactionStyle) {
+  Random rnd(301);
+  int max_key_level_insert = 200;
+  int max_key_universal_insert = 600;
+
+  // Stage 1: generate a db with level compaction
+  Options options = CurrentOptions();
+  options.write_buffer_size = 100<<10; //100KB
+  options.num_levels = 4;
+  options.level0_file_num_compaction_trigger = 3;
+  options.max_bytes_for_level_base = 500<<10; // 500KB
+  options.max_bytes_for_level_multiplier = 1;
+  options.target_file_size_base = 200<<10; // 200KB
+  options.target_file_size_multiplier = 1;
+  Reopen(&options);
+
+  for (int i = 0; i <= max_key_level_insert; i++) {
+    // each value is 10K
+    ASSERT_OK(Put(Key(i), RandomString(&rnd, 10000)));
+  }
+  dbfull()->Flush(FlushOptions());
+  dbfull()->TEST_WaitForCompact();
+
+  ASSERT_GT(TotalTableFiles(), 1);
+  int non_level0_num_files = 0;
+  for (int i = 1; i < dbfull()->NumberLevels(); i++) {
+    non_level0_num_files += NumTableFilesAtLevel(i);
+  }
+  ASSERT_GT(non_level0_num_files, 0);
+
+  // Stage 2: reopen with universal compaction - should fail
+  options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  Status s = TryReopen(&options);
+  ASSERT_TRUE(s.IsInvalidArgument());
+
+  // Stage 3: compact into a single file and move the file to level 0
+  options = CurrentOptions();
+  options.disable_auto_compactions = true;
+  options.target_file_size_base = INT_MAX;
+  options.target_file_size_multiplier = 1;
+  options.max_bytes_for_level_base = INT_MAX;
+  options.max_bytes_for_level_multiplier = 1;
+  Reopen(&options);
+
+  dbfull()->CompactRange(nullptr, nullptr,
+                         true /* reduce level */,
+                         0    /* reduce to level 0 */);
+
+  for (int i = 0; i < dbfull()->NumberLevels(); i++) {
+    int num = NumTableFilesAtLevel(i);
+    if (i == 0) {
+      ASSERT_EQ(num, 1);
+    } else {
+      ASSERT_EQ(num, 0);
+    }
+  }
+
+  // Stage 4: re-open in universal compaction style and do some db operations
+  options = CurrentOptions();
+  options.compaction_style = kCompactionStyleUniversal;
+  options.write_buffer_size = 100<<10; //100KB
+  options.level0_file_num_compaction_trigger = 3;
+  Reopen(&options);
+
+  for (int i = max_key_level_insert / 2; i <= max_key_universal_insert; i++) {
+    ASSERT_OK(Put(Key(i), RandomString(&rnd, 10000)));
+  }
+  dbfull()->Flush(FlushOptions());
+  dbfull()->TEST_WaitForCompact();
+
+  for (int i = 1; i < dbfull()->NumberLevels(); i++) {
+    ASSERT_EQ(NumTableFilesAtLevel(i), 0);
+  }
+
+  // verify keys inserted in both level compaction style and universal
+  // compaction style
+  std::string keys_in_db;
+  Iterator* iter = dbfull()->NewIterator(ReadOptions());
+  for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
+    keys_in_db.append(iter->key().ToString());
+    keys_in_db.push_back(',');
+  }
+  delete iter;
+
+  std::string expected_keys;
+  for (int i = 0; i <= max_key_universal_insert; i++) {
+    expected_keys.append(Key(i));
+    expected_keys.push_back(',');
+  }
+
+  ASSERT_EQ(keys_in_db, expected_keys);
+}
+
 void MinLevelHelper(DBTest* self, Options& options) {
   Random rnd(301);
 
@@ -3425,7 +3732,7 @@ class ModelDB: public DB {
     }
   }
   virtual void CompactRange(const Slice* start, const Slice* end,
-                            bool reduce_level ) {
+                            bool reduce_level, int target_level) {
   }
 
   virtual int NumberLevels()
@@ -3509,10 +3816,13 @@ class ModelDB: public DB {
   KVMap map_;
 };
 
-static std::string RandomKey(Random* rnd) {
-  int len = (rnd->OneIn(3)
-             ? 1                // Short sometimes to encourage collisions
-             : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
+static std::string RandomKey(Random* rnd, int minimum = 0) {
+  int len;
+  do {
+    len = (rnd->OneIn(3)
+           ? 1                // Short sometimes to encourage collisions
+           : (rnd->OneIn(100) ? rnd->Skewed(10) : rnd->Uniform(10)));
+  } while (len < minimum);
   return test::RandomKey(rnd, len);
 }
 
@@ -3574,8 +3884,12 @@ TEST(DBTest, Randomized) {
     for (int step = 0; step < N; step++) {
       // TODO(sanjay): Test Get() works
       int p = rnd.Uniform(100);
+      int minimum = 0;
+      if (option_config_ == kPrefixHashRep) {
+        minimum = 1;
+      }
       if (p < 45) {                               // Put
-        k = RandomKey(&rnd);
+        k = RandomKey(&rnd, minimum);
         v = RandomString(&rnd,
                          rnd.OneIn(20)
                          ? 100 + rnd.Uniform(100)
@@ -3584,7 +3898,7 @@ TEST(DBTest, Randomized) {
         ASSERT_OK(db_->Put(WriteOptions(), k, v));
 
       } else if (p < 90) {                        // Delete
-        k = RandomKey(&rnd);
+        k = RandomKey(&rnd, minimum);
         ASSERT_OK(model.Delete(WriteOptions(), k));
         ASSERT_OK(db_->Delete(WriteOptions(), k));
 
@@ -3594,7 +3908,7 @@ TEST(DBTest, Randomized) {
         const int num = rnd.Uniform(8);
         for (int i = 0; i < num; i++) {
           if (i == 0 || !rnd.OneIn(10)) {
-            k = RandomKey(&rnd);
+            k = RandomKey(&rnd, minimum);
           } else {
             // Periodically re-use the same key from the previous iter, so
             // we have multiple entries in the write batch for the same key
@@ -3750,6 +4064,9 @@ TEST(DBTest, PrefixScan) {
   snprintf(buf, sizeof(buf), "03______:");
   prefix = Slice(buf, 8);
   key = Slice(buf, 9);
+  auto prefix_extractor = NewFixedPrefixTransform(8);
+  auto memtable_factory =
+    std::make_shared<PrefixHashRepFactory>(prefix_extractor);
 
   // db configs
   env_->count_random_reads_ = true;
@@ -3757,12 +4074,13 @@ TEST(DBTest, PrefixScan) {
   options.env = env_;
   options.block_cache = NewLRUCache(0);  // Prevent cache hits
   options.filter_policy =  NewBloomFilterPolicy(10);
-  options.prefix_extractor = NewFixedPrefixTransform(8);
+  options.prefix_extractor = prefix_extractor;
   options.whole_key_filtering = false;
   options.disable_auto_compactions = true;
   options.max_background_compactions = 2;
   options.create_if_missing = true;
   options.disable_seek_compaction = true;
+  options.memtable_factory = memtable_factory;
 
   // prefix specified, with blooms: 2 RAND I/Os
   // SeekToFirst
@@ -3816,7 +4134,6 @@ TEST(DBTest, PrefixScan) {
   ASSERT_EQ(env_->random_read_counter_.Read(), 11);
   Close();
   delete options.filter_policy;
-  delete options.prefix_extractor;
 }
 
 std::string MakeKey(unsigned int num) {

db/dbformat.cc

@@ -6,7 +6,7 @@
 #include "db/dbformat.h"
 #include "port/port.h"
 #include "util/coding.h"
-#include "include/leveldb/perf_context.h"
+#include "util/perf_context_imp.h"
 
 namespace leveldb {
 
@@ -54,7 +54,7 @@ int InternalKeyComparator::Compare(const Slice& akey, const Slice& bkey) const {
   //    decreasing sequence number
   //    decreasing type (though sequence# should be enough to disambiguate)
   int r = user_comparator_->Compare(ExtractUserKey(akey), ExtractUserKey(bkey));
-  perf_context.user_key_comparison_count++;
+  BumpPerfCount(&perf_context.user_key_comparison_count);
   if (r == 0) {
     const uint64_t anum = DecodeFixed64(akey.data() + akey.size() - 8);
     const uint64_t bnum = DecodeFixed64(bkey.data() + bkey.size() - 8);

db/dbformat.h

@@ -6,12 +6,12 @@
 #define STORAGE_LEVELDB_DB_FORMAT_H_
 
 #include <stdio.h>
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/slice.h"
-#include "leveldb/table_builder.h"
-#include "leveldb/types.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/table_builder.h"
+#include "rocksdb/types.h"
 #include "util/coding.h"
 #include "util/logging.h"
 

db/deletefile_test.cc

@@ -0,0 +1,188 @@
+// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
+#include "rocksdb/db.h"
+#include "db/db_impl.h"
+#include "db/filename.h"
+#include "db/version_set.h"
+#include "db/write_batch_internal.h"
+#include "util/testharness.h"
+#include "util/testutil.h"
+#include "boost/lexical_cast.hpp"
+#include "rocksdb/env.h"
+#include <vector>
+#include <boost/algorithm/string.hpp>
+#include <stdlib.h>
+#include <map>
+
+namespace leveldb {
+
+class DeleteFileTest {
+ public:
+  std::string dbname_;
+  Options options_;
+  DB* db_;
+  Env* env_;
+  int numlevels_;
+
+  DeleteFileTest() {
+    db_ = nullptr;
+    env_ = Env::Default();
+    options_.write_buffer_size = 1024*1024*1000;
+    options_.target_file_size_base = 1024*1024*1000;
+    options_.max_bytes_for_level_base = 1024*1024*1000;
+    dbname_ = test::TmpDir() + "/deletefile_test";
+    DestroyDB(dbname_, options_);
+    numlevels_ = 7;
+    ASSERT_OK(ReopenDB(true));
+  }
+
+  Status ReopenDB(bool create) {
+    delete db_;
+    if (create) {
+      DestroyDB(dbname_, options_);
+    }
+    db_ = nullptr;
+    options_.create_if_missing = create;
+    return DB::Open(options_, dbname_, &db_);
+  }
+
+  void CloseDB() {
+    delete db_;
+  }
+
+  void AddKeys(int numkeys, int startkey = 0) {
+    WriteOptions options;
+    options.sync = false;
+    ReadOptions roptions;
+    for (int i = startkey; i < (numkeys + startkey) ; i++) {
+      std::string temp = boost::lexical_cast<std::string>(i);
+      Slice key(temp);
+      Slice value(temp);
+      ASSERT_OK(db_->Put(options, key, value));
+    }
+  }
+
+  int numKeysInLevels(
+    std::vector<LiveFileMetaData> &metadata,
+    std::vector<int> *keysperlevel = nullptr) {
+
+    if (keysperlevel != nullptr) {
+      keysperlevel->resize(numlevels_);
+    }
+
+    int numKeys = 0;
+    for (size_t i = 0; i < metadata.size(); i++) {
+      int startkey = atoi(metadata[i].smallestkey.c_str());
+      int endkey = atoi(metadata[i].largestkey.c_str());
+      int numkeysinfile = (endkey - startkey + 1);
+      numKeys += numkeysinfile;
+      if (keysperlevel != nullptr) {
+        (*keysperlevel)[(int)metadata[i].level] += numkeysinfile;
+      }
+      fprintf(stderr, "level %d name %s smallest %s largest %s\n",
+              metadata[i].level, metadata[i].name.c_str(),
+              metadata[i].smallestkey.c_str(),
+              metadata[i].largestkey.c_str());
+    }
+    return numKeys;
+  }
+
+  void CreateTwoLevels() {
+    AddKeys(50000, 10000);
+    DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
+    ASSERT_OK(dbi->TEST_CompactMemTable());
+    ASSERT_OK(dbi->TEST_WaitForCompactMemTable());
+
+    AddKeys(50000, 10000);
+    ASSERT_OK(dbi->TEST_CompactMemTable());
+    ASSERT_OK(dbi->TEST_WaitForCompactMemTable());
+  }
+
+};
+
+TEST(DeleteFileTest, AddKeysAndQueryLevels) {
+  CreateTwoLevels();
+  std::vector<LiveFileMetaData> metadata;
+  std::vector<int> keysinlevel;
+  db_->GetLiveFilesMetaData(&metadata);
+
+  std::string level1file = "";
+  int level1keycount = 0;
+  std::string level2file = "";
+  int level2keycount = 0;
+  int level1index = 0;
+  int level2index = 1;
+
+  ASSERT_EQ((int)metadata.size(), 2);
+  if (metadata[0].level == 2) {
+    level1index = 1;
+    level2index = 0;
+  }
+
+  level1file = metadata[level1index].name;
+  int startkey = atoi(metadata[level1index].smallestkey.c_str());
+  int endkey = atoi(metadata[level1index].largestkey.c_str());
+  level1keycount = (endkey - startkey + 1);
+  level2file = metadata[level2index].name;
+  startkey = atoi(metadata[level2index].smallestkey.c_str());
+  endkey = atoi(metadata[level2index].largestkey.c_str());
+  level2keycount = (endkey - startkey + 1);
+
+  // COntrolled setup. Levels 1 and 2 should both have 50K files.
+  // This is a little fragile as it depends on the current
+  // compaction heuristics.
+  ASSERT_EQ(level1keycount, 50000);
+  ASSERT_EQ(level2keycount, 50000);
+
+  Status status = db_->DeleteFile("0.sst");
+  ASSERT_TRUE(status.IsInvalidArgument());
+
+  // intermediate level files cannot be deleted.
+  status = db_->DeleteFile(level1file);
+  ASSERT_TRUE(status.IsInvalidArgument());
+
+  // Lowest level file deletion should succeed.
+  ASSERT_OK(db_->DeleteFile(level2file));
+
+  CloseDB();
+}
+
+
+TEST(DeleteFileTest, DeleteFileWithIterator) {
+  CreateTwoLevels();
+  ReadOptions options;
+  Iterator* it = db_->NewIterator(options);
+  std::vector<LiveFileMetaData> metadata;
+  db_->GetLiveFilesMetaData(&metadata);
+
+  std::string level2file = "";
+
+  ASSERT_EQ((int)metadata.size(), 2);
+  if (metadata[0].level == 1) {
+    level2file = metadata[1].name;
+  } else {
+    level2file = metadata[0].name;
+  }
+
+  Status status = db_->DeleteFile(level2file);
+  fprintf(stdout, "Deletion status %s: %s\n",
+          level2file.c_str(), status.ToString().c_str());
+  ASSERT_TRUE(status.ok());
+  it->SeekToFirst();
+  int numKeysIterated = 0;
+  while(it->Valid()) {
+    numKeysIterated++;
+    it->Next();
+  }
+  ASSERT_EQ(numKeysIterated, 50000);
+  delete it;
+  CloseDB();
+}
+} //namespace leveldb
+
+int main(int argc, char** argv) {
+  return leveldb::test::RunAllTests();
+}
+

db/filename.cc

@@ -7,7 +7,7 @@
 #include <ctype.h>
 #include <stdio.h>
 #include "db/dbformat.h"
-#include "leveldb/env.h"
+#include "rocksdb/env.h"
 #include "util/logging.h"
 
 namespace leveldb {
@@ -46,13 +46,10 @@ extern Status WriteStringToFileSync(Env* env, const Slice& data,
 static std::string MakeFileName(const std::string& name, uint64_t number,
                                 const char* suffix) {
   char buf[100];
-  snprintf(buf, sizeof(buf), "%06llu.%s",
+  snprintf(buf, sizeof(buf), "/%06llu.%s",
            static_cast<unsigned long long>(number),
            suffix);
-  if (name.empty()) {
-    return buf;
-  }
-  return name + "/" + buf;
+  return name + buf;
 }
 
 std::string LogFileName(const std::string& name, uint64_t number) {
@@ -65,7 +62,7 @@ std::string ArchivalDirectory(const std::string& dbname) {
 }
 std::string ArchivedLogFileName(const std::string& name, uint64_t number) {
   assert(number > 0);
-  return MakeFileName(name + "/archive", number, "log");
+  return MakeFileName(name + "/" + ARCHIVAL_DIR, number, "log");
 }
 
 std::string TableFileName(const std::string& name, uint64_t number) {
@@ -133,10 +130,14 @@ std::string MetaDatabaseName(const std::string& dbname, uint64_t number) {
 //    dbname/MANIFEST-[0-9]+
 //    dbname/[0-9]+.(log|sst)
 //    dbname/METADB-[0-9]+
+//    Disregards / at the beginning
 bool ParseFileName(const std::string& fname,
                    uint64_t* number,
                    FileType* type) {
   Slice rest(fname);
+  if (fname.length() > 1 && fname[0] == '/') {
+    rest.remove_prefix(1);
+  }
   if (rest == "CURRENT") {
     *number = 0;
     *type = kCurrentFile;

db/filename.h

@@ -9,8 +9,8 @@
 
 #include <stdint.h>
 #include <string>
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
 #include "port/port.h"
 
 namespace leveldb {

db/log_reader.cc

@@ -5,7 +5,7 @@
 #include "db/log_reader.h"
 
 #include <stdio.h>
-#include "leveldb/env.h"
+#include "rocksdb/env.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
 

db/log_reader.h

@@ -9,8 +9,8 @@
 #include <stdint.h>
 
 #include "db/log_format.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
 
 namespace leveldb {
 

db/log_test.cc

@@ -4,7 +4,7 @@
 
 #include "db/log_reader.h"
 #include "db/log_writer.h"
-#include "leveldb/env.h"
+#include "rocksdb/env.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
 #include "util/random.h"

db/log_writer.cc

@@ -5,7 +5,7 @@
 #include "db/log_writer.h"
 
 #include <stdint.h>
-#include "leveldb/env.h"
+#include "rocksdb/env.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
 

db/log_writer.h

@@ -8,8 +8,8 @@
 #include <memory>
 #include <stdint.h>
 #include "db/log_format.h"
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
 
 namespace leveldb {
 

db/memtable.cc

@@ -7,21 +7,15 @@
 #include <memory>
 
 #include "db/dbformat.h"
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "leveldb/merge_operator.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/merge_operator.h"
 #include "util/coding.h"
+#include "util/murmurhash.h"
 
 namespace leveldb {
 
-static Slice GetLengthPrefixedSlice(const char* data) {
-  uint32_t len;
-  const char* p = data;
-  p = GetVarint32Ptr(p, p + 5, &len);  // +5: we assume "p" is not corrupted
-  return Slice(p, len);
-}
-
 MemTable::MemTable(const InternalKeyComparator& cmp,
                    std::shared_ptr<MemTableRepFactory> table_factory,
                    int numlevel,
@@ -42,7 +36,8 @@ MemTable::~MemTable() {
 }
 
 size_t MemTable::ApproximateMemoryUsage() {
-  return arena_impl_.ApproximateMemoryUsage();
+  return arena_impl_.ApproximateMemoryUsage() +
+    table_->ApproximateMemoryUsage();
 }
 
 int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
@@ -53,6 +48,11 @@ int MemTable::KeyComparator::operator()(const char* aptr, const char* bptr)
   return comparator.Compare(a, b);
 }
 
+Slice MemTableRep::UserKey(const char* key) const {
+  Slice slice = GetLengthPrefixedSlice(key);
+  return Slice(slice.data(), slice.size() - 8);
+}
+
 // Encode a suitable internal key target for "target" and return it.
 // Uses *scratch as scratch space, and the returned pointer will point
 // into this scratch space.
@@ -68,6 +68,9 @@ class MemTableIterator: public Iterator {
   explicit MemTableIterator(MemTableRep* table)
     : iter_(table->GetIterator()) { }
 
+  MemTableIterator(MemTableRep* table, const Slice* prefix)
+    : iter_(table->GetPrefixIterator(*prefix)) { }
+
   virtual bool Valid() const { return iter_->Valid(); }
   virtual void Seek(const Slice& k) { iter_->Seek(EncodeKey(&tmp_, k)); }
   virtual void SeekToFirst() { iter_->SeekToFirst(); }
@@ -93,8 +96,12 @@ class MemTableIterator: public Iterator {
   void operator=(const MemTableIterator&);
 };
 
-Iterator* MemTable::NewIterator() {
-  return new MemTableIterator(table_.get());
+Iterator* MemTable::NewIterator(const Slice* prefix) {
+  if (prefix) {
+    return new MemTableIterator(table_.get(), prefix);
+  } else {
+    return new MemTableIterator(table_.get());
+  }
 }
 
 void MemTable::Add(SequenceNumber s, ValueType type,
@@ -132,7 +139,8 @@ void MemTable::Add(SequenceNumber s, ValueType type,
 bool MemTable::Get(const LookupKey& key, std::string* value, Status* s,
                    std::deque<std::string>* operands, const Options& options) {
   Slice memkey = key.memtable_key();
-  std::shared_ptr<MemTableRep::Iterator> iter(table_.get()->GetIterator());
+  std::shared_ptr<MemTableRep::Iterator> iter(
+    table_->GetIterator(key.user_key()));
   iter->Seek(memkey.data());
 
   // It is the caller's responsibility to allocate/delete operands list

db/memtable.h

@@ -11,8 +11,8 @@
 #include "db/dbformat.h"
 #include "db/skiplist.h"
 #include "db/version_set.h"
-#include "leveldb/db.h"
-#include "leveldb/memtablerep.h"
+#include "rocksdb/db.h"
+#include "rocksdb/memtablerep.h"
 #include "util/arena_impl.h"
 
 namespace leveldb {
@@ -61,7 +61,11 @@ class MemTable {
   // while the returned iterator is live.  The keys returned by this
   // iterator are internal keys encoded by AppendInternalKey in the
   // db/dbformat.{h,cc} module.
-  Iterator* NewIterator();
+  //
+  // If a prefix is supplied, it is passed to the underlying MemTableRep as a
+  // hint that the iterator only need to support access to keys with that
+  // prefix.
+  Iterator* NewIterator(const Slice* prefix = nullptr);
 
   // Add an entry into memtable that maps key to value at the
   // specified sequence number and with the specified type.
@@ -96,6 +100,9 @@ class MemTable {
   // memstore is flushed to storage
   void SetLogNumber(uint64_t num) { mem_logfile_number_ = num; }
 
+  // Notify the underlying storage that no more items will be added
+  void MarkImmutable() { table_->MarkReadOnly(); }
+
  private:
   ~MemTable();  // Private since only Unref() should be used to delete it
   friend class MemTableIterator;

db/memtablelist.cc

@@ -4,10 +4,10 @@
 #include "db/memtablelist.h"
 
 #include <string>
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 #include "db/memtable.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
 #include "util/coding.h"
 
 namespace leveldb {
@@ -42,7 +42,8 @@ int MemTableList::size() {
 // Returns true if there is at least one memtable on which flush has
 // not yet started.
 bool MemTableList::IsFlushPending(int min_write_buffer_number_to_merge) {
-  if (num_flush_not_started_ >= min_write_buffer_number_to_merge) {
+  if ((flush_requested_ && num_flush_not_started_ >= 1) ||
+      (num_flush_not_started_ >= min_write_buffer_number_to_merge)) {
     assert(imm_flush_needed.NoBarrier_Load() != nullptr);
     return true;
   }
@@ -63,6 +64,7 @@ void MemTableList::PickMemtablesToFlush(std::vector<MemTable*>* ret) {
       ret->push_back(m);
     }
   }
+  flush_requested_ = false; // start-flush request is complete
 }
 
 // Record a successful flush in the manifest file
@@ -172,6 +174,7 @@ void MemTableList::Add(MemTable* m) {
   assert(size_ >= num_flush_not_started_);
   size_++;
   memlist_.push_front(m);
+  m->MarkImmutable();
   num_flush_not_started_++;
   if (num_flush_not_started_ == 1) {
     imm_flush_needed.Release_Store((void *)1);

db/memtablelist.h

@@ -6,7 +6,7 @@
 #include <string>
 #include <list>
 #include <deque>
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 #include "db/dbformat.h"
 #include "db/skiplist.h"
 #include "memtable.h"
@@ -29,7 +29,8 @@ class MemTableList {
  public:
   // A list of memtables.
   MemTableList() : size_(0), num_flush_not_started_(0),
-    commit_in_progress_(false) {
+    commit_in_progress_(false),
+    flush_requested_(false) {
     imm_flush_needed.Release_Store(nullptr);
   }
   ~MemTableList() {};
@@ -76,6 +77,9 @@ class MemTableList {
   // Returns the list of underlying memtables.
   void GetMemTables(std::vector<MemTable*>* list);
 
+  // Request a flush of all existing memtables to storage
+  void FlushRequested() { flush_requested_ = true; }
+
   // Copying allowed
   // MemTableList(const MemTableList&);
   // void operator=(const MemTableList&);
@@ -90,6 +94,9 @@ class MemTableList {
   // committing in progress
   bool commit_in_progress_;
 
+  // Requested a flush of all memtables to storage
+  bool flush_requested_;
+
 };
 
 }  // namespace leveldb

db/merge_helper.cc

@@ -1,8 +1,8 @@
 #include "merge_helper.h"
 #include "db/dbformat.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/merge_operator.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/merge_operator.h"
 #include <string>
 #include <stdio.h>
 

db/merge_helper.h

@@ -2,8 +2,8 @@
 #define MERGE_HELPER_H
 
 #include "db/dbformat.h"
-#include "leveldb/slice.h"
-#include "leveldb/statistics.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/statistics.h"
 #include <string>
 #include <deque>
 

db/merge_operator.cc

@@ -5,7 +5,7 @@
  * Copyright 2013 Facebook
  */
 
-#include "leveldb/merge_operator.h"
+#include "rocksdb/merge_operator.h"
 
 namespace leveldb {
 

db/merge_test.cc

@@ -2,11 +2,11 @@
 #include <memory>
 #include <iostream>
 
-#include "leveldb/cache.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/merge_operator.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/merge_operator.h"
 #include "db/dbformat.h"
 #include "db/db_impl.h"
 #include "utilities/merge_operators.h"

db/perf_context.cc

@@ -1,12 +0,0 @@
-#include "include/leveldb/perf_context.h"
-
-
-namespace leveldb {
-
-void PerfContext::Reset() {
-  user_key_comparison_count = 0;
-}
-
-__thread PerfContext perf_context;
-
-}

db/perf_context_test.cc

@@ -1,7 +1,10 @@
 #include <iostream>
+#include <vector>
 
-#include "leveldb/db.h"
-#include "leveldb/perf_context.h"
+#include "rocksdb/db.h"
+#include "rocksdb/perf_context.h"
+#include "util/histogram.h"
+#include "util/stop_watch.h"
 #include "util/testharness.h"
 
 
@@ -10,11 +13,11 @@ namespace leveldb {
 // Path to the database on file system
 const std::string kDbName = test::TmpDir() + "/perf_context_test";
 
-std::shared_ptr<DB> OpenDb() {
+std::shared_ptr<DB> OpenDb(size_t write_buffer_size) {
     DB* db;
     Options options;
     options.create_if_missing = true;
-    options.write_buffer_size = 1000000000;     // give it a big memtable
+    options.write_buffer_size = write_buffer_size;
     Status s = DB::Open(options, kDbName,  &db);
     ASSERT_OK(s);
     return std::shared_ptr<DB>(db);
@@ -24,11 +27,48 @@ class PerfContextTest { };
 
 int kTotalKeys = 100;
 
-TEST(PerfContextTest, KeyComparisonCount) {
+TEST(PerfContextTest, StopWatchNanoOverhead) {
+  // profile the timer cost by itself!
+  const int kTotalIterations = 1000000;
+  std::vector<uint64_t> timings(kTotalIterations);
 
+  StopWatchNano timer(Env::Default(), true);
+  for (auto& timing : timings) {
+    timing = timer.ElapsedNanos(true /* reset */);
+  }
+
+  HistogramImpl histogram;
+  for (const auto timing : timings) {
+    histogram.Add(timing);
+  }
+
+  std::cout << histogram.ToString();
+}
+
+TEST(PerfContextTest, StopWatchOverhead) {
+  // profile the timer cost by itself!
+  const int kTotalIterations = 1000000;
+  std::vector<uint64_t> timings(kTotalIterations);
+
+  StopWatch timer(Env::Default());
+  for (auto& timing : timings) {
+    timing = timer.ElapsedMicros();
+  }
+
+  HistogramImpl histogram;
+  uint64_t prev_timing = 0;
+  for (const auto timing : timings) {
+    histogram.Add(timing - prev_timing);
+    prev_timing = timing;
+  }
+
+  std::cout << histogram.ToString();
+}
+
+void ProfileKeyComparison() {
   DestroyDB(kDbName, Options());    // Start this test with a fresh DB
 
-  auto db = OpenDb();
+  auto db = OpenDb(1000000000);
 
   WriteOptions write_options;
   ReadOptions read_options;
@@ -63,12 +103,20 @@ TEST(PerfContextTest, KeyComparisonCount) {
             << max_user_key_comparison_get << "\n"
             << "avg user key comparison get:"
             << total_user_key_comparison_get/kTotalKeys << "\n";
-
 }
 
+TEST(PerfContextTest, KeyComparisonCount) {
+  SetPerfLevel(kDisable);
+  ProfileKeyComparison();
 
+  SetPerfLevel(kEnableCount);
+  ProfileKeyComparison();
+
+  SetPerfLevel(kEnableTime);
+  ProfileKeyComparison();
 }
 
+}
 
 int main(int argc, char** argv) {
 

db/prefix_filter_iterator.h

@@ -12,7 +12,7 @@
 #ifndef STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
 #define STORAGE_LEVELDB_DB_PREFIX_FILTER_ITERATOR_H_
 
-#include "leveldb/iterator.h"
+#include "rocksdb/iterator.h"
 
 namespace leveldb {
 

db/repair.cc

@@ -34,9 +34,9 @@
 #include "db/table_cache.h"
 #include "db/version_edit.h"
 #include "db/write_batch_internal.h"
-#include "leveldb/comparator.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
 
 namespace leveldb {
 

db/skiplist_test.cc

@@ -4,7 +4,7 @@
 
 #include "db/skiplist.h"
 #include <set>
-#include "leveldb/env.h"
+#include "rocksdb/env.h"
 #include "util/arena_impl.h"
 #include "util/hash.h"
 #include "util/random.h"

db/snapshot.h

@@ -5,7 +5,7 @@
 #ifndef STORAGE_LEVELDB_DB_SNAPSHOT_H_
 #define STORAGE_LEVELDB_DB_SNAPSHOT_H_
 
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 
 namespace leveldb {
 

db/table_cache.cc

@@ -6,7 +6,7 @@
 
 #include "db/filename.h"
 
-#include "leveldb/statistics.h"
+#include "rocksdb/statistics.h"
 #include "table/table.h"
 #include "util/coding.h"
 #include "util/stop_watch.h"
@@ -48,7 +48,7 @@ Status TableCache::FindTable(const EnvOptions& toptions,
   *handle = cache_->Lookup(key);
   if (*handle == nullptr) {
     if (no_io) { // Dont do IO and return a not-found status
-      return Status::NotFound("Table not found in table_cache, no_io is set");
+      return Status::Incomplete("Table not found in table_cache, no_io is set");
     }
     if (table_io != nullptr) {
       *table_io = true;    // we had to do IO from storage
@@ -90,7 +90,8 @@ Iterator* TableCache::NewIterator(const ReadOptions& options,
   }
 
   Cache::Handle* handle = nullptr;
-  Status s = FindTable(toptions, file_number, file_size, &handle);
+  Status s = FindTable(toptions, file_number, file_size, &handle,
+                       nullptr, options.read_tier == kBlockCacheTier);
   if (!s.ok()) {
     return NewErrorIterator(s);
   }
@@ -117,17 +118,17 @@ Status TableCache::Get(const ReadOptions& options,
                        void* arg,
                        bool (*saver)(void*, const Slice&, const Slice&, bool),
                        bool* table_io,
-                       void (*mark_key_may_exist)(void*),
-                       const bool no_io) {
+                       void (*mark_key_may_exist)(void*)) {
   Cache::Handle* handle = nullptr;
   Status s = FindTable(storage_options_, file_number, file_size,
-                       &handle, table_io, no_io);
+                       &handle, table_io,
+                       options.read_tier == kBlockCacheTier);
   if (s.ok()) {
     Table* t =
       reinterpret_cast<Table*>(cache_->Value(handle));
-    s = t->InternalGet(options, k, arg, saver, mark_key_may_exist, no_io);
+    s = t->InternalGet(options, k, arg, saver, mark_key_may_exist);
     cache_->Release(handle);
-  } else if (no_io && s.IsNotFound()) {
+  } else if (options.read_tier && s.IsIncomplete()) {
     // Couldnt find Table in cache but treat as kFound if no_io set
     (*mark_key_may_exist)(arg);
     return Status::OK();
@@ -135,6 +136,24 @@ Status TableCache::Get(const ReadOptions& options,
   return s;
 }
 
+bool TableCache::PrefixMayMatch(const ReadOptions& options,
+                                uint64_t file_number,
+                                uint64_t file_size,
+                                const Slice& internal_prefix,
+                                bool* table_io) {
+  Cache::Handle* handle = nullptr;
+  Status s = FindTable(storage_options_, file_number,
+                       file_size, &handle, table_io);
+  bool may_match = true;
+  if (s.ok()) {
+    Table* t =
+      reinterpret_cast<Table*>(cache_->Value(handle));
+    may_match = t->PrefixMayMatch(internal_prefix);
+    cache_->Release(handle);
+  }
+  return may_match;
+}
+
 void TableCache::Evict(uint64_t file_number) {
   char buf[sizeof(file_number)];
   EncodeFixed64(buf, file_number);

db/table_cache.h

@@ -10,8 +10,8 @@
 #include <string>
 #include <stdint.h>
 #include "db/dbformat.h"
-#include "leveldb/env.h"
-#include "leveldb/cache.h"
+#include "rocksdb/env.h"
+#include "rocksdb/cache.h"
 #include "port/port.h"
 #include "table/table.h"
 
@@ -49,8 +49,13 @@ class TableCache {
              void* arg,
              bool (*handle_result)(void*, const Slice&, const Slice&, bool),
              bool* table_io,
-             void (*mark_key_may_exist)(void*) = nullptr,
-             const bool no_io = false);
+             void (*mark_key_may_exist)(void*) = nullptr);
+
+  // Determine whether the table may contain the specified prefix.  If
+  // the table index of blooms are not in memory, this may cause an I/O
+  bool PrefixMayMatch(const ReadOptions& options, uint64_t file_number,
+                      uint64_t file_size, const Slice& internal_prefix,
+                      bool* table_io);
 
   // Evict any entry for the specified file number
   void Evict(uint64_t file_number);

db/transaction_log_impl.h

@@ -4,10 +4,10 @@
 
 #include <vector>
 
-#include "leveldb/env.h"
-#include "leveldb/options.h"
-#include "leveldb/types.h"
-#include "leveldb/transaction_log.h"
+#include "rocksdb/env.h"
+#include "rocksdb/options.h"
+#include "rocksdb/types.h"
+#include "rocksdb/transaction_log.h"
 #include "db/log_reader.h"
 #include "db/filename.h"
 
@@ -31,7 +31,12 @@ class LogFileImpl : public LogFile {
     sizeFileBytes_(sizeBytes) {
   }
 
-  std::string Filename() const { return LogFileName("", logNumber_); }
+  std::string PathName() const {
+    if (type_ == kArchivedLogFile) {
+      return ArchivedLogFileName("", logNumber_);
+    }
+    return LogFileName("", logNumber_);
+  }
 
   uint64_t LogNumber() const { return logNumber_; }
 

db/version_set.cc

@@ -12,9 +12,9 @@
 #include "db/log_writer.h"
 #include "db/memtable.h"
 #include "db/table_cache.h"
-#include "leveldb/env.h"
-#include "leveldb/merge_operator.h"
-#include "leveldb/table_builder.h"
+#include "rocksdb/env.h"
+#include "rocksdb/merge_operator.h"
+#include "rocksdb/table_builder.h"
 #include "table/merger.h"
 #include "table/two_level_iterator.h"
 #include "util/coding.h"
@@ -189,7 +189,14 @@ static Iterator* GetFileIterator(void* arg,
     return NewErrorIterator(
         Status::Corruption("FileReader invoked with unexpected value"));
   } else {
-    return cache->NewIterator(options,
+    ReadOptions options_copy;
+    if (options.prefix) {
+      // suppress prefix filtering since we have already checked the
+      // filters once at this point
+      options_copy = options;
+      options_copy.prefix = nullptr;
+    }
+    return cache->NewIterator(options.prefix ? options_copy : options,
                               soptions,
                               DecodeFixed64(file_value.data()),
                               DecodeFixed64(file_value.data() + 8),
@@ -198,22 +205,55 @@ static Iterator* GetFileIterator(void* arg,
   }
 }
 
+bool Version::PrefixMayMatch(const ReadOptions& options,
+                             const EnvOptions& soptions,
+                             const Slice& internal_prefix,
+                             Iterator* level_iter) const {
+  bool may_match = true;
+  level_iter->Seek(internal_prefix);
+  if (!level_iter->Valid()) {
+    // we're past end of level
+    may_match = false;
+  } else if (ExtractUserKey(level_iter->key()).starts_with(
+                                             ExtractUserKey(internal_prefix))) {
+    // TODO(tylerharter): do we need this case?  Or are we guaranteed
+    // key() will always be the biggest value for this SST?
+    may_match = true;
+  } else {
+    may_match = vset_->table_cache_->PrefixMayMatch(
+                           options,
+                           DecodeFixed64(level_iter->value().data()),
+                           DecodeFixed64(level_iter->value().data() + 8),
+                           internal_prefix, nullptr);
+  }
+  return may_match;
+}
+
 Iterator* Version::NewConcatenatingIterator(const ReadOptions& options,
                                             const EnvOptions& soptions,
                                             int level) const {
-  return NewTwoLevelIterator(
-      new LevelFileNumIterator(vset_->icmp_, &files_[level]),
-      &GetFileIterator, vset_->table_cache_, options, soptions);
+  Iterator* level_iter = new LevelFileNumIterator(vset_->icmp_, &files_[level]);
+  if (options.prefix) {
+    InternalKey internal_prefix(*options.prefix, 0, kTypeValue);
+    if (!PrefixMayMatch(options, soptions,
+                        internal_prefix.Encode(), level_iter)) {
+      delete level_iter;
+      // nothing in this level can match the prefix
+      return NewEmptyIterator();
+    }
+  }
+  return NewTwoLevelIterator(level_iter, &GetFileIterator,
+                             vset_->table_cache_, options, soptions);
 }
 
 void Version::AddIterators(const ReadOptions& options,
                            const EnvOptions& soptions,
                            std::vector<Iterator*>* iters) {
   // Merge all level zero files together since they may overlap
-  for (size_t i = 0; i < files_[0].size(); i++) {
+  for (const FileMetaData* file : files_[0]) {
     iters->push_back(
         vset_->table_cache_->NewIterator(
-            options, soptions, files_[0][i]->number, files_[0][i]->file_size));
+            options, soptions, file->number, file->file_size));
   }
 
   // For levels > 0, we can use a concatenating iterator that sequentially
@@ -357,6 +397,7 @@ static bool NewestFirstBySeqNo(FileMetaData* a, FileMetaData* b) {
 
 Version::Version(VersionSet* vset, uint64_t version_number)
     : vset_(vset), next_(this), prev_(this), refs_(0),
+      files_(new std::vector<FileMetaData*>[vset->NumberLevels()]),
       files_by_size_(vset->NumberLevels()),
       next_file_to_compact_by_size_(vset->NumberLevels()),
       file_to_compact_(nullptr),
@@ -365,7 +406,6 @@ Version::Version(VersionSet* vset, uint64_t version_number)
       compaction_level_(vset->NumberLevels()),
       offset_manifest_file_(0),
       version_number_(version_number) {
-  files_ = new std::vector<FileMetaData*>[vset->NumberLevels()];
 }
 
 void Version::Get(const ReadOptions& options,
@@ -375,7 +415,6 @@ void Version::Get(const ReadOptions& options,
                   std::deque<std::string>* operands,
                   GetStats* stats,
                   const Options& db_options,
-                  const bool no_io,
                   bool* value_found) {
   Slice ikey = k.internal_key();
   Slice user_key = k.user_key();
@@ -385,9 +424,6 @@ void Version::Get(const ReadOptions& options,
   auto logger = db_options.info_log;
 
   assert(status->ok() || status->IsMergeInProgress());
-  if (no_io) {
-    assert(status->ok());
-  }
   Saver saver;
   saver.state = status->ok()? kNotFound : kMerge;
   saver.ucmp = ucmp;
@@ -476,7 +512,7 @@ void Version::Get(const ReadOptions& options,
       bool tableIO = false;
       *status = vset_->table_cache_->Get(options, f->number, f->file_size,
                                          ikey, &saver, SaveValue, &tableIO,
-                                         MarkKeyMayExist, no_io);
+                                         MarkKeyMayExist);
       // TODO: examine the behavior for corrupted key
       if (!status->ok()) {
         return;
@@ -2637,6 +2673,41 @@ void VersionSet::SetupOtherInputs(Compaction* c) {
   c->edit_->SetCompactPointer(level, largest);
 }
 
+Status VersionSet::GetMetadataForFile(
+    uint64_t number,
+    int *filelevel,
+    FileMetaData *meta) {
+  for (int level = 0; level < NumberLevels(); level++) {
+    const std::vector<FileMetaData*>& files = current_->files_[level];
+    for (size_t i = 0; i < files.size(); i++) {
+      if (files[i]->number == number) {
+        *meta = *files[i];
+        *filelevel = level;
+        return Status::OK();
+      }
+    }
+  }
+  return Status::NotFound("File not present in any level");
+}
+
+void VersionSet::GetLiveFilesMetaData(
+    std::vector<LiveFileMetaData> * metadata) {
+  for (int level = 0; level < NumberLevels(); level++) {
+    const std::vector<FileMetaData*>& files = current_->files_[level];
+    for (size_t i = 0; i < files.size(); i++) {
+      LiveFileMetaData filemetadata;
+      filemetadata.name = TableFileName("", files[i]->number);
+      filemetadata.level = level;
+      filemetadata.size = files[i]->file_size;
+      filemetadata.smallestkey = files[i]->smallest.user_key().ToString();
+      filemetadata.largestkey = files[i]->largest.user_key().ToString();
+      filemetadata.smallest_seqno = files[i]->smallest_seqno;
+      filemetadata.largest_seqno = files[i]->largest_seqno;
+      metadata->push_back(filemetadata);
+    }
+  }
+}
+
 Compaction* VersionSet::CompactRange(
     int level,
     const InternalKey* begin,

db/version_set.h

@@ -76,7 +76,7 @@ class Version {
   };
   void Get(const ReadOptions&, const LookupKey& key, std::string* val,
            Status* status, std::deque<std::string>* operands, GetStats* stats,
-           const Options& db_option, const bool no_io = false,
+           const Options& db_option,
            bool* value_found = nullptr);
 
   // Adds "stats" into the current state.  Returns true if a new
@@ -152,6 +152,8 @@ class Version {
   Iterator* NewConcatenatingIterator(const ReadOptions&,
                                      const EnvOptions& soptions,
                                      int level) const;
+  bool PrefixMayMatch(const ReadOptions& options, const EnvOptions& soptions,
+                      const Slice& internal_prefix, Iterator* level_iter) const;
 
   VersionSet* vset_;            // VersionSet to which this Version belongs
   Version* next_;               // Next version in linked list
@@ -405,6 +407,12 @@ class VersionSet {
 
   double MaxBytesForLevel(int level);
 
+  Status GetMetadataForFile(
+    uint64_t number, int *filelevel, FileMetaData *metadata);
+
+  void GetLiveFilesMetaData(
+    std::vector<LiveFileMetaData> *metadata);
+
  private:
   class Builder;
   struct ManifestWriter;

db/write_batch.cc

@@ -14,10 +14,10 @@
 //    len: varint32
 //    data: uint8[len]
 
-#include "leveldb/write_batch.h"
+#include "rocksdb/write_batch.h"
 
-#include "leveldb/options.h"
-#include "leveldb/statistics.h"
+#include "rocksdb/options.h"
+#include "rocksdb/statistics.h"
 #include "db/dbformat.h"
 #include "db/db_impl.h"
 #include "db/memtable.h"
@@ -48,6 +48,10 @@ void WriteBatch::Handler::LogData(const Slice& blob) {
   // them.
 }
 
+bool WriteBatch::Handler::Continue() {
+  return true;
+}
+
 void WriteBatch::Clear() {
   rep_.clear();
   rep_.resize(kHeader);
@@ -66,7 +70,7 @@ Status WriteBatch::Iterate(Handler* handler) const {
   input.remove_prefix(kHeader);
   Slice key, value, blob;
   int found = 0;
-  while (!input.empty()) {
+  while (!input.empty() && handler->Continue()) {
     char tag = input[0];
     input.remove_prefix(1);
     switch (tag) {

db/write_batch_internal.h

@@ -5,10 +5,10 @@
 #ifndef STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
 #define STORAGE_LEVELDB_DB_WRITE_BATCH_INTERNAL_H_
 
-#include "leveldb/types.h"
-#include "leveldb/write_batch.h"
-#include "leveldb/db.h"
-#include "leveldb/options.h"
+#include "rocksdb/types.h"
+#include "rocksdb/write_batch.h"
+#include "rocksdb/db.h"
+#include "rocksdb/options.h"
 
 namespace leveldb {
 

db/write_batch_test.cc

@@ -2,13 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 
 #include <memory>
-#include "db/skiplistrep.h"
 #include "db/memtable.h"
 #include "db/write_batch_internal.h"
-#include "leveldb/env.h"
+#include "rocksdb/env.h"
+#include "rocksdb/memtablerep.h"
 #include "util/logging.h"
 #include "util/testharness.h"
 
@@ -134,6 +134,24 @@ TEST(WriteBatchTest, Append) {
   ASSERT_EQ(4, b1.Count());
 }
 
+namespace {
+  struct TestHandler : public WriteBatch::Handler {
+    std::string seen;
+    virtual void Put(const Slice& key, const Slice& value) {
+      seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
+    }
+    virtual void Merge(const Slice& key, const Slice& value) {
+      seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
+    }
+    virtual void LogData(const Slice& blob) {
+      seen += "LogData(" + blob.ToString() + ")";
+    }
+    virtual void Delete(const Slice& key) {
+      seen += "Delete(" + key.ToString() + ")";
+    }
+  };
+}
+
 TEST(WriteBatchTest, Blob) {
   WriteBatch batch;
   batch.Put(Slice("k1"), Slice("v1"));
@@ -151,21 +169,7 @@ TEST(WriteBatchTest, Blob) {
             "Put(k3, v3)@2",
             PrintContents(&batch));
 
-  struct Handler : public WriteBatch::Handler {
-    std::string seen;
-    virtual void Put(const Slice& key, const Slice& value) {
-      seen += "Put(" + key.ToString() + ", " + value.ToString() + ")";
-    }
-    virtual void Merge(const Slice& key, const Slice& value) {
-      seen += "Merge(" + key.ToString() + ", " + value.ToString() + ")";
-    }
-    virtual void LogData(const Slice& blob) {
-      seen += "LogData(" + blob.ToString() + ")";
-    }
-    virtual void Delete(const Slice& key) {
-      seen += "Delete(" + key.ToString() + ")";
-    }
-  } handler;
+  TestHandler handler;
   batch.Iterate(&handler);
   ASSERT_EQ(
             "Put(k1, v1)"
@@ -178,6 +182,45 @@ TEST(WriteBatchTest, Blob) {
             handler.seen);
 }
 
+TEST(WriteBatchTest, Continue) {
+  WriteBatch batch;
+
+  struct Handler : public TestHandler {
+    int num_seen = 0;
+    virtual void Put(const Slice& key, const Slice& value) {
+      ++num_seen;
+      TestHandler::Put(key, value);
+    }
+    virtual void Merge(const Slice& key, const Slice& value) {
+      ++num_seen;
+      TestHandler::Merge(key, value);
+    }
+    virtual void LogData(const Slice& blob) {
+      ++num_seen;
+      TestHandler::LogData(blob);
+    }
+    virtual void Delete(const Slice& key) {
+      ++num_seen;
+      TestHandler::Delete(key);
+    }
+    virtual bool Continue() override {
+      return num_seen < 3;
+    }
+  } handler;
+
+  batch.Put(Slice("k1"), Slice("v1"));
+  batch.PutLogData(Slice("blob1"));
+  batch.Delete(Slice("k1"));
+  batch.PutLogData(Slice("blob2"));
+  batch.Merge(Slice("foo"), Slice("bar"));
+  batch.Iterate(&handler);
+  ASSERT_EQ(
+            "Put(k1, v1)"
+            "LogData(blob1)"
+            "Delete(k1)",
+            handler.seen);
+}
+
 }  // namespace leveldb
 
 int main(int argc, char** argv) {

hdfs/env_hdfs.h

@@ -25,8 +25,8 @@
 #include <sys/time.h>
 #include <time.h>
 #include <iostream>
-#include "leveldb/env.h"
-#include "leveldb/status.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
 
 #ifdef USE_HDFS
 #include "hdfs/hdfs.h"

helpers/memenv/memenv.cc

@@ -4,8 +4,8 @@
 
 #include "helpers/memenv/memenv.h"
 
-#include "leveldb/env.h"
-#include "leveldb/status.h"
+#include "rocksdb/env.h"
+#include "rocksdb/status.h"
 #include "port/port.h"
 #include "util/mutexlock.h"
 #include <map>

helpers/memenv/memenv.h

@@ -2,9 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
-#define STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
-
+#ifndef STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
+#define STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_
 namespace leveldb {
 
 class Env;
@@ -17,4 +16,4 @@ Env* NewMemEnv(Env* base_env);
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_HELPERS_MEMENV_MEMENV_H_
+#endif  // STORAGE_ROCKSDB_HELPERS_MEMENV_MEMENV_H_

helpers/memenv/memenv_test.cc

@@ -5,8 +5,8 @@
 #include "helpers/memenv/memenv.h"
 
 #include "db/db_impl.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
 #include "util/testharness.h"
 #include <memory>
 #include <string>

include/leveldb/memtablerep.h

@@ -1,88 +0,0 @@
-// This file contains the interface that must be implemented by any collection
-// to be used as the backing store for a MemTable. Such a collection must
-// satisfy the following properties:
-//  (1) It does not store duplicate items.
-//  (2) It uses MemTableRep::KeyComparator to compare items for iteration and
-//     equality.
-//  (3) It can be accessed concurrently by multiple readers but need not support
-//      concurrent writes.
-//  (4) Items are never deleted.
-// The liberal use of assertions is encouraged to enforce (1).
-
-#ifndef STORAGE_LEVELDB_DB_TABLE_H_
-#define STORAGE_LEVELDB_DB_TABLE_H_
-
-#include <memory>
-#include "leveldb/arena.h"
-
-namespace leveldb {
-
-class MemTableRep {
- public:
-  // KeyComparator(a, b) returns a negative value if a is less than b, 0 if they
-  // are equal, and a positive value if b is greater than a
-  class KeyComparator {
-  public:
-    virtual int operator()(const char* a, const char* b) const = 0;
-    virtual ~KeyComparator() { }
-  };
-
-  // Insert key into the collection. (The caller will pack key and value into a
-  // single buffer and pass that in as the parameter to Insert)
-  // REQUIRES: nothing that compares equal to key is currently in the
-  // collection.
-  virtual void Insert(const char* key) = 0;
-
-  // Returns true iff an entry that compares equal to key is in the collection.
-  virtual bool Contains(const char* key) const = 0;
-
-  virtual ~MemTableRep() { }
-
-  // Iteration over the contents of a skip collection
-  class Iterator {
-   public:
-    // Initialize an iterator over the specified collection.
-    // The returned iterator is not valid.
-    // explicit Iterator(const MemTableRep* collection);
-    virtual ~Iterator() { };
-
-    // Returns true iff the iterator is positioned at a valid node.
-    virtual bool Valid() const = 0;
-
-    // Returns the key at the current position.
-    // REQUIRES: Valid()
-    virtual const char* key() const = 0;
-
-    // Advances to the next position.
-    // REQUIRES: Valid()
-    virtual void Next() = 0;
-
-    // Advances to the previous position.
-    // REQUIRES: Valid()
-    virtual void Prev() = 0;
-
-    // Advance to the first entry with a key >= target
-    virtual void Seek(const char* target) = 0;
-
-    // Position at the first entry in collection.
-    // Final state of iterator is Valid() iff collection is not empty.
-    virtual void SeekToFirst() = 0;
-
-    // Position at the last entry in collection.
-    // Final state of iterator is Valid() iff collection is not empty.
-    virtual void SeekToLast() = 0;
-  };
-
-  virtual std::shared_ptr<Iterator> GetIterator() = 0;
-};
-
-class MemTableRepFactory {
- public:
-  virtual ~MemTableRepFactory() { };
-  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
-    MemTableRep::KeyComparator&, Arena* arena) = 0;
-};
-
-}
-
-#endif // STORAGE_LEVELDB_DB_TABLE_H_

include/leveldb/perf_context.h

@@ -1,24 +0,0 @@
-#ifndef STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
-#define STORAGE_LEVELDB_INCLUDE_PERF_CONTEXT_H
-
-#include <stdint.h>
-
-namespace leveldb {
-
-// A thread local context for gathering performance counter efficiently
-// and transparently.
-
-struct PerfContext {
-
-  void Reset(); // reset all performance counters to zero
-
-
-  uint64_t user_key_comparison_count; // total number of user key comparisons
-};
-
-extern __thread PerfContext perf_context;
-
-}
-
-
-#endif

include/rocksdb/arena.h

@@ -5,8 +5,11 @@
 // Arena class defines memory allocation methods. It's used by memtable and
 // skiplist.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_ARENA_H_
-#define STORAGE_LEVELDB_INCLUDE_ARENA_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_ARENA_H_
+#define STORAGE_ROCKSDB_INCLUDE_ARENA_H_
+
+#include <limits>
+#include <memory>
 
 namespace leveldb {
 
@@ -35,4 +38,4 @@ class Arena {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_ARENA_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_ARENA_H_

include/rocksdb/c.h

@@ -37,8 +37,8 @@
   (5) All of the pointer arguments must be non-NULL.
 */
 
-#ifndef STORAGE_LEVELDB_INCLUDE_C_H_
-#define STORAGE_LEVELDB_INCLUDE_C_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_C_H_
+#define STORAGE_ROCKSDB_INCLUDE_C_H_
 
 #ifdef __cplusplus
 extern "C" {
@@ -278,4 +278,4 @@ extern void leveldb_env_destroy(leveldb_env_t*);
 }  /* end extern "C" */
 #endif
 
-#endif  /* STORAGE_LEVELDB_INCLUDE_C_H_ */
+#endif  /* STORAGE_ROCKSDB_INCLUDE_C_H_ */

include/rocksdb/cache.h

@@ -15,12 +15,12 @@
 // they want something more sophisticated (like scan-resistance, a
 // custom eviction policy, variable cache sizing, etc.)
 
-#ifndef STORAGE_LEVELDB_INCLUDE_CACHE_H_
-#define STORAGE_LEVELDB_INCLUDE_CACHE_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_CACHE_H_
+#define STORAGE_ROCKSDB_INCLUDE_CACHE_H_
 
 #include <memory>
 #include <stdint.h>
-#include "leveldb/slice.h"
+#include "rocksdb/slice.h"
 
 namespace leveldb {
 
@@ -103,4 +103,4 @@ class Cache {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_UTIL_CACHE_H_
+#endif  // STORAGE_ROCKSDB_UTIL_CACHE_H_

include/rocksdb/compaction_filter.h

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
-#define STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
+#define STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_
 
 #include <string>
 
@@ -68,4 +68,4 @@ class DefaultCompactionFilterFactory : public CompactionFilterFactory {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_COMPACTION_FILTER_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_COMPACTION_FILTER_H_

include/rocksdb/comparator.h

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
+#define STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_
 
 #include <string>
 
@@ -60,4 +60,4 @@ extern const Comparator* BytewiseComparator();
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_COMPARATOR_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_COMPARATOR_H_

include/rocksdb/db.h

@@ -2,17 +2,17 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_DB_H_
-#define STORAGE_LEVELDB_INCLUDE_DB_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_DB_H_
+#define STORAGE_ROCKSDB_INCLUDE_DB_H_
 
 #include <stdint.h>
 #include <stdio.h>
 #include <memory>
 #include <vector>
-#include "leveldb/iterator.h"
-#include "leveldb/options.h"
-#include "leveldb/types.h"
-#include "leveldb/transaction_log.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/options.h"
+#include "rocksdb/types.h"
+#include "rocksdb/transaction_log.h"
 
 namespace leveldb {
 
@@ -28,6 +28,17 @@ struct WriteOptions;
 struct FlushOptions;
 class WriteBatch;
 
+// Metadata associated with each SST file.
+struct LiveFileMetaData {
+  std::string name;        // Name of the file
+  int level;               // Level at which this file resides.
+  size_t size;             // File size in bytes.
+  std::string smallestkey; // Smallest user defined key in the file.
+  std::string largestkey;  // Largest user defined key in the file.
+  SequenceNumber smallest_seqno; // smallest seqno in file
+  SequenceNumber largest_seqno;  // largest seqno in file
+};
+
 // Abstract handle to particular state of a DB.
 // A Snapshot is an immutable object and can therefore be safely
 // accessed from multiple threads without any external synchronization.
@@ -198,9 +209,10 @@ class DB {
   // after compaction is reduced, that level might not be appropriate for
   // hosting all the files. In this case, client could set reduce_level
   // to true, to move the files back to the minimum level capable of holding
-  // the data set.
+  // the data set or a given level (specified by non-negative target_level).
   virtual void CompactRange(const Slice* begin, const Slice* end,
-                            bool reduce_level = false) = 0;
+                            bool reduce_level = false,
+                            int target_level = -1) = 0;
 
   // Number of levels used for this DB.
   virtual int NumberLevels() = 0;
@@ -223,12 +235,15 @@ class DB {
   // Allow compactions to delete obselete files.
   virtual Status EnableFileDeletions() = 0;
 
+  // GetLiveFiles followed by GetSortedWalFiles can generate a lossless backup
+
+  // THIS METHOD IS DEPRECATED. Use the GetTableMetaData to get more
+  // detailed information on the live files.
   // Retrieve the list of all files in the database. The files are
-  // relative to the dbname and are not absolute paths. This list
-  // can be used to generate a backup. The valid size of the manifest
-  // file is returned in manifest_file_size. The manifest file is
-  // an ever growing file, but only the portion specified
-  // by manifest_file_size is valid for this snapshot.
+  // relative to the dbname and are not absolute paths. The valid size of the
+  // manifest file is returned in manifest_file_size. The manifest file is an
+  // ever growing file, but only the portion specified by manifest_file_size is
+  // valid for this snapshot.
   virtual Status GetLiveFiles(std::vector<std::string>&,
                               uint64_t* manifest_file_size) = 0;
 
@@ -237,7 +252,6 @@ class DB {
 
   // Delete wal files in files. These can be either live or archived.
   // Returns Status::OK if all files could be deleted, otherwise Status::IOError
-  // which contains information about files that could not be deleted.
   virtual Status DeleteWalFiles(const VectorLogPtr& files) = 0;
 
   // The sequence number of the most recent transaction.
@@ -256,6 +270,19 @@ class DB {
   // an update is read.
   virtual Status GetUpdatesSince(SequenceNumber seq_number,
                                  unique_ptr<TransactionLogIterator>* iter) = 0;
+
+  // Delete the file name from the db directory and update the internal
+  // state to reflect that.
+  virtual Status DeleteFile(std::string name) {
+    return Status::OK();
+  }
+
+  // Returns a list of all table files with their level, start key
+  // and end key
+  virtual void GetLiveFilesMetaData(
+    std::vector<LiveFileMetaData> *metadata) {
+  }
+
  private:
   // No copying allowed
   DB(const DB&);
@@ -274,4 +301,4 @@ Status RepairDB(const std::string& dbname, const Options& options);
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_DB_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_DB_H_

include/rocksdb/env.h

@@ -10,15 +10,15 @@
 // All Env implementations are safe for concurrent access from
 // multiple threads without any external synchronization.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_ENV_H_
-#define STORAGE_LEVELDB_INCLUDE_ENV_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_ENV_H_
+#define STORAGE_ROCKSDB_INCLUDE_ENV_H_
 
 #include <cstdarg>
 #include <string>
 #include <memory>
 #include <vector>
 #include <stdint.h>
-#include "leveldb/status.h"
+#include "rocksdb/status.h"
 
 namespace leveldb {
 
@@ -190,6 +190,13 @@ class Env {
   // useful for computing deltas of time.
   virtual uint64_t NowMicros() = 0;
 
+  // Returns the number of nano-seconds since some fixed point in time. Only
+  // useful for computing deltas of time in one run.
+  // Default implementation simply relies on NowMicros
+  virtual uint64_t NowNanos() {
+    return NowMicros() * 1000;
+  }
+
   // Sleep/delay the thread for the perscribed number of micro-seconds.
   virtual void SleepForMicroseconds(int micros) = 0;
 
@@ -531,4 +538,4 @@ class EnvWrapper : public Env {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_ENV_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_ENV_H_

include/rocksdb/filter_policy.h

@@ -13,8 +13,8 @@
 // Most people will want to use the builtin bloom filter support (see
 // NewBloomFilterPolicy() below).
 
-#ifndef STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
-#define STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
+#define STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_
 
 #include <string>
 
@@ -67,4 +67,4 @@ extern const FilterPolicy* NewBloomFilterPolicy(int bits_per_key);
 
 }
 
-#endif  // STORAGE_LEVELDB_INCLUDE_FILTER_POLICY_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_FILTER_POLICY_H_

include/rocksdb/iterator.h

@@ -12,11 +12,11 @@
 // non-const method, all threads accessing the same Iterator must use
 // external synchronization.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
+#define STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_
 
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
 
 namespace leveldb {
 
@@ -65,6 +65,8 @@ class Iterator {
   virtual Slice value() const = 0;
 
   // If an error has occurred, return it.  Else return an ok status.
+  // If non-blocking IO is requested and this operation cannot be
+  // satisfied without doing some IO, then this returns Status::Incomplete().
   virtual Status status() const = 0;
 
   // Clients are allowed to register function/arg1/arg2 triples that
@@ -97,4 +99,4 @@ extern Iterator* NewErrorIterator(const Status& status);
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_ITERATOR_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_ITERATOR_H_

include/rocksdb/ldb_tool.h

@@ -1,7 +1,7 @@
 // Copyright 2008-present Facebook. All Rights Reserved.
-#ifndef STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
-#define STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
-#include "leveldb/options.h"
+#ifndef STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
+#define STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H
+#include "rocksdb/options.h"
 
 namespace leveldb {
 
@@ -11,4 +11,4 @@ class LDBTool {
 };
 
 } // namespace leveldb
-#endif // STORAGE_LEVELDB_INCLUDE_LDB_TOOL_H
+#endif // STORAGE_ROCKSDB_INCLUDE_LDB_TOOL_H

include/rocksdb/memtablerep.h

@@ -0,0 +1,241 @@
+// This file contains the interface that must be implemented by any collection
+// to be used as the backing store for a MemTable. Such a collection must
+// satisfy the following properties:
+//  (1) It does not store duplicate items.
+//  (2) It uses MemTableRep::KeyComparator to compare items for iteration and
+//     equality.
+//  (3) It can be accessed concurrently by multiple readers and can support
+//     during reads. However, it needn't support multiple concurrent writes.
+//  (4) Items are never deleted.
+// The liberal use of assertions is encouraged to enforce (1).
+//
+// The factory will be passed an Arena object when a new MemTableRep is
+// requested. The API for this object is in leveldb/arena.h.
+//
+// Users can implement their own memtable representations. We include four
+// types built in:
+//  - SkipListRep: This is the default; it is backed by a skip list.
+//  - TransformRep: This is backed by an std::unordered_map<Slice,
+// std::set>. On construction, they are given a SliceTransform object. This
+// object is applied to the user key of stored items which indexes into the
+// unordered map to yield a set containing all records that share the same user
+// key under the transform function.
+//  - UnsortedRep: A subclass of TransformRep where the transform function is
+// the identity function. Optimized for point lookups.
+//  - PrefixHashRep: A subclass of TransformRep where the transform function is
+// a fixed-size prefix extractor. If you use PrefixHashRepFactory, the transform
+// must be identical to options.prefix_extractor, otherwise it will be discarded
+// and the default will be used. It is optimized for ranged scans over a
+// prefix.
+//  - VectorRep: This is backed by an unordered std::vector. On iteration, the
+// vector is sorted. It is intelligent about sorting; once the MarkReadOnly()
+// has been called, the vector will only be sorted once. It is optimized for
+// random-write-heavy workloads.
+//
+// The last four implementations are designed for situations in which
+// iteration over the entire collection is rare since doing so requires all the
+// keys to be copied into a sorted data structure.
+
+#ifndef STORAGE_ROCKSDB_DB_MEMTABLEREP_H_
+#define STORAGE_ROCKSDB_DB_MEMTABLEREP_H_
+
+#include <memory>
+#include "rocksdb/arena.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
+
+namespace leveldb {
+
+class MemTableRep {
+ public:
+  // KeyComparator provides a means to compare keys, which are internal keys
+  // concatenated with values.
+  class KeyComparator {
+   public:
+    // Compare a and b. Return a negative value if a is less than b, 0 if they
+    // are equal, and a positive value if a is greater than b
+    virtual int operator()(const char* a, const char* b) const = 0;
+
+    virtual ~KeyComparator() { }
+  };
+
+  // Insert key into the collection. (The caller will pack key and value into a
+  // single buffer and pass that in as the parameter to Insert)
+  // REQUIRES: nothing that compares equal to key is currently in the
+  // collection.
+  virtual void Insert(const char* key) = 0;
+
+  // Returns true iff an entry that compares equal to key is in the collection.
+  virtual bool Contains(const char* key) const = 0;
+
+  // Notify this table rep that it will no longer be added to. By default, does
+  // nothing.
+  virtual void MarkReadOnly() { }
+
+  // Report an approximation of how much memory has been used other than memory
+  // that was allocated through the arena.
+  virtual size_t ApproximateMemoryUsage() = 0;
+
+  virtual ~MemTableRep() { }
+
+  // Iteration over the contents of a skip collection
+  class Iterator {
+   public:
+    // Initialize an iterator over the specified collection.
+    // The returned iterator is not valid.
+    // explicit Iterator(const MemTableRep* collection);
+    virtual ~Iterator() { };
+
+    // Returns true iff the iterator is positioned at a valid node.
+    virtual bool Valid() const = 0;
+
+    // Returns the key at the current position.
+    // REQUIRES: Valid()
+    virtual const char* key() const = 0;
+
+    // Advances to the next position.
+    // REQUIRES: Valid()
+    virtual void Next() = 0;
+
+    // Advances to the previous position.
+    // REQUIRES: Valid()
+    virtual void Prev() = 0;
+
+    // Advance to the first entry with a key >= target
+    virtual void Seek(const char* target) = 0;
+
+    // Position at the first entry in collection.
+    // Final state of iterator is Valid() iff collection is not empty.
+    virtual void SeekToFirst() = 0;
+
+    // Position at the last entry in collection.
+    // Final state of iterator is Valid() iff collection is not empty.
+    virtual void SeekToLast() = 0;
+  };
+
+  // Return an iterator over the keys in this representation.
+  virtual std::shared_ptr<Iterator> GetIterator() = 0;
+
+  // Return an iterator over at least the keys with the specified user key. The
+  // iterator may also allow access to other keys, but doesn't have to. Default:
+  // GetIterator().
+  virtual std::shared_ptr<Iterator> GetIterator(const Slice& user_key) {
+    return GetIterator();
+  }
+
+  // Return an iterator over at least the keys with the specified prefix. The
+  // iterator may also allow access to other keys, but doesn't have to. Default:
+  // GetIterator().
+  virtual std::shared_ptr<Iterator> GetPrefixIterator(const Slice& prefix) {
+    return GetIterator();
+  }
+
+ protected:
+  // When *key is an internal key concatenated with the value, returns the
+  // user key.
+  virtual Slice UserKey(const char* key) const;
+};
+
+// This is the base class for all factories that are used by RocksDB to create
+// new MemTableRep objects
+class MemTableRepFactory {
+ public:
+  virtual ~MemTableRepFactory() { };
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) = 0;
+};
+
+// This creates MemTableReps that are backed by an std::vector. On iteration,
+// the vector is sorted. This is useful for workloads where iteration is very
+// rare and writes are generally not issued after reads begin.
+//
+// Parameters:
+//   count: Passed to the constructor of the underlying std::vector of each
+//     VectorRep. On initialization, the underlying array will be at least count
+//     size.
+class VectorRepFactory : public MemTableRepFactory {
+  const size_t count_;
+public:
+  explicit VectorRepFactory(size_t count = 0) : count_(count) { }
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) override;
+};
+
+// This uses a skip list to store keys. It is the default.
+class SkipListFactory : public MemTableRepFactory {
+public:
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) override;
+};
+
+// TransformReps are backed by an unordered map of buffers to buckets. When
+// looking up a key, the user key is extracted and a user-supplied transform
+// function (see leveldb/slice_transform.h) is applied to get the key into the
+// unordered map. This allows the user to bin user keys based on arbitrary
+// criteria. Two example implementations are UnsortedRepFactory and
+// PrefixHashRepFactory.
+//
+// Iteration over the entire collection is implemented by dumping all the keys
+// into an std::set. Thus, these data structures are best used when iteration
+// over the entire collection is rare.
+//
+// Parameters:
+//   transform: The SliceTransform to bucket user keys on. TransformRepFactory
+//     owns the pointer.
+//   bucket_count: Passed to the constructor of the underlying
+//     std::unordered_map of each TransformRep. On initialization, the
+//     underlying array will be at least bucket_count size.
+//   num_locks: Number of read-write locks to have for the rep. Each bucket is
+//     hashed onto a read-write lock which controls access to that lock. More
+//     locks means finer-grained concurrency but more memory overhead.
+class TransformRepFactory : public MemTableRepFactory {
+ public:
+  explicit TransformRepFactory(const SliceTransform* transform,
+    size_t bucket_count, size_t num_locks = 1000)
+    : transform_(transform),
+      bucket_count_(bucket_count),
+      num_locks_(num_locks) { }
+
+  virtual ~TransformRepFactory() { delete transform_; }
+
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) override;
+
+  const SliceTransform* GetTransform() { return transform_; }
+
+ protected:
+  const SliceTransform* transform_;
+  const size_t bucket_count_;
+  const size_t num_locks_;
+};
+
+// UnsortedReps bin user keys based on an identity function transform -- that
+// is, transform(key) = key. This optimizes for point look-ups.
+//
+// Parameters: See TransformRepFactory.
+class UnsortedRepFactory : public TransformRepFactory {
+public:
+  explicit UnsortedRepFactory(size_t bucket_count = 0, size_t num_locks = 1000)
+    : TransformRepFactory(NewNoopTransform(),
+                          bucket_count,
+                          num_locks) { }
+};
+
+// PrefixHashReps bin user keys based on a fixed-size prefix. This optimizes for
+// short ranged scans over a given prefix.
+//
+// Parameters: See TransformRepFactory.
+class PrefixHashRepFactory : public TransformRepFactory {
+public:
+  explicit PrefixHashRepFactory(const SliceTransform* prefix_extractor,
+    size_t bucket_count = 0, size_t num_locks = 1000)
+    : TransformRepFactory(prefix_extractor, bucket_count, num_locks)
+    { }
+
+  virtual std::shared_ptr<MemTableRep> CreateMemTableRep(
+    MemTableRep::KeyComparator&, Arena*) override;
+};
+
+}
+
+#endif // STORAGE_ROCKSDB_DB_MEMTABLEREP_H_

include/rocksdb/merge_operator.h

@@ -2,12 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
+#define STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_
 
 #include <string>
 #include <deque>
-#include "leveldb/slice.h"  // TODO: Remove this when migration is done;
+#include "rocksdb/slice.h"  // TODO: Remove this when migration is done;
 
 namespace leveldb {
 
@@ -144,4 +144,4 @@ class AssociativeMergeOperator : public MergeOperator {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_MERGE_OPERATOR_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_MERGE_OPERATOR_H_

include/rocksdb/options.h

@@ -2,20 +2,19 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
-#define STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
+#define STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_
 
 #include <stddef.h>
 #include <string>
 #include <memory>
 #include <vector>
 #include <stdint.h>
-#include "leveldb/slice.h"
-#include "leveldb/statistics.h"
-#include "leveldb/universal_compaction.h"
-#include "leveldb/memtablerep.h"
-
-#include "leveldb/slice_transform.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/statistics.h"
+#include "rocksdb/universal_compaction.h"
+#include "rocksdb/memtablerep.h"
+#include "rocksdb/slice_transform.h"
 
 namespace leveldb {
 
@@ -533,6 +532,13 @@ struct Options {
   // Default: false
   bool filter_deletes;
 
+  // An iteration->Next() sequentially skips over keys with the same
+  // user-key unless this option is set. This number specifies the number
+  // of keys (with the same userkey) that will be sequentially
+  // skipped before a reseek is issued.
+  // Default: 8
+  uint64_t max_sequential_skip_in_iterations;
+
   // This is a factory that provides MemTableRep objects.
   // Default: a factory that provides a skip-list-based implementation of
   // MemTableRep.
@@ -544,6 +550,18 @@ struct Options {
   std::shared_ptr<CompactionFilterFactory> compaction_filter_factory;
 };
 
+//
+// An application can issue a read request (via Get/Iterators) and specify
+// if that read should process data that ALREADY resides on a specified cache
+// level. For example, if an application specifies kBlockCacheTier then the
+// Get call will process data that is already processed in the memtable or
+// the block cache. It will not page in data from the OS cache or data that
+// resides in storage.
+enum ReadTier {
+  kReadAllTier    = 0x0, // data in memtable, block cache, OS cache or storage
+  kBlockCacheTier = 0x1  // data in memtable or block cache
+};
+
 // Options that control read operations
 struct ReadOptions {
   // If true, all data read from underlying storage will be
@@ -576,15 +594,23 @@ struct ReadOptions {
   // Default: nullptr
   const Slice* prefix;
 
+  // Specify if this read request should process data that ALREADY
+  // resides on a particular cache. If the required data is not
+  // found at the specified cache, then Status::Incomplete is returned.
+  // Default: kReadAllTier
+  ReadTier read_tier;
+
   ReadOptions()
       : verify_checksums(false),
         fill_cache(true),
         snapshot(nullptr),
-        prefix(nullptr) {
+        prefix(nullptr),
+        read_tier(kReadAllTier) {
   }
   ReadOptions(bool cksum, bool cache) :
               verify_checksums(cksum), fill_cache(cache),
-              snapshot(nullptr), prefix(nullptr) {
+              snapshot(nullptr), prefix(nullptr),
+              read_tier(kReadAllTier) {
   }
 };
 
@@ -631,4 +657,4 @@ struct FlushOptions {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_OPTIONS_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_OPTIONS_H_

include/rocksdb/perf_context.h

@@ -0,0 +1,38 @@
+#ifndef STORAGE_ROCKSDB_INCLUDE_PERF_CONTEXT_H
+#define STORAGE_ROCKSDB_INCLUDE_PERF_CONTEXT_H
+
+#include <stdint.h>
+
+namespace leveldb {
+
+enum PerfLevel {
+  kDisable        = 0,  // disable perf stats
+  kEnableCount    = 1,  // enable only count stats
+  kEnableTime     = 2   // enable time stats too
+};
+
+// set the perf stats level
+void SetPerfLevel(PerfLevel level);
+
+// A thread local context for gathering performance counter efficiently
+// and transparently.
+
+struct PerfContext {
+
+  void Reset(); // reset all performance counters to zero
+
+  uint64_t user_key_comparison_count; // total number of user key comparisons
+  uint64_t block_cache_hit_count;
+  uint64_t block_read_count;
+  uint64_t block_read_byte;
+  uint64_t block_read_time;
+  uint64_t block_checksum_time;
+  uint64_t block_decompress_time;
+};
+
+extern __thread PerfContext perf_context;
+
+}
+
+
+#endif

include/rocksdb/slice.h

@@ -12,8 +12,8 @@
 // non-const method, all threads accessing the same Slice must use
 // external synchronization.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_H_
-#define STORAGE_LEVELDB_INCLUDE_SLICE_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_SLICE_H_
+#define STORAGE_ROCKSDB_INCLUDE_SLICE_H_
 
 #include <assert.h>
 #include <stddef.h>
@@ -31,9 +31,11 @@ class Slice {
   Slice(const char* d, size_t n) : data_(d), size_(n) { }
 
   // Create a slice that refers to the contents of "s"
+  /* implicit */
   Slice(const std::string& s) : data_(s.data()), size_(s.size()) { }
 
   // Create a slice that refers to s[0,strlen(s)-1]
+  /* implicit */
   Slice(const char* s) : data_(s), size_(strlen(s)) { }
 
   // Return a pointer to the beginning of the referenced data
@@ -117,5 +119,4 @@ inline int Slice::compare(const Slice& b) const {
 
 }  // namespace leveldb
 
-
-#endif  // STORAGE_LEVELDB_INCLUDE_SLICE_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_SLICE_H_

include/rocksdb/slice_transform.h

@@ -8,8 +8,8 @@
 // define InDomain and InRange to determine which slices are in either
 // of these sets respectively.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
-#define STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_
+#define STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_
 
 #include <string>
 
@@ -36,6 +36,8 @@ class SliceTransform {
 
 extern const SliceTransform* NewFixedPrefixTransform(size_t prefix_len);
 
+extern const SliceTransform* NewNoopTransform();
+
 }
 
-#endif  // STORAGE_LEVELDB_INCLUDE_SLICE_TRANSFORM_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_SLICE_TRANSFORM_H_

include/rocksdb/statistics.h

@@ -2,8 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
-#define STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_
+#define STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_
 
 #include <atomic>
 #include <cassert>
@@ -58,13 +58,27 @@ enum Tickers {
   NUMBER_MULTIGET_KEYS_READ = 19,
   NUMBER_MULTIGET_BYTES_READ = 20,
 
+  // Number of deletes records that were not required to be
+  // written to storage because key does not exist
   NUMBER_FILTERED_DELETES = 21,
   NUMBER_MERGE_FAILURES = 22,
   SEQUENCE_NUMBER = 23,
 
-  TICKER_ENUM_MAX = 24
+  // number of times bloom was checked before creating iterator on a
+  // file, and the number of times the check was useful in avoiding
+  // iterator creation (and thus likely IOPs).
+  BLOOM_FILTER_PREFIX_CHECKED = 24,
+  BLOOM_FILTER_PREFIX_USEFUL = 25,
+
+  // Number of times we had to reseek inside an iteration to skip
+  // over large number of keys with same userkey.
+  NUMBER_OF_RESEEKS_IN_ITERATION = 26,
+
+  TICKER_ENUM_MAX = 27
 };
 
+// The order of items listed in  Tickers should be the same as
+// the order listed in TickersNameMap
 const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
   { BLOCK_CACHE_MISS, "rocksdb.block.cache.miss" },
   { BLOCK_CACHE_HIT, "rocksdb.block.cache.hit" },
@@ -89,7 +103,10 @@ const std::vector<std::pair<Tickers, std::string>> TickersNameMap = {
   { NUMBER_MULTIGET_BYTES_READ, "rocksdb.number.multiget.bytes.read" },
   { NUMBER_FILTERED_DELETES, "rocksdb.number.deletes.filtered" },
   { NUMBER_MERGE_FAILURES, "rocksdb.number.merge.failures" },
-  { SEQUENCE_NUMBER, "rocksdb.sequence.number" }
+  { SEQUENCE_NUMBER, "rocksdb.sequence.number" },
+  { BLOOM_FILTER_PREFIX_CHECKED, "rocksdb.bloom.filter.prefix.checked" },
+  { BLOOM_FILTER_PREFIX_USEFUL, "rocksdb.bloom.filter.prefix.useful" },
+  { NUMBER_OF_RESEEKS_IN_ITERATION, "rocksdb.number.reseeks.iteration" }
 };
 
 /**
@@ -236,4 +253,4 @@ inline void SetTickerCount(std::shared_ptr<Statistics> statistics,
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_STATISTICS_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_STATISTICS_H_

include/rocksdb/status.h

@@ -10,11 +10,11 @@
 // non-const method, all threads accessing the same Status must use
 // external synchronization.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_STATUS_H_
-#define STORAGE_LEVELDB_INCLUDE_STATUS_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_STATUS_H_
+#define STORAGE_ROCKSDB_INCLUDE_STATUS_H_
 
 #include <string>
-#include "leveldb/slice.h"
+#include "rocksdb/slice.h"
 
 namespace leveldb {
 
@@ -50,6 +50,9 @@ class Status {
   static Status MergeInProgress(const Slice& msg, const Slice& msg2 = Slice()) {
     return Status(kMergeInProgress, msg, msg2);
   }
+  static Status Incomplete(const Slice& msg, const Slice& msg2 = Slice()) {
+    return Status(kIncomplete, msg, msg2);
+  }
 
   // Returns true iff the status indicates success.
   bool ok() const { return (state_ == nullptr); }
@@ -72,6 +75,9 @@ class Status {
   // Returns true iff the status indicates an MergeInProgress.
   bool IsMergeInProgress() const { return code() == kMergeInProgress; }
 
+  // Returns true iff the status indicates Incomplete
+  bool IsIncomplete() const { return code() == kIncomplete; }
+
   // Return a string representation of this status suitable for printing.
   // Returns the string "OK" for success.
   std::string ToString() const;
@@ -91,7 +97,8 @@ class Status {
     kNotSupported = 3,
     kInvalidArgument = 4,
     kIOError = 5,
-    kMergeInProgress = 6
+    kMergeInProgress = 6,
+    kIncomplete = 7
   };
 
   Code code() const {
@@ -116,4 +123,4 @@ inline void Status::operator=(const Status& s) {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_STATUS_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_STATUS_H_

include/rocksdb/table_builder.h

@@ -10,12 +10,12 @@
 // non-const method, all threads accessing the same TableBuilder must use
 // external synchronization.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
-#define STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_TABLE_BUILDER_H_
+#define STORAGE_ROCKSDB_INCLUDE_TABLE_BUILDER_H_
 
 #include <stdint.h>
-#include "leveldb/options.h"
-#include "leveldb/status.h"
+#include "rocksdb/options.h"
+#include "rocksdb/status.h"
 
 namespace leveldb {
 
@@ -92,4 +92,4 @@ class TableBuilder {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_TABLE_BUILDER_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_TABLE_BUILDER_H_

include/rocksdb/transaction_log.h

@@ -1,10 +1,10 @@
 // Copyright 2008-present Facebook. All Rights Reserved.
-#ifndef STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
-#define STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
+#define STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
 
-#include "leveldb/status.h"
-#include "leveldb/types.h"
-#include "leveldb/write_batch.h"
+#include "rocksdb/status.h"
+#include "rocksdb/types.h"
+#include "rocksdb/write_batch.h"
 
 namespace leveldb {
 
@@ -27,8 +27,11 @@ class LogFile {
   LogFile() {}
   virtual ~LogFile() {}
 
-  // Returns log file's name excluding the db path
-  virtual std::string Filename() const = 0;
+  // Returns log file's pathname relative to the main db dir
+  // Eg. For a live-log-file = /000003.log
+  //     For an archived-log-file = /archive/000003.log
+  virtual std::string PathName() const = 0;
+
 
   // Primary identifier for log file.
   // This is directly proportional to creation time of the log file
@@ -75,4 +78,4 @@ class TransactionLogIterator {
 };
 } //  namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_TRANSACTION_LOG_ITERATOR_H_

include/rocksdb/types.h

@@ -1,5 +1,5 @@
-#ifndef STORAGE_LEVELDB_INCLUDE_TYPES_H_
-#define STORAGE_LEVELDB_INCLUDE_TYPES_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_TYPES_H_
+#define STORAGE_ROCKSDB_INCLUDE_TYPES_H_
 
 #include <stdint.h>
 
@@ -11,4 +11,4 @@ namespace leveldb {
 typedef uint64_t SequenceNumber;
 
 }  //  namespace leveldb
-#endif //  STORAGE_LEVELDB_INCLUDE_TYPES_H_
+#endif //  STORAGE_ROCKSDB_INCLUDE_TYPES_H_

include/rocksdb/universal_compaction.h

@@ -11,8 +11,8 @@
 #include <vector>
 #include <stdint.h>
 #include <climits>
-#include "leveldb/slice.h"
-#include "leveldb/statistics.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/statistics.h"
 
 namespace leveldb {
 

include/rocksdb/write_batch.h

@@ -18,11 +18,11 @@
 // non-const method, all threads accessing the same WriteBatch must use
 // external synchronization.
 
-#ifndef STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
-#define STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
+#ifndef STORAGE_ROCKSDB_INCLUDE_WRITE_BATCH_H_
+#define STORAGE_ROCKSDB_INCLUDE_WRITE_BATCH_H_
 
 #include <string>
-#include "leveldb/status.h"
+#include "rocksdb/status.h"
 
 namespace leveldb {
 
@@ -43,12 +43,13 @@ class WriteBatch {
   // If the database contains a mapping for "key", erase it.  Else do nothing.
   void Delete(const Slice& key);
 
-  // Append a blob of arbitrary to the records in this batch. The blob will be
-  // stored in the transaction log but not in any other file. In particular, it
-  // will not be persisted to the SST files. When iterating over this
+  // Append a blob of arbitrary size to the records in this batch. The blob will
+  // be stored in the transaction log but not in any other file. In particular,
+  // it will not be persisted to the SST files. When iterating over this
   // WriteBatch, WriteBatch::Handler::LogData will be called with the contents
   // of the blob as it is encountered. Blobs, puts, deletes, and merges will be
-  // encountered in the same order in thich they were inserted.
+  // encountered in the same order in thich they were inserted. The blob will
+  // NOT consume sequence number(s) and will NOT increase the count of the batch
   //
   // Example application: add timestamps to the transaction log for use in
   // replication.
@@ -69,6 +70,10 @@ class WriteBatch {
     // The default implementation of LogData does nothing.
     virtual void LogData(const Slice& blob);
     virtual void Delete(const Slice& key) = 0;
+    // Continue is called by WriteBatch::Iterate. If it returns false,
+    // iteration is halted. Otherwise, it continues iterating. The default
+    // implementation always returns true.
+    virtual bool Continue();
   };
   Status Iterate(Handler* handler) const;
 
@@ -91,4 +96,4 @@ class WriteBatch {
 
 }  // namespace leveldb
 
-#endif  // STORAGE_LEVELDB_INCLUDE_WRITE_BATCH_H_
+#endif  // STORAGE_ROCKSDB_INCLUDE_WRITE_BATCH_H_

include/utilities/stackable_db.h

@@ -5,7 +5,7 @@
 #ifndef LEVELDB_INCLUDE_UTILITIES_STACKABLE_DB_H_
 #define LEVELDB_INCLUDE_UTILITIES_STACKABLE_DB_H_
 
-#include "leveldb/db.h"
+#include "rocksdb/db.h"
 
 namespace leveldb {
 
@@ -20,6 +20,8 @@ class StackableDB : public DB {
   }
 
   // convert a DB to StackableDB
+  // TODO: This function does not work yet. Passing nullptr to StackableDB in
+  //       NewStackableDB's constructor will cause segfault on object's usage
   static StackableDB* DBToStackableDB(DB* db) {
     class NewStackableDB : public StackableDB {
      public:
@@ -102,8 +104,9 @@ class StackableDB : public DB {
   }
 
   virtual void CompactRange(const Slice* begin, const Slice* end,
-                            bool reduce_level = false) override {
-    return sdb_->CompactRange(begin, end, reduce_level);
+                            bool reduce_level = false,
+                            int target_level = -1) override {
+    return sdb_->CompactRange(begin, end, reduce_level, target_level);
   }
 
   virtual int NumberLevels() override {
@@ -143,9 +146,8 @@ class StackableDB : public DB {
     return sdb_->GetSortedWalFiles(files);
   }
 
-  virtual Status DeleteWalFiles(const VectorLogPtr& files)
-    override{
-      return sdb_->DeleteWalFiles(files);
+  virtual Status DeleteWalFiles(const VectorLogPtr& files) override {
+    return sdb_->DeleteWalFiles(files);
   }
 
   virtual Status GetUpdatesSince(SequenceNumber seq_number,

port/port_posix.h

@@ -44,7 +44,7 @@
 #include <stdint.h>
 #include <string>
 #include <string.h>
-#include "leveldb/options.h"
+#include "rocksdb/options.h"
 #include "port/atomic_pointer.h"
 
 #ifndef PLATFORM_IS_LITTLE_ENDIAN

table/block.cc

@@ -8,7 +8,7 @@
 
 #include <vector>
 #include <algorithm>
-#include "leveldb/comparator.h"
+#include "rocksdb/comparator.h"
 #include "table/format.h"
 #include "util/coding.h"
 #include "util/logging.h"

table/block.h

@@ -7,7 +7,7 @@
 
 #include <stddef.h>
 #include <stdint.h>
-#include "leveldb/iterator.h"
+#include "rocksdb/iterator.h"
 
 namespace leveldb {
 

table/block_builder.cc

@@ -30,8 +30,8 @@
 
 #include <algorithm>
 #include <assert.h>
-#include "leveldb/comparator.h"
-#include "leveldb/table_builder.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/table_builder.h"
 #include "util/coding.h"
 
 namespace leveldb {

table/block_builder.h

@@ -8,7 +8,7 @@
 #include <vector>
 
 #include <stdint.h>
-#include "leveldb/slice.h"
+#include "rocksdb/slice.h"
 
 namespace leveldb {
 

table/block_test.cc

@@ -5,10 +5,10 @@
 #include "db/dbformat.h"
 #include "db/memtable.h"
 #include "db/write_batch_internal.h"
-#include "leveldb/db.h"
-#include "leveldb/env.h"
-#include "leveldb/iterator.h"
-#include "leveldb/table_builder.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/table_builder.h"
 #include "table/block.h"
 #include "table/block_builder.h"
 #include "table/table.h"

table/filter_block.cc

@@ -5,7 +5,7 @@
 #include "table/filter_block.h"
 
 #include "db/dbformat.h"
-#include "leveldb/filter_policy.h"
+#include "rocksdb/filter_policy.h"
 #include "util/coding.h"
 
 namespace leveldb {
@@ -45,6 +45,7 @@ bool FilterBlockBuilder::SamePrefix(const Slice &key1,
 }
 
 void FilterBlockBuilder::AddKey(const Slice& key) {
+  // get slice for most recently added entry
   Slice prev;
   if (start_.size() > 0) {
     size_t prev_start = start_[start_.size() - 1];
@@ -53,17 +54,23 @@ void FilterBlockBuilder::AddKey(const Slice& key) {
     prev = Slice(base, length);
   }
 
+  // add key to filter if needed
   if (whole_key_filtering_) {
     start_.push_back(entries_.size());
     entries_.append(key.data(), key.size());
   }
 
-  if (prefix_extractor_ && prefix_extractor_->InDomain(key)) {
+  // add prefix to filter if needed
+  if (prefix_extractor_ && prefix_extractor_->InDomain(ExtractUserKey(key))) {
+    // If prefix_extractor_, this filter_block layer assumes we only
+    // operate on internal keys.
+    Slice user_key = ExtractUserKey(key);
     // this assumes prefix(prefix(key)) == prefix(key), as the last
     // entry in entries_ may be either a key or prefix, and we use
     // prefix(last entry) to get the prefix of the last key.
-    if (prev.size() == 0 || ! SamePrefix(key, prev)) {
-      Slice prefix = prefix_extractor_->Transform(key);
+    if (prev.size() == 0 ||
+        !SamePrefix(user_key, ExtractUserKey(prev))) {
+      Slice prefix = prefix_extractor_->Transform(user_key);
       InternalKey internal_prefix_tmp(prefix, 0, kTypeValue);
       Slice internal_prefix = internal_prefix_tmp.Encode();
       assert(comparator_->Compare(internal_prefix, key) <= 0);

table/filter_block.h

@@ -13,9 +13,9 @@
 #include <stdint.h>
 #include <string>
 #include <vector>
-#include "leveldb/options.h"
-#include "leveldb/slice.h"
-#include "leveldb/slice_transform.h"
+#include "rocksdb/options.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/slice_transform.h"
 #include "util/hash.h"
 
 namespace leveldb {

table/filter_block_test.cc

@@ -4,7 +4,7 @@
 
 #include "table/filter_block.h"
 
-#include "leveldb/filter_policy.h"
+#include "rocksdb/filter_policy.h"
 #include "util/coding.h"
 #include "util/hash.h"
 #include "util/logging.h"

table/format.cc

@@ -4,11 +4,12 @@
 
 #include "table/format.h"
 
-#include "leveldb/env.h"
 #include "port/port.h"
+#include "rocksdb/env.h"
 #include "table/block.h"
 #include "util/coding.h"
 #include "util/crc32c.h"
+#include "util/perf_context_imp.h"
 
 namespace leveldb {
 
@@ -69,7 +70,8 @@ Status Footer::DecodeFrom(Slice* input) {
 Status ReadBlockContents(RandomAccessFile* file,
                          const ReadOptions& options,
                          const BlockHandle& handle,
-                         BlockContents* result) {
+                         BlockContents* result,
+                         Env* env) {
   result->data = Slice();
   result->cachable = false;
   result->heap_allocated = false;
@@ -79,7 +81,14 @@ Status ReadBlockContents(RandomAccessFile* file,
   size_t n = static_cast<size_t>(handle.size());
   char* buf = new char[n + kBlockTrailerSize];
   Slice contents;
+
+  StopWatchNano timer(env);
+  StartPerfTimer(&timer);
   Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf);
+  BumpPerfCount(&perf_context.block_read_count);
+  BumpPerfCount(&perf_context.block_read_byte, n + kBlockTrailerSize);
+  BumpPerfTime(&perf_context.block_read_time, &timer);
+
   if (!s.ok()) {
     delete[] buf;
     return s;
@@ -99,6 +108,7 @@ Status ReadBlockContents(RandomAccessFile* file,
       s = Status::Corruption("block checksum mismatch");
       return s;
     }
+    BumpPerfTime(&perf_context.block_checksum_time, &timer);
   }
 
   char* ubuf = nullptr;
@@ -172,6 +182,8 @@ Status ReadBlockContents(RandomAccessFile* file,
       return Status::Corruption("bad block type");
   }
 
+  BumpPerfTime(&perf_context.block_decompress_time, &timer);
+
   return Status::OK();
 }
 

table/format.h

@@ -7,9 +7,9 @@
 
 #include <string>
 #include <stdint.h>
-#include "leveldb/slice.h"
-#include "leveldb/status.h"
-#include "leveldb/table_builder.h"
+#include "rocksdb/slice.h"
+#include "rocksdb/status.h"
+#include "rocksdb/table_builder.h"
 
 namespace leveldb {
 
@@ -94,7 +94,8 @@ struct BlockContents {
 extern Status ReadBlockContents(RandomAccessFile* file,
                                 const ReadOptions& options,
                                 const BlockHandle& handle,
-                                BlockContents* result);
+                                BlockContents* result,
+                                Env* env);
 
 // Implementation details follow.  Clients should ignore,
 

table/iter_heap.h

@@ -4,7 +4,7 @@
 
 #include <queue>
 
-#include "leveldb/comparator.h"
+#include "rocksdb/comparator.h"
 #include "table/iterator_wrapper.h"
 
 namespace leveldb {

table/iterator.cc

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "leveldb/iterator.h"
+#include "rocksdb/iterator.h"
 
 namespace leveldb {
 

table/merger.cc

@@ -4,11 +4,13 @@
 
 #include "table/merger.h"
 
-#include "leveldb/comparator.h"
-#include "leveldb/iterator.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/iterator.h"
 #include "table/iter_heap.h"
 #include "table/iterator_wrapper.h"
 
+#include <vector>
+
 namespace leveldb {
 
 namespace {
@@ -17,8 +19,7 @@ class MergingIterator : public Iterator {
  public:
   MergingIterator(const Comparator* comparator, Iterator** children, int n)
       : comparator_(comparator),
-        children_(new IteratorWrapper[n]),
-        n_(n),
+        children_(n),
         current_(nullptr),
         direction_(kForward),
         maxHeap_(NewMaxIterHeap(comparator_)),
@@ -26,16 +27,14 @@ class MergingIterator : public Iterator {
     for (int i = 0; i < n; i++) {
       children_[i].Set(children[i]);
     }
-    for (int i = 0; i < n; ++i) {
-      if (children_[i].Valid()) {
-        minHeap_.push(&children_[i]);
+    for (auto& child : children_) {
+      if (child.Valid()) {
+        minHeap_.push(&child);
       }
     }
   }
 
-  virtual ~MergingIterator() {
-    delete[] children_;
-  }
+  virtual ~MergingIterator() { }
 
   virtual bool Valid() const {
     return (current_ != nullptr);
@@ -43,10 +42,10 @@ class MergingIterator : public Iterator {
 
   virtual void SeekToFirst() {
     ClearHeaps();
-    for (int i = 0; i < n_; i++) {
-      children_[i].SeekToFirst();
-      if (children_[i].Valid()) {
-        minHeap_.push(&children_[i]);
+    for (auto& child : children_) {
+      child.SeekToFirst();
+      if (child.Valid()) {
+        minHeap_.push(&child);
       }
     }
     FindSmallest();
@@ -55,10 +54,10 @@ class MergingIterator : public Iterator {
 
   virtual void SeekToLast() {
     ClearHeaps();
-    for (int i = 0; i < n_; i++) {
-      children_[i].SeekToLast();
-      if (children_[i].Valid()) {
-        maxHeap_.push(&children_[i]);
+    for (auto& child : children_) {
+      child.SeekToLast();
+      if (child.Valid()) {
+        maxHeap_.push(&child);
       }
     }
     FindLargest();
@@ -67,10 +66,10 @@ class MergingIterator : public Iterator {
 
   virtual void Seek(const Slice& target) {
     ClearHeaps();
-    for (int i = 0; i < n_; i++) {
-      children_[i].Seek(target);
-      if (children_[i].Valid()) {
-        minHeap_.push(&children_[i]);
+    for (auto& child : children_) {
+      child.Seek(target);
+      if (child.Valid()) {
+        minHeap_.push(&child);
       }
     }
     FindSmallest();
@@ -87,16 +86,15 @@ class MergingIterator : public Iterator {
     // we explicitly position the non-current_ children.
     if (direction_ != kForward) {
       ClearHeaps();
-      for (int i = 0; i < n_; i++) {
-        IteratorWrapper* child = &children_[i];
-        if (child != current_) {
-          child->Seek(key());
-          if (child->Valid() &&
-              comparator_->Compare(key(), child->key()) == 0) {
-            child->Next();
+      for (auto& child : children_) {
+        if (&child != current_) {
+          child.Seek(key());
+          if (child.Valid() &&
+              comparator_->Compare(key(), child.key()) == 0) {
+            child.Next();
           }
-          if (child->Valid()) {
-            minHeap_.push(child);
+          if (child.Valid()) {
+            minHeap_.push(&child);
           }
         }
       }
@@ -121,19 +119,18 @@ class MergingIterator : public Iterator {
     // we explicitly position the non-current_ children.
     if (direction_ != kReverse) {
       ClearHeaps();
-      for (int i = 0; i < n_; i++) {
-        IteratorWrapper* child = &children_[i];
-        if (child != current_) {
-          child->Seek(key());
-          if (child->Valid()) {
+      for (auto& child : children_) {
+        if (&child != current_) {
+          child.Seek(key());
+          if (child.Valid()) {
             // Child is at first entry >= key().  Step back one to be < key()
-            child->Prev();
+            child.Prev();
           } else {
             // Child has no entries >= key().  Position at last entry.
-            child->SeekToLast();
+            child.SeekToLast();
           }
-          if (child->Valid()) {
-            maxHeap_.push(child);
+          if (child.Valid()) {
+            maxHeap_.push(&child);
           }
         }
       }
@@ -159,8 +156,8 @@ class MergingIterator : public Iterator {
 
   virtual Status status() const {
     Status status;
-    for (int i = 0; i < n_; i++) {
-      status = children_[i].status();
+    for (auto& child : children_) {
+      status = child.status();
       if (!status.ok()) {
         break;
       }
@@ -174,8 +171,7 @@ class MergingIterator : public Iterator {
   void ClearHeaps();
 
   const Comparator* comparator_;
-  IteratorWrapper* children_;
-  int n_;
+  std::vector<IteratorWrapper> children_;
   IteratorWrapper* current_;
   // Which direction is the iterator moving?
   enum Direction {

table/table.cc

@@ -6,12 +6,12 @@
 
 #include "db/dbformat.h"
 
-#include "leveldb/cache.h"
-#include "leveldb/comparator.h"
-#include "leveldb/env.h"
-#include "leveldb/filter_policy.h"
-#include "leveldb/options.h"
-#include "leveldb/statistics.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/comparator.h"
+#include "rocksdb/env.h"
+#include "rocksdb/filter_policy.h"
+#include "rocksdb/options.h"
+#include "rocksdb/statistics.h"
 
 #include "table/block.h"
 #include "table/filter_block.h"
@@ -19,6 +19,7 @@
 #include "table/two_level_iterator.h"
 
 #include "util/coding.h"
+#include "util/perf_context_imp.h"
 #include "util/stop_watch.h"
 
 namespace leveldb {
@@ -81,9 +82,10 @@ Status ReadBlock(RandomAccessFile* file,
                  const ReadOptions& options,
                  const BlockHandle& handle,
                  Block** result,
+                 Env* env,
                  bool* didIO = nullptr) {
   BlockContents contents;
-  Status s = ReadBlockContents(file, options, handle, &contents);
+  Status s = ReadBlockContents(file, options, handle, &contents, env);
   if (s.ok()) {
     *result = new Block(contents);
   }
@@ -118,14 +120,14 @@ Status Table::Open(const Options& options,
     return Status::InvalidArgument("file is too short to be an sstable");
   }
 
-
   Footer footer;
   s = footer.DecodeFrom(&footer_input);
   if (!s.ok()) return s;
 
   Block* index_block = nullptr;
   // TODO: we never really verify check sum for index block
-  s = ReadBlock(file.get(), ReadOptions(), footer.index_handle(), &index_block);
+  s = ReadBlock(file.get(), ReadOptions(), footer.index_handle(), &index_block,
+                options.env);
 
   if (s.ok()) {
     // We've successfully read the footer and the index block: we're
@@ -176,7 +178,7 @@ void Table::ReadMeta(const Footer& footer) {
   //  TODO: we never really verify check sum for meta index block
   Block* meta = nullptr;
   if (!ReadBlock(rep_->file.get(), ReadOptions(), footer.metaindex_handle(),
-                 &meta).ok()) {
+                 &meta, rep_->options.env).ok()) {
     // Do not propagate errors since meta info is not needed for operation
     return;
   }
@@ -203,7 +205,8 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
   // requiring checksum verification in Table::Open.
   ReadOptions opt;
   BlockContents block;
-  if (!ReadBlockContents(rep_->file.get(), opt, filter_handle, &block).ok()) {
+  if (!ReadBlockContents(rep_->file.get(), opt, filter_handle, &block,
+                        rep_->options.env).ok()) {
     return;
   }
   if (block.heap_allocated) {
@@ -237,8 +240,8 @@ Iterator* Table::BlockReader(void* arg,
                              const ReadOptions& options,
                              const Slice& index_value,
                              bool* didIO,
-                             bool for_compaction,
-                             const bool no_io) {
+                             bool for_compaction) {
+  const bool no_io = (options.read_tier == kBlockCacheTier);
   Table* table = reinterpret_cast<Table*>(arg);
   Cache* block_cache = table->rep_->options.block_cache.get();
   std::shared_ptr<Statistics> statistics = table->rep_->options.statistics;
@@ -266,9 +269,11 @@ Iterator* Table::BlockReader(void* arg,
       if (cache_handle != nullptr) {
         block = reinterpret_cast<Block*>(block_cache->Value(cache_handle));
 
+        BumpPerfCount(&perf_context.block_cache_hit_count);
         RecordTick(statistics, BLOCK_CACHE_HIT);
       } else if (no_io) {
-        return nullptr; // Did not find in block_cache and can't do IO
+        // Did not find in block_cache and can't do IO
+        return NewErrorIterator(Status::Incomplete("no blocking io"));
       } else {
         Histograms histogram = for_compaction ?
           READ_BLOCK_COMPACTION_MICROS : READ_BLOCK_GET_MICROS;
@@ -279,6 +284,7 @@ Iterator* Table::BlockReader(void* arg,
                 options,
                 handle,
                 &block,
+                table->rep_->options.env,
                 didIO
               );
         }
@@ -292,9 +298,11 @@ Iterator* Table::BlockReader(void* arg,
         RecordTick(statistics, BLOCK_CACHE_MISS);
       }
     } else if (no_io) {
-      return nullptr; // Could not read from block_cache and can't do IO
-    }else {
-      s = ReadBlock(table->rep_->file.get(), options, handle, &block, didIO);
+      // Could not read from block_cache and can't do IO
+      return NewErrorIterator(Status::Incomplete("no blocking io"));
+    } else {
+      s = ReadBlock(table->rep_->file.get(), options, handle, &block,
+                    table->rep_->options.env, didIO);
     }
   }
 
@@ -328,6 +336,11 @@ Iterator* Table::BlockReader(void* arg,
 // 1) key.starts_with(prefix(key))
 // 2) Compare(prefix(key), key) <= 0.
 // 3) If Compare(key1, key2) <= 0, then Compare(prefix(key1), prefix(key2)) <= 0
+//
+// TODO(tylerharter): right now, this won't cause I/O since blooms are
+// in memory.  When blooms may need to be paged in, we should refactor so that
+// this is only ever called lazily.  In particular, this shouldn't be called
+// while the DB lock is held like it is now.
 bool Table::PrefixMayMatch(const Slice& internal_prefix) const {
   FilterBlockReader* filter = rep_->filter;
   bool may_match = true;
@@ -337,12 +350,14 @@ bool Table::PrefixMayMatch(const Slice& internal_prefix) const {
     return true;
   }
 
-  Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
+  std::unique_ptr<Iterator> iiter(rep_->index_block->NewIterator(
+                                           rep_->options.comparator));
   iiter->Seek(internal_prefix);
-  if (! iiter->Valid()) {
+  if (!iiter->Valid()) {
     // we're past end of file
     may_match = false;
-  } else if (iiter->key().starts_with(internal_prefix)) {
+  } else if (ExtractUserKey(iiter->key()).starts_with(
+                                             ExtractUserKey(internal_prefix))) {
     // we need to check for this subtle case because our only
     // guarantee is that "the key is a string >= last key in that data
     // block" according to the doc/table_format.txt spec.
@@ -366,7 +381,12 @@ bool Table::PrefixMayMatch(const Slice& internal_prefix) const {
     assert(s.ok());
     may_match = filter->PrefixMayMatch(handle.offset(), internal_prefix);
   }
-  delete iiter;
+
+  RecordTick(rep_->options.statistics, BLOOM_FILTER_PREFIX_CHECKED);
+  if (!may_match) {
+    RecordTick(rep_->options.statistics, BLOOM_FILTER_PREFIX_USEFUL);
+  }
+
   return may_match;
 }
 
@@ -389,8 +409,7 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k,
                           void* arg,
                           bool (*saver)(void*, const Slice&, const Slice&,
                                         bool),
-                          void (*mark_key_may_exist)(void*),
-                          const bool no_io) {
+                          void (*mark_key_may_exist)(void*)) {
   Status s;
   Iterator* iiter = rep_->index_block->NewIterator(rep_->options.comparator);
   bool done = false;
@@ -408,10 +427,11 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k,
       break;
     } else {
       bool didIO = false;
-      Iterator* block_iter = BlockReader(this, options, iiter->value(),
-                                         &didIO, no_io);
+      std::unique_ptr<Iterator> block_iter(
+        BlockReader(this, options, iiter->value(), &didIO));
 
-      if (no_io && !block_iter) { // couldn't get block from block_cache
+      if (options.read_tier && block_iter->status().IsIncomplete()) {
+        // couldn't get block from block_cache
         // Update Saver.state to Found because we are only looking for whether
         // we can guarantee the key is not there when "no_io" is set
         (*mark_key_may_exist)(arg);
@@ -426,7 +446,6 @@ Status Table::InternalGet(const ReadOptions& options, const Slice& k,
         }
       }
       s = block_iter->status();
-      delete block_iter;
     }
   }
   if (s.ok()) {

table/table.h

@@ -7,8 +7,8 @@
 
 #include <memory>
 #include <stdint.h>
-#include "leveldb/iterator.h"
-#include "leveldb/env.h"
+#include "rocksdb/iterator.h"
+#include "rocksdb/env.h"
 
 namespace leveldb {
 
@@ -47,7 +47,7 @@ class Table {
 
   ~Table();
 
-  bool PrefixMayMatch(const Slice& prefix) const;
+  bool PrefixMayMatch(const Slice& internal_prefix) const;
 
   // Returns a new iterator over the table contents.
   // The result of NewIterator() is initially invalid (caller must
@@ -79,8 +79,7 @@ class Table {
                                const EnvOptions& soptions, const Slice&,
                                bool for_compaction);
   static Iterator* BlockReader(void*, const ReadOptions&, const Slice&,
-                               bool* didIO, bool for_compaction = false,
-                               const bool no_io = false);
+                               bool* didIO, bool for_compaction = false);
 
   // Calls (*handle_result)(arg, ...) repeatedly, starting with the entry found
   // after a call to Seek(key), until handle_result returns false.
@@ -90,8 +89,7 @@ class Table {
       const ReadOptions&, const Slice& key,
       void* arg,
       bool (*handle_result)(void* arg, const Slice& k, const Slice& v, bool),
-      void (*mark_key_may_exist)(void*) = nullptr,
-      const bool no_io = false);
+      void (*mark_key_may_exist)(void*) = nullptr);
 
 
   void ReadMeta(const Footer& footer);