Bump version to 5.5.2

Summary:
With a regression bug was introduced two years ago, by https://github.com/facebook/rocksdb/commit/6e9fbeb27c38329f33ae541302c44c8db8374f8c , we fail to check return status of fsync call. This can cause we miss the information from the file system and can potentially cause corrupted data which we could have been detected.
Closes https://github.com/facebook/rocksdb/pull/2495

Reviewed By: ajkr

Differential Revision: D5321949

Pulled By: siying

fbshipit-source-id: c68117914bb40700198fc37d0e4c63163a8a1031

CMakeLists.txt

@@ -106,58 +106,31 @@ if(NOT WIN32)
   string(STRIP "${ROCKSDB_VERSION_MAJOR}" ROCKSDB_VERSION_MAJOR)
 endif()
 
-option(WITH_MD_LIBRARY "build with MD" ON)
-if(WIN32 AND MSVC)
-  if(WITH_MD_LIBRARY)
-    set(RUNTIME_LIBRARY "MD")
-  else()
-    set(RUNTIME_LIBRARY "MT")
+if(WIN32)
+  option(WITH_AVX2 "build with AVX2" ON)
+  if(WITH_AVX2)
+    if(MSVC)
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+    else()
+      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2")
+    endif()
   endif()
-endif()
 
-option(PORTABLE "build a portable binary" OFF)
-option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
-if(PORTABLE)
-  # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
-  # is available, it is available by default.
-  if(FORCE_SSE42 AND NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
+  option(WITH_MD_LIBRARY "build with MD" ON)
+  if(MSVC)
+    if(WITH_MD_LIBRARY)
+      set(RUNTIME_LIBRARY "MD")
+    else()
+      set(RUNTIME_LIBRARY "MT")
+    endif()
   endif()
 else()
-  if(MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
-  else()
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native")
+  option(WITH_SSE42 "build with SSE4.2" ON)
+  if(WITH_SSE42)
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2")
   endif()
 endif()
 
-set(CMAKE_REQUIRED_FLAGS ${CMAKE_CXX_FLAGS})
-include(CheckCXXSourceCompiles)
-CHECK_CXX_SOURCE_COMPILES("
-#include <cstdint>
-#include <nmmintrin.h>
-int main() {
-  volatile uint32_t x = _mm_crc32_u32(0, 0);
-}
-" HAVE_SSE42)
-if(HAVE_SSE42)
-  add_definitions(-DHAVE_SSE42)
-elseif(FORCE_SSE42)
-  message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
-endif()
-
-CHECK_CXX_SOURCE_COMPILES("
-#if defined(_MSC_VER) && !defined(__thread)
-#define __thread __declspec(thread)
-#endif
-int main() {
-  static __thread int tls;
-}
-" HAVE_THREAD_LOCAL)
-if(HAVE_THREAD_LOCAL)
-  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
-endif()
-
 set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
 configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
 add_library(build_version OBJECT ${BUILD_VERSION_CC})
@@ -548,7 +521,7 @@ if(WIN32)
   set(SYSTEM_LIBS ${SYSTEM_LIBS} Shlwapi.lib Rpcrt4.lib)
   set(LIBS ${ROCKSDB_STATIC_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 else()
-  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
+  set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT} rt)
   set(LIBS ${ROCKSDB_SHARED_LIB} ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
   add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})

HISTORY.md

@@ -1,5 +1,10 @@
 # Rocksdb Change Log
-## Unreleased
+## 5.5.2 (07/13/2017)
+No change from 5.5.1.
+
+## 5.5.1 (06/27/2017)
+### Bug Fixes
+* Shouldn't ignore return value of fsync() in flush.
 
 ## 5.5.0 (05/17/2017)
 ### New Features
@@ -13,7 +18,10 @@
 * Add debugging function `GetAllKeyVersions` to see internal versions of a range of keys.
 * Support file ingestion with universal compaction style
 * Support file ingestion behind with option `allow_ingest_behind`
-* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled .
+* New option enable_pipelined_write which may improve write throughput in case writing from multiple threads and WAL enabled.
+
+### Bug Fixes
+* Fix the bug that Direct I/O uses direct reads for non-SST file
 
 ## 5.4.0 (04/11/2017)
 ### Public API Change

INSTALL.md

@@ -21,11 +21,9 @@ depend on gflags. You will need to have gflags installed to run `make all`. This
 use binaries compiled by `make all` in production.
 
 * By default the binary we produce is optimized for the platform you're compiling on
-(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
-CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
-`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
-to build a portable binary, add `PORTABLE=1` before your make commands, like this:
-`PORTABLE=1 make static_lib`.
+(-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
+your make commands, like this: `PORTABLE=1 make static_lib`. If you want to build a binary that
+makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SSE=1 make static_lib`.
 
 ## Dependencies
 

Makefile

@@ -1576,12 +1576,12 @@ rocksdbjavastaticrelease: rocksdbjavastatic
 rocksdbjavastaticreleasedocker: rocksdbjavastatic
 	DOCKER_LINUX_X64_CONTAINER=`docker ps -aqf name=rocksdb_linux_x64-be`; \
 	if [ -z "$$DOCKER_LINUX_X64_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos5_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
+		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x64-be evolvedbinary/rocksjava:centos6_x64-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
 	fi
 	docker start -a rocksdb_linux_x64-be
 	DOCKER_LINUX_X86_CONTAINER=`docker ps -aqf name=rocksdb_linux_x86-be`; \
 	if [ -z "$$DOCKER_LINUX_X86_CONTAINER" ]; then \
-		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos5_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
+		docker container create --attach stdin --attach stdout --attach stderr --volume `pwd`:/rocksdb-host --name rocksdb_linux_x86-be evolvedbinary/rocksjava:centos6_x86-be /rocksdb-host/java/crossbuild/docker-build-linux-centos.sh; \
 	fi
 	docker start -a rocksdb_linux_x86-be
 	cd java;jar -cf target/$(ROCKSDB_JAR_ALL) HISTORY*.md

TARGETS

@@ -11,7 +11,6 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_MALLOC_USABLE_SIZE",
   "-DROCKSDB_RANGESYNC_PRESENT",
   "-DROCKSDB_SCHED_GETCPU_PRESENT",
-  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",
@@ -248,6 +247,7 @@ cpp_library(
       "tools/ldb_cmd.cc",
       "tools/ldb_tool.cc",
       "tools/sst_dump_tool.cc",
+      "utilities/blob_db/blob_dump_tool.cc",
     ],
     deps = [],
     preprocessor_flags = rocksdb_preprocessor_flags,
@@ -303,7 +303,7 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['cache_test', 'cache/cache_test.cc', 'serial'],
  ['options_file_test', 'db/options_file_test.cc', 'serial'],
  ['compaction_picker_test', 'db/compaction_picker_test.cc', 'serial'],
- ['corruption_test', 'db/corruption_test.cc', 'serial'],
+ ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
  ['thread_list_test', 'util/thread_list_test.cc', 'serial'],
  ['table_properties_collector_test',
   'db/table_properties_collector_test.cc',
@@ -406,6 +406,7 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['ttl_test', 'utilities/ttl/ttl_test.cc', 'serial'],
  ['merge_helper_test', 'db/merge_helper_test.cc', 'serial'],
  ['file_indexer_test', 'db/file_indexer_test.cc', 'serial'],
+ ['db_statistics_test', 'db/db_statistics_test.cc', 'serial'],
  ['memory_test', 'utilities/memory/memory_test.cc', 'serial'],
  ['log_test', 'db/log_test.cc', 'serial'],
  ['env_timed_test', 'utilities/env_timed_test.cc', 'serial'],
@@ -429,12 +430,13 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['delete_scheduler_test', 'util/delete_scheduler_test.cc', 'serial'],
  ['transaction_test', 'utilities/transactions/transaction_test.cc', 'serial'],
  ['db_io_failure_test', 'db/db_io_failure_test.cc', 'serial'],
- ['db_log_iter_test', 'db/db_log_iter_test.cc', 'serial'],
+ ['corruption_test', 'db/corruption_test.cc', 'serial'],
  ['compact_on_deletion_collector_test',
   'utilities/table_properties_collectors/compact_on_deletion_collector_test.cc',
   'serial'],
  ['env_test', 'env/env_test.cc', 'serial'],
  ['db_wal_test', 'db/db_wal_test.cc', 'parallel'],
+ ['timer_queue_test', 'util/timer_queue_test.cc', 'serial'],
  ['sim_cache_test', 'utilities/simulator_cache/sim_cache_test.cc', 'serial'],
  ['db_memtable_test', 'db/db_memtable_test.cc', 'serial'],
  ['db_universal_compaction_test',
@@ -453,8 +455,7 @@ ROCKS_TESTS = [['merger_test', 'table/merger_test.cc', 'serial'],
  ['version_edit_test', 'db/version_edit_test.cc', 'serial'],
  ['skiplist_test', 'memtable/skiplist_test.cc', 'serial'],
  ['lru_cache_test', 'cache/lru_cache_test.cc', 'serial'],
- ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial'],
- ['db_statistics_test', 'db/db_statistics_test.cc', 'serial']]
+ ['plain_table_db_test', 'db/plain_table_db_test.cc', 'serial']]
 
 
 # Generate a test rule for each entry in ROCKS_TESTS

buckifier/targets_cfg.py

@@ -13,6 +13,9 @@ rocksdb_compiler_flags = [
   "-DROCKSDB_LIB_IO_POSIX",
   "-DROCKSDB_FALLOCATE_PRESENT",
   "-DROCKSDB_MALLOC_USABLE_SIZE",
+  "-DROCKSDB_RANGESYNC_PRESENT",
+  "-DROCKSDB_SCHED_GETCPU_PRESENT",
+  "-DROCKSDB_SUPPORT_THREAD_LOCAL",
   "-DOS_LINUX",
   # Flags to enable libs we include
   "-DSNAPPY",

build_tools/build_detect_platform

@@ -442,8 +442,14 @@ if test "$USE_HDFS"; then
   JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
 fi
 
+if [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
+  # Intel SSE instructions breaks compilation on FreeBSD i386
+  unset USE_SSE
+fi
+
 if test "$USE_SSE"; then
-  COMMON_FLAGS="$COMMON_FLAGS -msse4.2"
+  # if Intel SSE instruction set is supported, set USE_SSE=1
+  COMMON_FLAGS="$COMMON_FLAGS -msse -msse4.2 "
 elif test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
@@ -456,36 +462,6 @@ elif test -z "$PORTABLE"; then
   fi
 fi
 
-$CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <nmmintrin.h>
-  int main() {
-    volatile uint32_t x = _mm_crc32_u32(0, 0);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
-fi
-
-# iOS doesn't support thread-local storage, but this check would erroneously
-# succeed because the cross-compiler flags are added by the Makefile, not this
-# script.
-if [ "$PLATFORM" != IOS ]; then
-  $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #if defined(_MSC_VER) && !defined(__thread)
-  #define __thread __declspec(thread)
-  #endif
-  int main() {
-    static __thread int tls;
-  }
-EOF
-  if [ "$?" = 0 ]; then
-    COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
-  fi
-fi
-
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS $COMMON_FLAGS"
 

build_tools/fbcode_config.sh

@@ -128,7 +128,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_SCHED_GETCPU_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL"
 CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"

build_tools/fbcode_config4.8.1.sh

@@ -93,7 +93,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_SCHED_GETCPU_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL"
 CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
 CXXFLAGS+=" $CFLAGS"
 

db/builder.cc

@@ -179,7 +179,7 @@ Status BuildTable(
     // Finish and check for file errors
     if (s.ok() && !empty) {
       StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
-      file_writer->Sync(ioptions.use_fsync);
+      s = file_writer->Sync(ioptions.use_fsync);
     }
     if (s.ok() && !empty) {
       s = file_writer->Close();

db/db_iter.cc

@@ -933,12 +933,7 @@ void DBIter::FindPrevUserKey() {
     return;
   }
   size_t num_skipped = 0;
-// Suppress false positive clang analyzer warnings.
-#ifdef __clang_analyzer__
-  ParsedInternalKey ikey(Slice(), 0, 0);
-#else
   ParsedInternalKey ikey;
-#endif  // __clang_analyzer__
   FindParseableKey(&ikey, kReverse);
   int cmp;
   while (iter_->Valid() &&

db/db_properties_test.cc

@@ -1300,6 +1300,18 @@ TEST_F(DBPropertiesTest, NeedCompactHintPersistentTest) {
     SetPerfLevel(kDisable);
   }
 }
+
+TEST_F(DBPropertiesTest, EstimateNumKeysUnderflow) {
+  Options options;
+  Reopen(options);
+  Put("foo", "bar");
+  Delete("foo");
+  Delete("foo");
+  uint64_t num_keys = 0;
+  ASSERT_TRUE(dbfull()->GetIntProperty("rocksdb.estimate-num-keys", &num_keys));
+  ASSERT_EQ(0, num_keys);
+}
+
 #endif  // ROCKSDB_LITE
 }  // namespace rocksdb
 

db/internal_stats.cc

@@ -679,12 +679,14 @@ bool InternalStats::HandleEstimateNumKeys(uint64_t* value, DBImpl* db,
   // Estimate number of entries in the column family:
   // Use estimated entries in tables + total entries in memtables.
   const auto* vstorage = cfd_->current()->storage_info();
-  *value = cfd_->mem()->num_entries() +
-           cfd_->imm()->current()->GetTotalNumEntries() -
-           (cfd_->mem()->num_deletes() +
-            cfd_->imm()->current()->GetTotalNumDeletes()) *
-               2 +
-           vstorage->GetEstimatedActiveKeys();
+  uint64_t estimate_keys = cfd_->mem()->num_entries() +
+                           cfd_->imm()->current()->GetTotalNumEntries() +
+                           vstorage->GetEstimatedActiveKeys();
+  uint64_t estimate_deletes =
+      cfd_->mem()->num_deletes() + cfd_->imm()->current()->GetTotalNumDeletes();
+  *value = estimate_keys > estimate_deletes * 2
+               ? estimate_keys - (estimate_deletes * 2)
+               : 0;
   return true;
 }
 

include/rocksdb/env.h

@@ -900,7 +900,7 @@ class EnvWrapper : public Env {
  public:
   // Initialize an EnvWrapper that delegates all calls to *t
   explicit EnvWrapper(Env* t) : target_(t) { }
-  virtual ~EnvWrapper();
+  ~EnvWrapper() override;
 
   // Return the target to which this Env forwards all calls
   Env* target() const { return target_; }
@@ -935,8 +935,8 @@ class EnvWrapper : public Env {
                          const EnvOptions& options) override {
     return target_->NewRandomRWFile(fname, result, options);
   }
-  virtual Status NewDirectory(const std::string& name,
-                              unique_ptr<Directory>* result) override {
+  Status NewDirectory(const std::string& name, unique_ptr<Directory>* result)
+      override {
     return target_->NewDirectory(name, result);
   }
   Status FileExists(const std::string& f) override {
@@ -998,15 +998,14 @@ class EnvWrapper : public Env {
     return target_->StartThread(f, a);
   }
   void WaitForJoin() override { return target_->WaitForJoin(); }
-  virtual unsigned int GetThreadPoolQueueLen(
-      Priority pri = LOW) const override {
+  unsigned int GetThreadPoolQueueLen(Priority pri = LOW) const override {
     return target_->GetThreadPoolQueueLen(pri);
   }
-  virtual Status GetTestDirectory(std::string* path) override {
+  Status GetTestDirectory(std::string* path) override {
     return target_->GetTestDirectory(path);
   }
-  virtual Status NewLogger(const std::string& fname,
-                           shared_ptr<Logger>* result) override {
+  Status NewLogger(const std::string& fname, shared_ptr<Logger>* result)
+      override {
     return target_->NewLogger(fname, result);
   }
   uint64_t NowMicros() override { return target_->NowMicros(); }
@@ -1091,10 +1090,10 @@ class WritableFileWrapper : public WritableFile {
     return target_->InvalidateCache(offset, length);
   }
 
-  virtual void SetPreallocationBlockSize(size_t size) override {
+  void SetPreallocationBlockSize(size_t size) override {
     target_->SetPreallocationBlockSize(size);
   }
-  virtual void PrepareWrite(size_t offset, size_t len) override {
+  void PrepareWrite(size_t offset, size_t len) override {
     target_->PrepareWrite(offset, len);
   }
 

include/rocksdb/flush_block_policy.h

@@ -48,11 +48,11 @@ class FlushBlockBySizePolicyFactory : public FlushBlockPolicyFactory {
  public:
   FlushBlockBySizePolicyFactory() {}
 
-  virtual const char* Name() const override {
+  const char* Name() const override {
     return "FlushBlockBySizePolicyFactory";
   }
 
-  virtual FlushBlockPolicy* NewFlushBlockPolicy(
+  FlushBlockPolicy* NewFlushBlockPolicy(
       const BlockBasedTableOptions& table_options,
       const BlockBuilder& data_block_builder) const override;
 

include/rocksdb/iostats_context.h

@@ -46,13 +46,12 @@ struct IOStatsContext {
   uint64_t logger_nanos;
 };
 
-#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL
-  #if defined(_MSC_VER) && !defined(__thread)
-    // Thread local storage on Linux
-    // There is thread_local in C++11
-    #define __thread __declspec(thread)
-  #endif
-  extern __thread IOStatsContext iostats_context;
-#endif
+#ifndef IOS_CROSS_COMPILE
+# ifdef _MSC_VER
+extern __declspec(thread) IOStatsContext iostats_context;
+# else
+extern __thread IOStatsContext iostats_context;
+# endif
+#endif  // IOS_CROSS_COMPILE
 
 }  // namespace rocksdb

include/rocksdb/merge_operator.h

@@ -188,7 +188,7 @@ class MergeOperator {
 // The simpler, associative merge operator.
 class AssociativeMergeOperator : public MergeOperator {
  public:
-  virtual ~AssociativeMergeOperator() {}
+  ~AssociativeMergeOperator() override {}
 
   // Gives the client a way to express the read -> modify -> write semantics
   // key:           (IN) The key that's associated with this merge operation.
@@ -212,14 +212,16 @@ class AssociativeMergeOperator : public MergeOperator {
 
  private:
   // Default implementations of the MergeOperator functions
-  virtual bool FullMergeV2(const MergeOperationInput& merge_in,
-                           MergeOperationOutput* merge_out) const override;
+  bool FullMergeV2(
+      const MergeOperationInput& merge_in,
+      MergeOperationOutput* merge_out) const override;
 
-  virtual bool PartialMerge(const Slice& key,
-                            const Slice& left_operand,
-                            const Slice& right_operand,
-                            std::string* new_value,
-                            Logger* logger) const override;
+  bool PartialMerge(
+      const Slice& key,
+      const Slice& left_operand,
+      const Slice& right_operand,
+      std::string* new_value,
+      Logger* logger) const override;
 };
 
 }  // namespace rocksdb

include/rocksdb/perf_context.h

@@ -150,18 +150,15 @@ struct PerfContext {
   uint64_t env_new_logger_nanos;
 };
 
-#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
+#if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
 extern PerfContext perf_context;
+#elif defined(_MSC_VER)
+extern __declspec(thread) PerfContext perf_context;
 #else
   #if defined(OS_SOLARIS)
     PerfContext *getPerfContext();
     #define perf_context (*getPerfContext())
   #else
-    #if defined(_MSC_VER) && !defined(__thread)
-      // Thread local storage on Linux
-      // There is thread_local in C++11
-      #define __thread __declspec(thread)
-    #endif
     extern __thread PerfContext perf_context;
   #endif
 #endif

include/rocksdb/thread_status.h

@@ -22,7 +22,8 @@
 
 #if !defined(ROCKSDB_LITE) && \
     !defined(NROCKSDB_THREAD_STATUS) && \
-    defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
+    !defined(OS_MACOSX) && \
+    !defined(IOS_CROSS_COMPILE)
 #define ROCKSDB_USING_THREAD_STATUS
 #endif
 

include/rocksdb/utilities/sim_cache.h

@@ -39,9 +39,11 @@ class SimCache : public Cache {
  public:
   SimCache() {}
 
-  virtual ~SimCache() {}
+  ~SimCache() override {}
 
-  virtual const char* Name() const override { return "SimCache"; }
+  const char* Name() const override {
+    return "SimCache";
+  }
 
   // returns the maximum configured capacity of the simcache for simulation
   virtual size_t GetSimCapacity() const = 0;

include/rocksdb/utilities/transaction_db.h

@@ -148,7 +148,7 @@ class TransactionDB : public StackableDB {
       StackableDB* db, const TransactionDBOptions& txn_db_options,
       const std::vector<size_t>& compaction_enabled_cf_indices,
       const std::vector<ColumnFamilyHandle*>& handles, TransactionDB** dbptr);
-  virtual ~TransactionDB() {}
+  ~TransactionDB() override {}
 
   // Starts a new Transaction.
   //

include/rocksdb/utilities/write_batch_with_index.h

@@ -97,7 +97,7 @@ class WriteBatchWithIndex : public WriteBatchBase {
       size_t reserved_bytes = 0, bool overwrite_key = false,
       size_t max_bytes = 0);
 
-  virtual ~WriteBatchWithIndex();
+  ~WriteBatchWithIndex() override;
 
   using WriteBatchBase::Put;
   Status Put(ColumnFamilyHandle* column_family, const Slice& key,

include/rocksdb/version.h

@@ -6,7 +6,7 @@
 
 #define ROCKSDB_MAJOR 5
 #define ROCKSDB_MINOR 5
-#define ROCKSDB_PATCH 0
+#define ROCKSDB_PATCH 2
 
 // Do not use these. We made the mistake of declaring macros starting with
 // double underscore. Now we have to live with our choice. We'll deprecate these

include/rocksdb/write_batch.h

@@ -61,7 +61,7 @@ struct SavePoint {
 class WriteBatch : public WriteBatchBase {
  public:
   explicit WriteBatch(size_t reserved_bytes = 0, size_t max_bytes = 0);
-  ~WriteBatch();
+  ~WriteBatch() override;
 
   using WriteBatchBase::Put;
   // Store the mapping "key->value" in the database.

java/crossbuild/Vagrantfile

@@ -7,16 +7,17 @@ VAGRANTFILE_API_VERSION = "2"
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "linux32" do |linux32|
-    linux32.vm.box = "hansode/centos-5.11-i386"
+    linux32.vm.box = "hansode/centos-6.7-i386"
   end
 
   config.vm.define "linux64" do |linux64|
-    linux64.vm.box = "hansode/centos-5.11-x86_64"
+    linux64.vm.box = "hansode/centos-6.7-x86_64"
   end
 
   config.vm.provider "virtualbox" do |v|
     v.memory = 2048
     v.cpus = 4
+    v.customize ["modifyvm", :id, "--nictype1", "virtio" ]
   end
 
   config.vm.provision :shell, path: "build-linux-centos.sh"

java/crossbuild/build-linux-centos.sh

@@ -2,34 +2,24 @@
 
 set -e
 
-# CentOS 5 is now end of life so we need to switch its repos over to the vault
-sed -i -e '/^mirrorlist=/d' -e 's%^#baseurl=http://mirror.centos.org/centos/$releasever/%baseurl=http://vault.centos.org/5.11/%g' /etc/yum.repos.d/CentOS-Base.repo
-sed -i -e '/^mirrorlist=/d' -e 's%^#baseurl=http://mirror.centos.org/centos/$releasever/%baseurl=http://vault.centos.org/5.11/%g' /etc/yum.repos.d/CentOS-fasttrack.repo
-if [ -f /etc/yum.repos.d/libselinux.repo ]; then
-	sed -i -e '/^mirrorlist=/d' -e 's%^#baseurl=http://mirror.centos.org/centos/$releasever/%baseurl=http://vault.centos.org/5.11/%g' /etc/yum.repos.d/libselinux.repo
-fi
+# remove fixed relesever variable present in the hanscode boxes
+sudo rm -f /etc/yum/vars/releasever
 
 # enable EPEL
 sudo yum -y install epel-release
 
 # install all required packages for rocksdb that are available through yum
-ARCH=$(uname -i)
-sudo yum -y install openssl java-1.7.0-openjdk-devel.$ARCH zlib-devel bzip2-devel lz4-devel snappy-devel
+sudo yum -y install openssl java-1.7.0-openjdk-devel zlib-devel bzip2-devel lz4-devel snappy-devel libzstd-devel
 
-# install gcc/g++ 4.8.2 via CERN (http://linux.web.cern.ch/linux/devtoolset/)
-sudo wget -O /etc/yum.repos.d/slc5-devtoolset.repo http://linuxsoft.cern.ch/cern/devtoolset/slc5-devtoolset.repo
-sudo wget -O /etc/pki/rpm-gpg/RPM-GPG-KEY-cern http://ftp.mirrorservice.org/sites/ftp.scientificlinux.org/linux/scientific/obsolete/51/i386/RPM-GPG-KEYs/RPM-GPG-KEY-cern
-sudo yum -y install devtoolset-2
+# install gcc/g++ 4.8.2 from tru/devtools-2
+sudo wget -O /etc/yum.repos.d/devtools-2.repo https://people.centos.org/tru/devtools-2/devtools-2.repo
+sudo yum -y install devtoolset-2-binutils devtoolset-2-gcc devtoolset-2-gcc-c++
 
 # install gflags
 wget https://github.com/gflags/gflags/archive/v2.0.tar.gz -O gflags-2.0.tar.gz
 tar xvfz gflags-2.0.tar.gz; cd gflags-2.0; scl enable devtoolset-2 ./configure; scl enable devtoolset-2 make; sudo make install
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib
 
-# install zstandard
-wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz -O zstd-1.1.3.tar.gz
-tar zxvf zstd-1.1.3.tar.gz; cd zstd-1.1.3; scl enable devtoolset-2 make; sudo make install
-
 # set java home so we can build rocksdb jars
 export JAVA_HOME=/usr/lib/jvm/java-1.7.0
 

monitoring/iostats_context.cc

@@ -9,9 +9,13 @@
 
 namespace rocksdb {
 
-#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL
+#ifndef IOS_CROSS_COMPILE
+# ifdef _MSC_VER
+__declspec(thread) IOStatsContext iostats_context;
+# else
 __thread IOStatsContext iostats_context;
-#endif
+# endif
+#endif  // IOS_CROSS_COMPILE
 
 void IOStatsContext::Reset() {
   thread_pool_id = Env::Priority::TOTAL;

monitoring/iostats_context_imp.h

@@ -9,7 +9,7 @@
 #include "monitoring/perf_step_timer.h"
 #include "rocksdb/iostats_context.h"
 
-#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL
+#ifndef IOS_CROSS_COMPILE
 
 // increment a specific counter by the specified value
 #define IOSTATS_ADD(metric, value)     \
@@ -41,7 +41,7 @@
   PerfStepTimer iostats_step_timer_ ## metric(&(iostats_context.metric));  \
   iostats_step_timer_ ## metric.Start();
 
-#else  // ROCKSDB_SUPPORT_THREAD_LOCAL
+#else  // IOS_CROSS_COMPILE
 
 #define IOSTATS_ADD(metric, value)
 #define IOSTATS_ADD_IF_POSITIVE(metric, value)
@@ -53,4 +53,4 @@
 
 #define IOSTATS_TIMER_GUARD(metric)
 
-#endif  // ROCKSDB_SUPPORT_THREAD_LOCAL
+#endif  // IOS_CROSS_COMPILE

monitoring/perf_context.cc

@@ -11,8 +11,10 @@
 
 namespace rocksdb {
 
-#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
+#if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
   PerfContext perf_context;
+#elif defined(_MSC_VER)
+  __declspec(thread) PerfContext perf_context;
 #else
   #if defined(OS_SOLARIS)
     __thread PerfContext perf_context_;
@@ -22,7 +24,7 @@ namespace rocksdb {
 #endif
 
 void PerfContext::Reset() {
-#if !defined(NPERF_CONTEXT) && defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
+#if !defined(NPERF_CONTEXT) && !defined(IOS_CROSS_COMPILE)
   user_key_comparison_count = 0;
   block_cache_hit_count = 0;
   block_read_count = 0;
@@ -96,7 +98,7 @@ void PerfContext::Reset() {
   }
 
 std::string PerfContext::ToString(bool exclude_zero_counters) const {
-#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
+#if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
   return "";
 #else
   std::ostringstream ss;

monitoring/perf_context_imp.h

@@ -12,7 +12,7 @@
 
 namespace rocksdb {
 
-#if defined(NPERF_CONTEXT) || !defined(ROCKSDB_SUPPORT_THREAD_LOCAL)
+#if defined(NPERF_CONTEXT) || defined(IOS_CROSS_COMPILE)
 
 #define PERF_TIMER_GUARD(metric)
 #define PERF_CONDITIONAL_TIMER_FOR_MUTEX_GUARD(metric, condition)

monitoring/perf_level.cc

@@ -7,14 +7,16 @@
 //
 
 #include <assert.h>
+#include <sstream>
 #include "monitoring/perf_level_imp.h"
+#include "port/port.h"
 
 namespace rocksdb {
 
-#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL
-__thread PerfLevel perf_level = kEnableCount;
-#else
+#if defined(IOS_CROSS_COMPILE)
 PerfLevel perf_level = kEnableCount;
+#else
+__thread PerfLevel perf_level = kEnableCount;
 #endif
 
 void SetPerfLevel(PerfLevel level) {

monitoring/perf_level_imp.h

@@ -11,10 +11,10 @@
 
 namespace rocksdb {
 
-#ifdef ROCKSDB_SUPPORT_THREAD_LOCAL
-extern __thread PerfLevel perf_level;
-#else
+#if defined(IOS_CROSS_COMPILE)
 extern PerfLevel perf_level;
+#else
+extern __thread PerfLevel perf_level;
 #endif
 
 }  // namespace rocksdb

tools/db_stress.cc

@@ -2360,8 +2360,8 @@ int main(int argc, char** argv) {
   SetUsageMessage(std::string("\nUSAGE:\n") + std::string(argv[0]) +
                   " [OPTIONS]...");
   ParseCommandLineFlags(&argc, &argv, true);
-#if !defined(OS_MACOSX) && !defined(OS_WIN) && !defined(OS_SOLARIS) &&  \
-  !defined(OS_AIX)
+#if !defined(NDEBUG) && !defined(OS_MACOSX) && !defined(OS_WIN) && \
+  !defined(OS_SOLARIS) && !defined(OS_AIX)
   rocksdb::SyncPoint::GetInstance()->SetCallBack(
     "NewWritableFile:O_DIRECT", [&](void* arg) {
       int* val = static_cast<int*>(arg);

util/crc32c.cc

@@ -15,9 +15,14 @@
 #include "util/crc32c.h"
 
 #include <stdint.h>
-#ifdef HAVE_SSE42
+#ifdef __SSE4_2__
 #include <nmmintrin.h>
 #endif
+#if defined(_WIN64)
+#ifdef __AVX2__
+#include <nmmintrin.h>
+#endif
+#endif
 #include "util/coding.h"
 
 namespace rocksdb {
@@ -293,12 +298,21 @@ static inline uint32_t LE_LOAD32(const uint8_t *p) {
   return DecodeFixed32(reinterpret_cast<const char*>(p));
 }
 
-#if defined(HAVE_SSE42) && (defined(__LP64__) || defined(_WIN64))
+#ifdef __SSE4_2__
+#ifdef __LP64__
 static inline uint64_t LE_LOAD64(const uint8_t *p) {
   return DecodeFixed64(reinterpret_cast<const char*>(p));
 }
 #endif
+#endif
 
+#if defined(_WIN64)
+#ifdef __AVX2__
+static inline uint64_t LE_LOAD64(const uint8_t *p) {
+  return DecodeFixed64(reinterpret_cast<const char*>(p));
+}
+#endif
+#endif
 static inline void Slow_CRC32(uint64_t* l, uint8_t const **p) {
   uint32_t c = static_cast<uint32_t>(*l ^ LE_LOAD32(*p));
   *p += 4;
@@ -316,9 +330,8 @@ static inline void Slow_CRC32(uint64_t* l, uint8_t const **p) {
 }
 
 static inline void Fast_CRC32(uint64_t* l, uint8_t const **p) {
-#ifndef HAVE_SSE42
-  Slow_CRC32(l, p);
-#elif defined(__LP64__) || defined(_WIN64)
+#ifdef __SSE4_2__
+#ifdef __LP64__
   *l = _mm_crc32_u64(*l, LE_LOAD64(*p));
   *p += 8;
 #else
@@ -327,6 +340,16 @@ static inline void Fast_CRC32(uint64_t* l, uint8_t const **p) {
   *l = _mm_crc32_u32(static_cast<unsigned int>(*l), LE_LOAD32(*p));
   *p += 4;
 #endif
+#elif defined(_WIN64)
+#ifdef __AVX2__
+  *l = _mm_crc32_u64(*l, LE_LOAD64(*p));
+  *p += 8;
+#else
+  Slow_CRC32(l, p);
+#endif
+#else
+  Slow_CRC32(l, p);
+#endif
 }
 
 template<void (*CRC32)(uint64_t*, uint8_t const**)>
@@ -395,7 +418,9 @@ static inline Function Choose_Extend() {
 }
 
 bool IsFastCrc32Supported() {
-#if defined(__SSE4_2__) || defined(_WIN64)
+#ifdef __SSE4_2__
+  return isSSE42();
+#elif defined(_WIN64)
   return isSSE42();
 #else
   return false;

util/thread_local.h

@@ -20,6 +20,14 @@
 #include "util/autovector.h"
 #include "port/port.h"
 
+#ifndef ROCKSDB_SUPPORT_THREAD_LOCAL
+#  if defined(OS_WIN) || defined(OS_MACOSX) || defined(IOS_CROSS_COMPILE)
+#    define ROCKSDB_SUPPORT_THREAD_LOCAL 0
+#  else
+#    define ROCKSDB_SUPPORT_THREAD_LOCAL 1
+#  endif
+#endif
+
 namespace rocksdb {
 
 // Cleanup function that will be called for a stored thread local