TESTING ppc build based on known good base revision

This commit is actually a cherry-pick of #6643 but for testing purposes
only (see Issue #6653)

.circleci/cat_ignore_eagain

@@ -1,54 +0,0 @@
-#! /bin/bash
-
-# Work around issue with parallel make output causing random error, as in
-# make[1]: write error: stdout
-# Probably due to a kernel bug:
-#   https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
-# Seems to affect image ubuntu-1604:201903-01 and ubuntu-1604:202004-01
-
-cd "$(dirname $0)"
-
-if [ ! -x cat_ignore_eagain.out ]; then
-    cc -x c -o cat_ignore_eagain.out - << EOF
-#include <unistd.h>
-#include <errno.h>
-#include <stdio.h>
-int main() {
-    int n, m, p;
-    char buf[1024];
-    for (;;) {
-        n = read(STDIN_FILENO, buf, 1024);
-        if (n > 0 && n <= 1024) {
-            for (m = 0; m < n;) {
-                p = write(STDOUT_FILENO, buf + m, n - m);
-                if (p < 0) {
-                    if (errno == EAGAIN) {
-                        // ignore but pause a bit
-                        usleep(100);
-                    } else {
-                        perror("write failed");
-                        return 42;
-                    }
-                } else {
-                    m += p;
-                }
-            }
-        } else if (n < 0) {
-            if (errno == EAGAIN) {
-                // ignore but pause a bit
-                usleep(100);
-            } else {
-                // Some non-ignorable error
-                perror("read failed");
-                return 43;
-            }
-        } else {
-            // EOF
-            return 0;
-        }
-    }
-}
-EOF
-fi
-
-exec ./cat_ignore_eagain.out

.circleci/config.yml

@@ -2,131 +2,6 @@ version: 2.1
 
 orbs:
   win: circleci/windows@2.4.0
-  slack: circleci/slack@3.4.2
-
-aliases:
-  - &notify-on-master-failure
-    fail_only: true
-    only_for_branches: master
-
-commands:
-  install-pyenv-on-macos:
-    steps:
-      - run:
-          name: Install pyenv on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv
-
-  install-cmake-on-macos:
-    steps:
-      - run:
-          name: Install cmake on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
-
-  increase-max-open-files-on-macos:
-    steps:
-      - run:
-          name: Increase max open files
-          command: |
-            sudo sysctl -w kern.maxfiles=1048576
-            sudo sysctl -w kern.maxfilesperproc=1048576
-            sudo launchctl limit maxfiles 1048576
-
-  pre-steps:
-    parameters:
-      python-version:
-        default: "3.5.9"
-        type: string
-    steps:
-      - checkout
-      - run: pyenv install --skip-existing <<parameters.python-version>>
-      - run: pyenv global <<parameters.python-version>>
-      - run:
-          name: Setup Environment Variables
-          command: |
-            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
-            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
-            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
-            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
-            echo "export GTEST_COLOR=1" >> $BASH_ENV
-            echo "export CTEST_OUTPUT_ON_FAILURE=1" >> $BASH_ENV
-            echo "export CTEST_TEST_TIMEOUT=300" >> $BASH_ENV
-
-  pre-steps-macos:
-      steps:
-        - pre-steps:
-            python-version: "3.6.0"
-
-  post-steps:
-    steps:
-      - slack/status: *notify-on-master-failure
-      - store_test_results: # store test result if there's any
-          path: /tmp/test-results
-      - store_artifacts: # store LOG for debugging if there's any
-          path: LOG
-      - run: # on fail, compress Test Logs for diagnosing the issue
-         name: Compress Test Logs
-         command: tar -cvzf t.tar.gz t
-         when: on_fail
-      - store_artifacts: # on fail, store Test Logs for diagnosing the issue
-          path: t.tar.gz
-          destination: test_logs
-          when: on_fail
-
-  install-clang-10:
-    steps:
-      - run:
-          name: Install Clang 10
-          command: |
-            echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
-            sudo apt-get update -y && sudo apt-get install -y clang-10
-
-  install-gflags:
-    steps:
-      - run:
-          name: Install gflags
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-
-  install-benchmark:
-    steps:
-      - run: # currently doesn't support ubuntu-1604 which doesn't have libbenchmark package, user can still install by building it youself
-          name: Install benchmark
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libbenchmark-dev
-
-  upgrade-cmake:
-    steps:
-      - run:
-          name: Upgrade cmake
-          command: |
-            sudo apt remove --purge cmake
-            sudo snap install cmake --classic
-
-  install-gflags-on-macos:
-    steps:
-      - run:
-          name: Install gflags on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
-
-  install-gtest-parallel:
-    steps:
-      - run:
-          name: Install gtest-parallel
-          command: |
-            git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
-            echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV
-
-  install-compression-libs:
-    steps:
-      - run:
-          name: Install compression libs
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
 
 executors:
   windows-2xlarge:
@@ -136,296 +11,20 @@ executors:
       shell: bash.exe
 
 jobs:
-  build-macos:
-    macos:
-      xcode: 11.3.0
-    resource_class: large
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-cmake:
-    macos:
-      xcode: 11.3.0
-    resource_class: large
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-cmake-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j32 && ctest -j10) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-mem-env:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-encrypted-env:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-shared_lib-alt_namespace-status_checked:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-release:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-      - post-steps
-
-  build-linux-release-rtti:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-
-  build-linux-lite:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-lite-release:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-      - post-steps
-
-  build-linux-clang-no_test_run:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev libtbb-dev
-      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-clang10-asan:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-mini-tsan:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
-      - post-steps
-
-  build-linux-clang10-ubsan:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-clang-analyze:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
-      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
-      - post-steps
-
-  build-linux-cmake:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - upgrade-cmake
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 .. && make V=1 -j20 && ctest -j20) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-cmake-ubuntu-20:
-    machine:
-      image: ubuntu-2004:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-benchmark
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20 && make microbench) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-unity:
-    docker: # executor type
-      - image: gcc:latest
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: apt-get update -y && apt-get install -y libgflags-dev
-      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-gcc-4_8-no_test_run:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
-      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-8-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
-      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-9-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
-      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-10-cxx20-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: xlarge
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
-      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  # This job is only to make sure the microbench tests are able to run, the benchmark result is not meaningful as the CI host is changing.
-  build-linux-microbench:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: xlarge
-    steps:
-      - pre-steps
-      - install-benchmark
-      - run: DEBUG_LEVEL=0 make microbench | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-windows:
+  build:
     executor: windows-2xlarge
-    parameters:
-      extra_cmake_opt:
-        default: ""
-        type: string
-      vs_year:
-        default: "2019"
-        type: string
-      cmake_generator:
-        default: "Visual Studio 16 2019"
-        type: string
+    
     environment:
       THIRDPARTY_HOME: C:/Users/circleci/thirdparty
       CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
       CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
+      CMAKE_GENERATOR: Visual Studio 16 2019
       SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
       SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
       SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
-      VS_YEAR: <<parameters.vs_year>>
-      CMAKE_GENERATOR: <<parameters.cmake_generator>>
+
     steps:
       - checkout
-      - run:
-          name: "Setup VS"
-          command: |
-            if [[ "${VS_YEAR}" == "2019" ]]; then
-              echo "VS2019 already present."
-            elif [[ "${VS_YEAR}" == "2017" ]]; then
-              echo "Installing VS2017..."
-              powershell .circleci/vs2017_install.ps1
-            elif [[ "${VS_YEAR}" == "2015" ]]; then
-              echo "Installing VS2015..."
-              powershell .circleci/vs2015_install.ps1
-            fi
-      - store_artifacts:
-          path: \Users\circleci\AppData\Local\Temp\vslogs.zip
       - run:
           name: "Install thirdparty dependencies"
           command: |
@@ -447,361 +46,11 @@ jobs:
           command: |
             mkdir build
             cd build
-            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
+            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 ..
             cd ..
-            echo "Building with VS version: ${CMAKE_GENERATOR}"
             msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
       - run:
           name: "Test RocksDB"
           shell: powershell.exe
           command: |
-            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
-
-  build-linux-java:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-java-static:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-java:
-    macos:
-      xcode: 11.3.0
-    resource_class: medium
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-java-static:
-    macos:
-      xcode: 11.3.0
-    resource_class: medium
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - install-cmake-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-examples:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Build examples"
-          command: |
-            OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
-      - post-steps
-
-  build-cmake-mingw:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y mingw-w64
-      - run: sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
-      - run:
-          name: "Build cmake-mingw"
-          command: |
-            sudo apt-get install snapd && sudo snap install cmake --beta --classic
-            export PATH=/snap/bin:$PATH
-            sudo apt-get install -y openjdk-8-jdk
-            export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-            export PATH=$JAVA_HOME/bin:$PATH
-            echo "JAVA_HOME=${JAVA_HOME}"
-            which java && java -version
-            which javac && javac -version
-            mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-linux-non-shm:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    parameters:
-      start_test:
-        default: ""
-        type: string
-      end_test:
-        default: ""
-        type: string
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-gtest-parallel
-      - run:
-          name: "Build unit tests"
-          command: |
-            echo "env: $(env)"
-            echo "** done env"
-            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
-      - run:
-          name: "Run unit tests in parallel"
-          command: |
-            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
-            cat /tmp/test_list
-            export TEST_TMPDIR=/tmp/rocksdb_test_tmp
-            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
-      - post-steps
-
-  build-linux-arm-test-full:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: make V=1 J=4 -j4 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-arm:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-arm-cmake-no_test_run:
-    machine:
-      image: ubuntu-2004:202101-01
-    resource_class: arm.large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-8-openjdk-arm64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build with cmake"
-          command: |
-            mkdir build
-            cd build
-            cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
-            make -j4
-      - run:
-          name: "Build Java with cmake"
-          command: |
-            rm -rf build
-            mkdir build
-            cd build
-            cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
-            make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-format-compatible:
-    machine:
-      image: ubuntu-1604:202104-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-compression-libs
-      - run:
-          name: "test"
-          command: |
-            export TEST_TMPDIR=/dev/shm/rocksdb
-            rm -rf /dev/shm/rocksdb
-            mkdir /dev/shm/rocksdb
-            tools/check_format_compatible.sh
-      - post-steps
-
-workflows:
-  version: 2
-  build-linux:
-    jobs:
-      - build-linux
-  build-linux-cmake:
-    jobs:
-      - build-linux-cmake
-      - build-linux-cmake-ubuntu-20
-  build-linux-mem-env:
-    jobs:
-      - build-linux-mem-env
-  build-linux-encrypted-env:
-    jobs:
-      - build-linux-encrypted-env
-  build-linux-shared_lib-alt_namespace-status_checked:
-    jobs:
-      - build-linux-shared_lib-alt_namespace-status_checked
-  build-linux-lite:
-    jobs:
-      - build-linux-lite
-  build-linux-release:
-    jobs:
-      - build-linux-release
-  build-linux-release-rtti:
-    jobs:
-      - build-linux-release-rtti
-  build-linux-lite-release:
-    jobs:
-      - build-linux-lite-release
-  build-linux-clang10-asan:
-    jobs:
-      - build-linux-clang10-asan
-  build-linux-clang10-mini-tsan:
-    jobs:
-      - build-linux-clang10-mini-tsan
-  build-linux-clang10-ubsan:
-    jobs:
-      - build-linux-clang10-ubsan
-  build-linux-clang10-clang-analyze:
-    jobs:
-      - build-linux-clang10-clang-analyze
-  build-linux-unity:
-    jobs:
-      - build-linux-unity
-  build-windows-vs2019:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2019"
-  build-windows-vs2019-cxx20:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2019-cxx20"
-          extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
-  build-windows-vs2017:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2017"
-          vs_year: "2017"
-          cmake_generator: "Visual Studio 15 Win64"
-  build-java:
-    jobs:
-      - build-linux-java
-      - build-linux-java-static
-      - build-macos-java
-      - build-macos-java-static
-  build-examples:
-    jobs:
-      - build-examples
-  build-linux-non-shm:
-    jobs:
-      - build-linux-non-shm:
-          start_test: ""
-          end_test: "db_options_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "db_options_test" # make sure unique in src.mk
-          end_test: "filename_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "filename_test" # make sure unique in src.mk
-          end_test: "statistics_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "statistics_test" # make sure unique in src.mk
-          end_test: ""
-  build-linux-compilers-no_test_run:
-    jobs:
-      - build-linux-clang-no_test_run
-      - build-linux-gcc-4_8-no_test_run
-      - build-linux-gcc-8-no_test_run
-      - build-linux-gcc-9-no_test_run
-      - build-linux-gcc-10-cxx20-no_test_run
-      - build-linux-arm-cmake-no_test_run
-  build-macos:
-    jobs:
-      - build-macos
-  build-macos-cmake:
-    jobs:
-      - build-macos-cmake
-  build-cmake-mingw:
-    jobs:
-      - build-cmake-mingw
-  build-linux-arm:
-    jobs:
-      - build-linux-arm
-  build-microbench:
-    jobs:
-      - build-linux-microbench
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - build-format-compatible
-      - build-linux-arm-test-full
+            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,env_basic_test,env_test,db_merge_operand_test -Concurrency 16

.circleci/ubsan_suppression_list.txt

@@ -1,6 +0,0 @@
-# Supress UBSAN warnings related to stl_tree.h, e.g.
-# UndefinedBehaviorSanitizer: undefined-behavior /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43 in 
-# /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43:
-# runtime error: upcast of address 0x000001fa8820 with insufficient space for an object of type
-# 'std::_Rb_tree_node<std::pair<const std::__cxx11::basic_string<char>, rocksdb::(anonymous namespace)::LockHoldingInfo> >'
-src:*bits/stl_tree.h

.circleci/vs2015_install.ps1

@@ -1,24 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://go.microsoft.com/fwlink/?LinkId=691126"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2015 installer failed"
-    exit 1
-}
-$VS_INSTALL_ARGS = @("/Quiet", "/NoRestart")
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2015 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}
-echo "VS 2015 installed."

.circleci/vs2017_install.ps1

@@ -1,35 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.13",
-                                                     "--add Microsoft.Component.MSBuild",
-                                                     "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
-                                                     "--add Microsoft.VisualStudio.Component.TextTemplating",
-                                                     "--add Microsoft.VisualStudio.Component.VC.CoreIde",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
-
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2017 installer failed"
-    exit 1
-}
-
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}
-echo "VS 2017 installed."

.github/workflows/sanity_check.yml

@@ -1,41 +0,0 @@
-name: Check buck targets and code format
-on: [push, pull_request]
-jobs:
-  check:
-    name: Check TARGETS file and code format
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout feature branch
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: Fetch from upstream
-      run: |
-        git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream
-
-    - name: Where am I
-      run: |
-        echo git status && git status
-        echo "git remote -v" && git remote -v
-        echo git branch && git branch
-
-    - name: Setup Python
-      uses: actions/setup-python@v1
-
-    - name: Install Dependencies
-      run: python -m pip install --upgrade pip
-
-    - name: Install argparse
-      run: pip install argparse
-
-    - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py
-
-    - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
-
-    - name: Compare buckify output
-      run: make check-buck-targets

.gitignore

@@ -1,5 +1,4 @@
 make_config.mk
-rocksdb.pc
 
 *.a
 *.arc
@@ -8,7 +7,6 @@ rocksdb.pc
 *.gcda
 *.gcno
 *.o
-*.o.tmp
 *.so
 *.so.*
 *_test
@@ -54,7 +52,6 @@ db_test2
 trace_analyzer
 trace_analyzer_test
 block_cache_trace_analyzer
-io_tracer_parser
 .DS_Store
 .vs
 .vscode
@@ -88,10 +85,3 @@ buckifier/*.pyc
 buckifier/__pycache__
 
 compile_commands.json
-clang-format-diff.py
-.py3/
-
-fuzz/proto/gen/
-fuzz/crash-*
-
-cmake-build-*

.travis.yml

@@ -2,18 +2,20 @@ dist: xenial
 language: cpp
 os:
   - linux
+  - osx
 arch:
+  - amd64
   - arm64
   - ppc64le
 compiler:
   - clang
   - gcc
+osx_image: xcode9.4
 cache:
   - ccache
 
 addons:
   apt:
-    update: true
     sources:
       - ubuntu-toolchain-r-test
     packages:
@@ -24,6 +26,15 @@ addons:
       - liblzma-dev  # xv
       - libzstd-dev
       - zlib1g-dev
+  homebrew:
+    update: true
+    packages:
+      - ccache
+      - gflags
+      - lz4
+      - snappy
+      - xz
+      - zstd
 
 env:
   - TEST_GROUP=platform_dependent # 16-18 minutes
@@ -39,156 +50,136 @@ env:
   - JOB_NAME=examples # 5-7 minutes
   - JOB_NAME=cmake # 3-5 minutes
   - JOB_NAME=cmake-gcc8 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
   - JOB_NAME=cmake-mingw # 3 minutes
-  - JOB_NAME=make-gcc4.8
-  - JOB_NAME=status_checked
 
 matrix:
   exclude:
+  - os: osx
+    env: JOB_NAME=cmake-gcc8
+  - os: osx
+    env: JOB_NAME=cmake-mingw
+  - os: osx
+    arch: ppc64le
+  - os: osx
+    compiler: gcc
   - os : linux
     arch: arm64
     env: JOB_NAME=cmake-mingw
-  - os : linux
-    arch: arm64
-    env: JOB_NAME=make-gcc4.8
   - os: linux
     arch: ppc64le
     env: JOB_NAME=cmake-mingw
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=make-gcc4.8
   - os: linux
     compiler: clang
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: arm64
-    env: TEST_GROUP=platform_dependent
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  # Exclude most osx, arm64 and ppc64le tests for pull requests, but build in branches
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=1
+  - if: type != pull_request
     os : linux
     arch: arm64
     env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os: linux
     arch: ppc64le
     env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=2
+  - if: type != pull_request
     os : linux
     arch: arm64
     env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os: linux
     arch: ppc64le
     env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=3
+  - if: type != pull_request
     os : linux
     arch: arm64
     env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os: linux
     arch: ppc64le
     env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
+    os: osx
+    env: TEST_GROUP=4
+  - if: type != pull_request
     os : linux
     arch: arm64
     env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os: linux
     arch: ppc64le
     env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os : linux
     arch: arm64
-    env: JOB_NAME=cmake
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
+    env: JOB_NAME=java-test
+  - if: type != pull_request
     os: linux
     arch: ppc64le
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+    env: JOB_NAME=java-test
+  - if: type != pull_request
     os : linux
     arch: arm64
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+    env: JOB_NAME=lite-build
+  - if: type != pull_request
     os: linux
     arch: ppc64le
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+    env: JOB_NAME=lite-build
+  - if: type != pull_request
     os : linux
     arch: arm64
     env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os: linux
     arch: ppc64le
     env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os : linux
     arch: arm64
     env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
+  - if: type != pull_request
     os: linux
     arch: ppc64le
     env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=status_checked
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=status_checked
 
 install:
+  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
+      PATH=$PATH:/usr/local/opt/ccache/libexec;
+    fi
   - if [ "${JOB_NAME}" == cmake-gcc8 ]; then
-      sudo apt-get install -y g++-8 || exit $?;
+      sudo apt-get install -y g++-8;
       CC=gcc-8 && CXX=g++-8;
     fi
-  - if [ "${JOB_NAME}" == cmake-gcc9 ] || [ "${JOB_NAME}" == cmake-gcc9-c++20 ]; then
-      sudo apt-get install -y g++-9 || exit $?;
-      CC=gcc-9 && CXX=g++-9;
-    fi
   - if [ "${JOB_NAME}" == cmake-mingw ]; then
-      sudo apt-get install -y mingw-w64 || exit $?;
+      sudo apt-get install -y mingw-w64 ;
     fi
-  - if [ "${JOB_NAME}" == make-gcc4.8 ]; then
-      sudo apt-get install -y g++-4.8 || exit $?;
-      CC=gcc-4.8 && CXX=g++-4.8;
-    fi
-  - |
-    if [[ "${JOB_NAME}" == cmake* ]]; then
-      sudo apt-get remove -y cmake cmake-data
-      export CMAKE_DEB="cmake-3.14.5-Linux-$(uname -m).deb"
-      export CMAKE_DEB_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/${CMAKE_DEB}"
-      curl --silent --fail --show-error --location --output "${CMAKE_DEB}" "${CMAKE_DEB_URL}" || exit $?
-      sudo dpkg -i "${CMAKE_DEB}" || exit $?
-      which cmake && cmake --version
+  - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
+      CMAKE_DIST_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/cmake-3.14.5-Linux-$(uname -m).tar.bz2";
+      TAR_OPT="--strip-components=1 -xj";
+      if [ "aarch64" == "$(uname -m)" ]; then
+        sudo apt-get install -y libuv1 librhash0;
+        sudo apt-get upgrade -y libstdc++6;
+      fi;
+      mkdir cmake-dist && curl --silent --fail --show-error --location "${CMAKE_DIST_URL}" | tar -C cmake-dist ${TAR_OPT} && export PATH=$PWD/cmake-dist/bin:$PATH;
     fi
   - |
     if [[ "${JOB_NAME}" == java_test || "${JOB_NAME}" == cmake* ]]; then
       # Ensure JDK 8
-      sudo apt-get install -y openjdk-8-jdk || exit $?
-      export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
-      export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
+      if [ "${TRAVIS_OS_NAME}" == osx ]; then
+        brew tap AdoptOpenJDK/openjdk
+        brew cask install adoptopenjdk8
+        export JAVA_HOME=$(/usr/libexec/java_home)
+      else
+        sudo apt-get install -y openjdk-8-jdk
+        export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
+        export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
+      fi
       echo "JAVA_HOME=${JAVA_HOME}"
       which java && java -version
       which javac && javac -version
@@ -204,49 +195,37 @@ script:
   - if [ `command -v ccache` ]; then ccache -C; fi
   - case $TEST_GROUP in
     platform_dependent)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j4 all_but_some_tests check_some
+      OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some
       ;;
     1)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backupable_db_test make -j4 check_some
+      OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=db_iter_test make -j4 check_some
       ;;
     2)
-      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j4 tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backupable_db_test ROCKSDBTESTS_END=db_universal_compaction_test make -j4 check_some
+      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" V=1 make -j4 tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" V=1 ROCKSDBTESTS_START=db_iter_test ROCKSDBTESTS_END=options_file_test make -j4 check_some
       ;;
     3)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j4 check_some
+      OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=options_file_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some
       ;;
     4)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=table_properties_collector_test make -j4 check_some
+      OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some
       ;;
     esac
   - case $JOB_NAME in
     java_test)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make rocksdbjava jtest
+      OPT=-DTRAVIS V=1 make rocksdbjava jtest
       ;;
     lite_build)
-      OPT='-DTRAVIS -DROCKSDB_LITE' LIB_MODE=shared V=1 make -j4 all
+      OPT='-DTRAVIS -DROCKSDB_LITE' V=1 make -j4 static_lib tools
       ;;
     examples)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make -j4 static_lib && cd examples && make -j4
+      OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4
       ;;
     cmake-mingw)
       sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
       mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
       ;;
     cmake*)
-      case $JOB_NAME in
-        *-c++20)
-          OPT=-DCMAKE_CXX_STANDARD=20
-          ;;
-      esac
-
-      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j4 && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
-      ;;
-    make-gcc4.8)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 SKIP_LINK=1 make -j4 all && [ "Linking broken because libgflags compiled with newer ABI" ]
-      ;;
-    status_checked)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ASSERT_STATUS_CHECKED=1 make -j4 check_some
+      mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release && make -j4 rocksdb rocksdbjni
       ;;
     esac
 notifications:

CMakeLists.txt

@@ -32,11 +32,10 @@
 # 3. cmake ..
 # 4. make -j
 
-cmake_minimum_required(VERSION 3.10)
+cmake_minimum_required(VERSION 3.5.1)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake/modules/")
 include(ReadVersion)
-include(GoogleTest)
 get_rocksdb_version(rocksdb_VERSION)
 project(rocksdb
   VERSION ${rocksdb_VERSION}
@@ -63,7 +62,6 @@ if(CCACHE_FOUND)
 endif(CCACHE_FOUND)
 
 option(WITH_JEMALLOC "build with JeMalloc" OFF)
-option(WITH_LIBURING "build with liburing" ON)
 option(WITH_SNAPPY "build with SNAPPY" OFF)
 option(WITH_LZ4 "build with lz4" OFF)
 option(WITH_ZLIB "build with zlib" OFF)
@@ -72,12 +70,6 @@ option(WITH_WINDOWS_UTF8_FILENAMES "use UTF8 as characterset for opening files,
 if (WITH_WINDOWS_UTF8_FILENAMES)
   add_definitions(-DROCKSDB_WINDOWS_UTF8_FILENAMES)
 endif()
-
-if ($ENV{CIRCLECI})
-  message(STATUS "Build for CircieCI env, a few tests may be disabled")
-  add_definitions(-DCIRCLECI)
-endif()
-
 # third-party/folly is only validated to work on Linux and Windows for now.
 # So only turn it on there by default.
 if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
@@ -91,18 +83,15 @@ else()
   option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
 endif()
 
-if( NOT DEFINED CMAKE_CXX_STANDARD )
-  set(CMAKE_CXX_STANDARD 11)
-endif()
-
 include(CMakeDependentOption)
+CMAKE_DEPENDENT_OPTION(WITH_GFLAGS "build with GFlags" ON
+  "NOT MSVC;NOT MINGW" OFF)
 
 if(MSVC)
-  option(WITH_GFLAGS "build with GFlags" OFF)
   option(WITH_XPRESS "build with windows built in compression" OFF)
   include(${CMAKE_CURRENT_SOURCE_DIR}/thirdparty.inc)
 else()
-  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD" AND NOT CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
+  if(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
     # FreeBSD has jemalloc as default malloc
     # but it does not have all the jemalloc files in include/...
     set(WITH_JEMALLOC ON)
@@ -114,40 +103,18 @@ else()
     endif()
   endif()
 
-  if(MINGW)
-    option(WITH_GFLAGS "build with GFlags" OFF)
-  else()
-    option(WITH_GFLAGS "build with GFlags" ON)
-  endif()
-  set(GFLAGS_LIB)
+  # No config file for this
   if(WITH_GFLAGS)
-    # Config with namespace available since gflags 2.2.2
-    option(GFLAGS_USE_TARGET_NAMESPACE "Use gflags import target with namespace." ON)
-    find_package(gflags CONFIG)
-    if(gflags_FOUND)
-      if(TARGET ${GFLAGS_TARGET})
-        # Config with GFLAGS_TARGET available since gflags 2.2.0
-        set(GFLAGS_LIB ${GFLAGS_TARGET})
-      else()
-        # Config with GFLAGS_LIBRARIES available since gflags 2.1.0
-        set(GFLAGS_LIB ${gflags_LIBRARIES})
-      endif()
-    else()
-      find_package(gflags REQUIRED)
-      set(GFLAGS_LIB gflags::gflags)
-    endif()
-    include_directories(${GFLAGS_INCLUDE_DIR})
-    list(APPEND THIRDPARTY_LIBS ${GFLAGS_LIB})
+    find_package(gflags REQUIRED)
     add_definitions(-DGFLAGS=1)
+    include_directories(${gflags_INCLUDE_DIR})
+    list(APPEND THIRDPARTY_LIBS gflags::gflags)
   endif()
 
   if(WITH_SNAPPY)
-    find_package(Snappy CONFIG)
-    if(NOT Snappy_FOUND)
-      find_package(Snappy REQUIRED)
-    endif()
+    find_package(snappy REQUIRED)
     add_definitions(-DSNAPPY)
-    list(APPEND THIRDPARTY_LIBS Snappy::snappy)
+    list(APPEND THIRDPARTY_LIBS snappy::snappy)
   endif()
 
   if(WITH_ZLIB)
@@ -182,25 +149,23 @@ else()
   endif()
 endif()
 
-string(TIMESTAMP TS "%Y-%m-%d %H:%M:%S" UTC)
-set(BUILD_DATE "${TS}" CACHE STRING "the time we first built rocksdb")
+string(TIMESTAMP TS "%Y/%m/%d %H:%M:%S" UTC)
+set(GIT_DATE_TIME "${TS}" CACHE STRING "the time we first built rocksdb")
 
 find_package(Git)
 
 if(GIT_FOUND AND EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git")
-  execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_SHA COMMAND "${GIT_EXECUTABLE}" rev-parse HEAD )
-  execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" RESULT_VARIABLE GIT_MOD COMMAND "${GIT_EXECUTABLE}" diff-index HEAD --quiet)
-  execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_DATE COMMAND "${GIT_EXECUTABLE}" log -1 --date=format:"%Y-%m-%d %T" --format="%ad")
-  execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG RESULT_VARIABLE rv COMMAND "${GIT_EXECUTABLE}" symbolic-ref -q --short HEAD OUTPUT_STRIP_TRAILING_WHITESPACE)
-  if (rv AND NOT rv EQUAL 0)
-    execute_process(WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" OUTPUT_VARIABLE GIT_TAG COMMAND "${GIT_EXECUTABLE}" describe --tags --exact-match OUTPUT_STRIP_TRAILING_WHITESPACE)
+  if(WIN32)
+    execute_process(COMMAND $ENV{COMSPEC} /C ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
+  else()
+    execute_process(COMMAND ${GIT_EXECUTABLE} -C ${CMAKE_CURRENT_SOURCE_DIR} rev-parse HEAD OUTPUT_VARIABLE GIT_SHA)
   endif()
 else()
   set(GIT_SHA 0)
-  set(GIT_MOD 1)
 endif()
-string(REGEX REPLACE "[^0-9a-fA-F]+" "" GIT_SHA "${GIT_SHA}")
-string(REGEX REPLACE "[^0-9: /-]+" "" GIT_DATE "${GIT_DATE}")
+
+string(REGEX REPLACE "[^0-9a-f]+" "" GIT_SHA "${GIT_SHA}")
+
 
 option(WITH_MD_LIBRARY "build with MD" ON)
 if(WIN32 AND MSVC)
@@ -213,20 +178,20 @@ endif()
 
 set(BUILD_VERSION_CC ${CMAKE_BINARY_DIR}/build_version.cc)
 configure_file(util/build_version.cc.in ${BUILD_VERSION_CC} @ONLY)
-
+add_library(build_version OBJECT ${BUILD_VERSION_CC})
+target_include_directories(build_version PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/util)
 if(MSVC)
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /Zi /nologo /EHsc /GS /Gd /GR /GF /fp:precise /Zc:wchar_t /Zc:forScope /errorReport:queue")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /FC /d2Zi+ /W4 /wd4127 /wd4800 /wd4996 /wd4351 /wd4100 /wd4204 /wd4324")
 else()
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall -pthread")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -W -Wextra -Wall")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wsign-compare -Wshadow -Wno-unused-parameter -Wno-unused-variable -Woverloaded-virtual -Wnon-virtual-dtor -Wno-missing-field-initializers -Wno-strict-aliasing")
-  if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wstrict-prototypes")
-  endif()
   if(MINGW)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-format -fno-asynchronous-unwind-tables")
     add_definitions(-D_POSIX_C_SOURCE=1)
   endif()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
   if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
     include(CheckCXXCompilerFlag)
@@ -258,41 +223,23 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
   endif(HAS_ALTIVEC)
 endif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
 
-if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
+if(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
         CHECK_C_COMPILER_FLAG("-march=armv8-a+crc+crypto" HAS_ARMV8_CRC)
   if(HAS_ARMV8_CRC)
     message(STATUS " HAS_ARMV8_CRC yes")
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
   endif(HAS_ARMV8_CRC)
-endif(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
+endif(CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64|AARCH64")
 
 option(PORTABLE "build a portable binary" OFF)
 option(FORCE_SSE42 "force building with SSE4.2, even when PORTABLE=ON" OFF)
-option(FORCE_AVX "force building with AVX, even when PORTABLE=ON" OFF)
-option(FORCE_AVX2 "force building with AVX2, even when PORTABLE=ON" OFF)
 if(PORTABLE)
   # MSVC does not need a separate compiler flag to enable SSE4.2; if nmmintrin.h
   # is available, it is available by default.
   if(FORCE_SSE42 AND NOT MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -msse4.2 -mpclmul")
   endif()
-  if(MSVC)
-    if(FORCE_AVX)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX")
-    endif()
-    # MSVC automatically enables BMI / lzcnt with AVX2.
-    if(FORCE_AVX2)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
-    endif()
-  else()
-    if(FORCE_AVX)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx")
-    endif()
-    if(FORCE_AVX2)
-      set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -mavx2 -mbmi -mlzcnt")
-    endif()
-  endif()
 else()
   if(MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
@@ -304,11 +251,9 @@ else()
 endif()
 
 include(CheckCXXSourceCompiles)
-set(OLD_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS})
 if(NOT MSVC)
   set(CMAKE_REQUIRED_FLAGS "-msse4.2 -mpclmul")
 endif()
-
 CHECK_CXX_SOURCE_COMPILES("
 #include <cstdint>
 #include <nmmintrin.h>
@@ -321,6 +266,7 @@ int main() {
   auto d = _mm_cvtsi128_si64(c);
 }
 " HAVE_SSE42)
+unset(CMAKE_REQUIRED_FLAGS)
 if(HAVE_SSE42)
   add_definitions(-DHAVE_SSE42)
   add_definitions(-DHAVE_PCLMUL)
@@ -328,66 +274,18 @@ elseif(FORCE_SSE42)
   message(FATAL_ERROR "FORCE_SSE42=ON but unable to compile with SSE4.2 enabled")
 endif()
 
-# Check if -latomic is required or not
-if (NOT MSVC)
-  set(CMAKE_REQUIRED_FLAGS "--std=c++11")
-  CHECK_CXX_SOURCE_COMPILES("
-#include <atomic>
-std::atomic<uint64_t> x(0);
-int main() {
-  uint64_t i = x.load(std::memory_order_relaxed);
-  bool b = x.is_lock_free();
-  return 0;
-}
-" BUILTIN_ATOMIC)
-if (NOT BUILTIN_ATOMIC)
-  #TODO: Check if -latomic exists
-  list(APPEND THIRDPARTY_LIBS atomic)
-endif()
-endif()
-
-if (WITH_LIBURING)
-  set(CMAKE_REQUIRED_FLAGS "-luring")
-  CHECK_CXX_SOURCE_COMPILES("
-#include <liburing.h>
-int main() {
-  struct io_uring ring;
-  io_uring_queue_init(1, &ring, 0);
-  return 0;
-}
-" HAS_LIBURING)
-  if (HAS_LIBURING)
-    add_definitions(-DROCKSDB_IOURING_PRESENT)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -luring")
-  endif()
-endif()
-
-# Reset the required flags
-set(CMAKE_REQUIRED_FLAGS ${OLD_CMAKE_REQUIRED_FLAGS})
-
 CHECK_CXX_SOURCE_COMPILES("
 #if defined(_MSC_VER) && !defined(__thread)
 #define __thread __declspec(thread)
 #endif
 int main() {
   static __thread int tls;
-  (void)tls;
 }
 " HAVE_THREAD_LOCAL)
 if(HAVE_THREAD_LOCAL)
   add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
 endif()
 
-option(WITH_IOSTATS_CONTEXT "Enable IO stats context" ON)
-if (NOT WITH_IOSTATS_CONTEXT)
-  add_definitions(-DNIOSTATS_CONTEXT)
-endif()
-
-option(WITH_PERF_CONTEXT "Enable perf context" ON)
-if (NOT WITH_PERF_CONTEXT)
-  add_definitions(-DNPERF_CONTEXT)
-endif()
-
 option(FAIL_ON_WARNINGS "Treat compile warnings as errors" ON)
 if(FAIL_ON_WARNINGS)
   if(MSVC)
@@ -454,12 +352,6 @@ if(NOT WITH_DYNAMIC_EXTENSION)
   add_definitions(-DROCKSDB_NO_DYNAMIC_EXTENSION)
 endif()
 
-option(ASSERT_STATUS_CHECKED "build with assert status checked" OFF)
-if (ASSERT_STATUS_CHECKED)
-  message(STATUS "Build with assert status checked")
-  add_definitions(-DROCKSDB_ASSERT_STATUS_CHECKED)
-endif()
-
 if(DEFINED USE_RTTI)
   if(USE_RTTI)
     message(STATUS "Enabling RTTI")
@@ -494,15 +386,7 @@ if(MSVC)
     message(STATUS "Debug optimization is enabled")
     set(CMAKE_CXX_FLAGS_DEBUG "/Oxt")
   else()
-    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1")
-
-    # Minimal Build is deprecated after MSVC 2015
-    if( MSVC_VERSION GREATER 1900 )
-      set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm-")
-    else()
-      set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Gm")
-    endif()
-
+    set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /Od /RTC1 /Gm")
   endif()
   if(WITH_RUNTIME_DEBUG)
     set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} /${RUNTIME_LIBRARY}d")
@@ -529,12 +413,15 @@ if(CMAKE_SYSTEM_NAME MATCHES "Cygwin")
   add_definitions(-fno-builtin-memcmp -DCYGWIN)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
+  if(CMAKE_SYSTEM_PROCESSOR MATCHES arm)
+    add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE)
+    # no debug info for IOS, that will make our library big
+    add_definitions(-DNDEBUG)
+  endif()
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
   add_definitions(-DOS_LINUX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS")
   add_definitions(-DOS_SOLARIS)
-elseif(CMAKE_SYSTEM_NAME MATCHES "kFreeBSD")
-  add_definitions(-DOS_GNU_KFREEBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "FreeBSD")
   add_definitions(-DOS_FREEBSD)
 elseif(CMAKE_SYSTEM_NAME MATCHES "NetBSD")
@@ -593,11 +480,7 @@ if(HAVE_PTHREAD_MUTEX_ADAPTIVE_NP)
 endif()
 
 include(CheckCXXSymbolExists)
-if(CMAKE_SYSTEM_NAME MATCHES "^FreeBSD")
-  check_cxx_symbol_exists(malloc_usable_size malloc_np.h HAVE_MALLOC_USABLE_SIZE)
-else()
-  check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE)
-endif()
+check_cxx_symbol_exists(malloc_usable_size malloc.h HAVE_MALLOC_USABLE_SIZE)
 if(HAVE_MALLOC_USABLE_SIZE)
   add_definitions(-DROCKSDB_MALLOC_USABLE_SIZE)
 endif()
@@ -614,6 +497,7 @@ endif()
 
 include_directories(${PROJECT_SOURCE_DIR})
 include_directories(${PROJECT_SOURCE_DIR}/include)
+include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
 if(WITH_FOLLY_DISTRIBUTED_MUTEX)
   include_directories(${PROJECT_SOURCE_DIR}/third-party/folly)
 endif()
@@ -622,26 +506,16 @@ find_package(Threads REQUIRED)
 # Main library source code
 
 set(SOURCES
-        cache/cache.cc
-        cache/cache_entry_roles.cc
         cache/clock_cache.cc
         cache/lru_cache.cc
         cache/sharded_cache.cc
         db/arena_wrapped_db_iter.cc
-        db/blob/blob_fetcher.cc
         db/blob/blob_file_addition.cc
-        db/blob/blob_file_builder.cc
-        db/blob/blob_file_cache.cc
         db/blob/blob_file_garbage.cc
-        db/blob/blob_file_meta.cc
-        db/blob/blob_file_reader.cc
-        db/blob/blob_garbage_meter.cc
-        db/blob/blob_log_format.cc
-        db/blob/blob_log_sequential_reader.cc
-        db/blob/blob_log_writer.cc
         db/builder.cc
         db/c.cc
         db/column_family.cc
+        db/compacted_db_impl.cc
         db/compaction/compaction.cc
         db/compaction/compaction_iterator.cc
         db/compaction/compaction_picker.cc
@@ -649,10 +523,8 @@ set(SOURCES
         db/compaction/compaction_picker_fifo.cc
         db/compaction/compaction_picker_level.cc
         db/compaction/compaction_picker_universal.cc
-        db/compaction/sst_partitioner.cc
         db/convenience.cc
         db/db_filesnapshot.cc
-        db/db_impl/compacted_db_impl.cc
         db/db_impl/db_impl.cc
         db/db_impl/db_impl_write.cc
         db/db_impl/db_impl_compaction_flush.cc
@@ -683,8 +555,6 @@ set(SOURCES
         db/memtable_list.cc
         db/merge_helper.cc
         db/merge_operator.cc
-        db/output_validator.cc
-        db/periodic_work_scheduler.cc
         db/range_del_aggregator.cc
         db/range_tombstone_fragmenter.cc
         db/repair.cc
@@ -695,28 +565,22 @@ set(SOURCES
         db/trim_history_scheduler.cc
         db/version_builder.cc
         db/version_edit.cc
-        db/version_edit_handler.cc
         db/version_set.cc
-        db/wal_edit.cc
         db/wal_manager.cc
         db/write_batch.cc
         db/write_batch_base.cc
         db/write_controller.cc
         db/write_thread.cc
-        env/composite_env.cc
         env/env.cc
         env/env_chroot.cc
         env/env_encryption.cc
         env/env_hdfs.cc
         env/file_system.cc
-        env/file_system_tracer.cc
-        env/fs_remap.cc
         env/mock_env.cc
         file/delete_scheduler.cc
         file/file_prefetch_buffer.cc
         file/file_util.cc
         file/filename.cc
-        file/line_file_reader.cc
         file/random_access_file_reader.cc
         file/read_write_util.cc
         file/readahead_raf.cc
@@ -729,7 +593,6 @@ set(SOURCES
         memory/arena.cc
         memory/concurrent_arena.cc
         memory/jemalloc_nodump_allocator.cc
-        memory/memkind_kmem_allocator.cc
         memtable/alloc_tracker.cc
         memtable/hash_linklist_rep.cc
         memtable/hash_skiplist_rep.cc
@@ -750,12 +613,11 @@ set(SOURCES
         monitoring/thread_status_util.cc
         monitoring/thread_status_util_debug.cc
         options/cf_options.cc
-        options/configurable.cc
-        options/customizable.cc
         options/db_options.cc
         options/options.cc
         options/options_helper.cc
         options/options_parser.cc
+        options/options_sanity_check.cc
         port/stack_trace.cc
         table/adaptive/adaptive_table_factory.cc
         table/block_based/binary_search_index_reader.cc
@@ -799,10 +661,8 @@ set(SOURCES
         table/plain/plain_table_index.cc
         table/plain/plain_table_key_coding.cc
         table/plain/plain_table_reader.cc
-        table/sst_file_dumper.cc
         table/sst_file_reader.cc
         table/sst_file_writer.cc
-        table/table_factory.cc
         table/table_properties.cc
         table/two_level_iterator.cc
         test_util/sync_point.cc
@@ -811,14 +671,12 @@ set(SOURCES
         test_util/transaction_test_util.cc
         tools/block_cache_analyzer/block_cache_trace_analyzer.cc
         tools/dump/db_dump_tool.cc
-        tools/io_tracer_parser_tool.cc
         tools/ldb_cmd.cc
         tools/ldb_tool.cc
         tools/sst_dump_tool.cc
         tools/trace_analyzer_tool.cc
         trace_replay/trace_replay.cc
         trace_replay/block_cache_tracer.cc
-        trace_replay/io_tracer.cc
         util/coding.cc
         util/compaction_job_stats_impl.cc
         util/comparator.cc
@@ -830,7 +688,6 @@ set(SOURCES
         util/murmurhash.cc
         util/random.cc
         util/rate_limiter.cc
-        util/ribbon_config.cc
         util/slice.cc
         util/file_checksum_helper.cc
         util/status.cc
@@ -845,6 +702,9 @@ set(SOURCES
         utilities/blob_db/blob_db_impl_filesnapshot.cc
         utilities/blob_db/blob_dump_tool.cc
         utilities/blob_db/blob_file.cc
+        utilities/blob_db/blob_log_reader.cc
+        utilities/blob_db/blob_log_writer.cc
+        utilities/blob_db/blob_log_format.cc
         utilities/cassandra/cassandra_compaction_filter.cc
         utilities/cassandra/format.cc
         utilities/cassandra/merge_operator.cc
@@ -853,8 +713,6 @@ set(SOURCES
         utilities/debug.cc
         utilities/env_mirror.cc
         utilities/env_timed.cc
-        utilities/fault_injection_env.cc
-        utilities/fault_injection_fs.cc
         utilities/leveldb_options/leveldb_options.cc
         utilities/memory/memory_util.cc
         utilities/merge_operators/bytesxor.cc
@@ -876,11 +734,6 @@ set(SOURCES
         utilities/simulator_cache/sim_cache.cc
         utilities/table_properties_collectors/compact_on_deletion_collector.cc
         utilities/trace/file_trace_reader_writer.cc
-        utilities/transactions/lock/lock_manager.cc
-        utilities/transactions/lock/point/point_lock_tracker.cc
-        utilities/transactions/lock/point/point_lock_manager.cc
-        utilities/transactions/lock/range/range_tree/range_tree_lock_manager.cc
-        utilities/transactions/lock/range/range_tree/range_tree_lock_tracker.cc
         utilities/transactions/optimistic_transaction_db_impl.cc
         utilities/transactions/optimistic_transaction.cc
         utilities/transactions/pessimistic_transaction.cc
@@ -888,6 +741,7 @@ set(SOURCES
         utilities/transactions/snapshot_checker.cc
         utilities/transactions/transaction_base.cc
         utilities/transactions/transaction_db_mutex_impl.cc
+        utilities/transactions/transaction_lock_mgr.cc
         utilities/transactions/transaction_util.cc
         utilities/transactions/write_prepared_txn.cc
         utilities/transactions/write_prepared_txn_db.cc
@@ -895,21 +749,8 @@ set(SOURCES
         utilities/transactions/write_unprepared_txn_db.cc
         utilities/ttl/db_ttl_impl.cc
         utilities/write_batch_with_index/write_batch_with_index.cc
-        utilities/write_batch_with_index/write_batch_with_index_internal.cc)
-
-list(APPEND SOURCES
-  utilities/transactions/lock/range/range_tree/lib/locktree/concurrent_tree.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/keyrange.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/lock_request.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/locktree.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/manager.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/range_buffer.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/treenode.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/txnid_set.cc
-  utilities/transactions/lock/range/range_tree/lib/locktree/wfg.cc
-  utilities/transactions/lock/range/range_tree/lib/standalone_port.cc
-  utilities/transactions/lock/range/range_tree/lib/util/dbt.cc
-  utilities/transactions/lock/range/range_tree/lib/util/memarena.cc)
+        utilities/write_batch_with_index/write_batch_with_index_internal.cc
+        $<TARGET_OBJECTS:build_version>)
 
 if(HAVE_SSE42 AND NOT MSVC)
   set_source_files_properties(
@@ -936,6 +777,7 @@ if(WIN32)
     port/win/port_win.cc
     port/win/win_logger.cc
     port/win/win_thread.cc)
+
 if(WITH_XPRESS)
   list(APPEND SOURCES
     port/win/xpress_win.cc)
@@ -981,13 +823,13 @@ else()
   set(SYSTEM_LIBS ${CMAKE_THREAD_LIBS_INIT})
 endif()
 
-add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES} ${BUILD_VERSION_CC})
-target_link_libraries(${ROCKSDB_STATIC_LIB} PRIVATE
+add_library(${ROCKSDB_STATIC_LIB} STATIC ${SOURCES})
+target_link_libraries(${ROCKSDB_STATIC_LIB}
   ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
 if(ROCKSDB_BUILD_SHARED)
-  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES} ${BUILD_VERSION_CC})
-  target_link_libraries(${ROCKSDB_SHARED_LIB} PRIVATE
+  add_library(${ROCKSDB_SHARED_LIB} SHARED ${SOURCES})
+  target_link_libraries(${ROCKSDB_SHARED_LIB}
     ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
   if(WIN32)
@@ -1004,7 +846,8 @@ if(ROCKSDB_BUILD_SHARED)
                           LINKER_LANGUAGE CXX
                           VERSION ${rocksdb_VERSION}
                           SOVERSION ${rocksdb_VERSION_MAJOR}
-                          OUTPUT_NAME "rocksdb${ARTIFACT_SUFFIX}")
+                          CXX_STANDARD 11
+                          OUTPUT_NAME "rocksdb")
   endif()
 endif()
 
@@ -1015,16 +858,6 @@ else()
 endif()
 
 option(WITH_JNI "build with JNI" OFF)
-# Tests are excluded from Release builds
-CMAKE_DEPENDENT_OPTION(WITH_TESTS "build with tests" ON
-  "CMAKE_BUILD_TYPE STREQUAL Debug" OFF)
-option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON)
-option(WITH_CORE_TOOLS "build with ldb and sst_dump" ON)
-option(WITH_TOOLS "build with tools" ON)
-
-if(WITH_TESTS OR WITH_BENCHMARK_TOOLS OR WITH_TOOLS OR WITH_JNI OR JNI)
-  include_directories(SYSTEM ${PROJECT_SOURCE_DIR}/third-party/gtest-1.8.1/fused-src)
-endif()
 if(WITH_JNI OR JNI)
   message(STATUS "JNI library is enabled")
   add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/java)
@@ -1062,8 +895,6 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
 
   install(DIRECTORY include/rocksdb COMPONENT devel DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
-  install(DIRECTORY "${PROJECT_SOURCE_DIR}/cmake/modules" COMPONENT devel DESTINATION ${package_config_destination})
-
   install(
     TARGETS ${ROCKSDB_STATIC_LIB}
     EXPORT RocksDBTargets
@@ -1100,47 +931,31 @@ if(NOT WIN32 OR ROCKSDB_INSTALL_ON_WINDOWS)
   )
 endif()
 
-option(WITH_ALL_TESTS "Build all test, rather than a small subset" ON)
-
-if(WITH_TESTS OR WITH_BENCHMARK_TOOLS)
+# Tests are excluded from Release builds
+CMAKE_DEPENDENT_OPTION(WITH_TESTS "build with tests" ON
+  "CMAKE_BUILD_TYPE STREQUAL Debug" OFF)
+if(WITH_TESTS)
   add_subdirectory(third-party/gtest-1.8.1/fused-src/gtest)
   add_library(testharness STATIC
-  test_util/mock_time_env.cc
   test_util/testharness.cc)
   target_link_libraries(testharness gtest)
-endif()
 
-if(WITH_TESTS)
   set(TESTS
-        db/db_basic_test.cc
-        env/env_basic_test.cc
-  )
-  if(WITH_ALL_TESTS)
-    list(APPEND TESTS
         cache/cache_test.cc
         cache/lru_cache_test.cc
-        db/blob/blob_counting_iterator_test.cc
         db/blob/blob_file_addition_test.cc
-        db/blob/blob_file_builder_test.cc
-        db/blob/blob_file_cache_test.cc
         db/blob/blob_file_garbage_test.cc
-        db/blob/blob_file_reader_test.cc
-        db/blob/blob_garbage_meter_test.cc
-        db/blob/db_blob_basic_test.cc
-        db/blob/db_blob_compaction_test.cc
-        db/blob/db_blob_corruption_test.cc
         db/blob/db_blob_index_test.cc
         db/column_family_test.cc
         db/compact_files_test.cc
-        db/compaction/clipping_iterator_test.cc
         db/compaction/compaction_job_stats_test.cc
         db/compaction/compaction_job_test.cc
         db/compaction/compaction_iterator_test.cc
         db/compaction/compaction_picker_test.cc
-        db/compaction/compaction_service_test.cc
         db/comparator_db_test.cc
         db/corruption_test.cc
         db/cuckoo_table_db_test.cc
+        db/db_basic_test.cc
         db/db_with_timestamp_basic_test.cc
         db/db_block_cache_test.cc
         db/db_bloom_filter_test.cc
@@ -1153,7 +968,6 @@ if(WITH_TESTS)
         db/db_iter_test.cc
         db/db_iter_stress_test.cc
         db/db_iterator_test.cc
-        db/db_kv_checksum_test.cc
         db/db_log_iter_test.cc
         db/db_memtable_test.cc
         db/db_merge_operator_test.cc
@@ -1161,7 +975,7 @@ if(WITH_TESTS)
         db/db_options_test.cc
         db/db_properties_test.cc
         db/db_range_del_test.cc
-        db/db_secondary_test.cc
+        db/db_impl/db_secondary_test.cc
         db/db_sst_test.cc
         db/db_statistics_test.cc
         db/db_table_properties_test.cc
@@ -1171,7 +985,6 @@ if(WITH_TESTS)
         db/db_logical_block_size_cache_test.cc
         db/db_universal_compaction_test.cc
         db/db_wal_test.cc
-        db/db_with_timestamp_compaction_test.cc
         db/db_write_test.cc
         db/dbformat_test.cc
         db/deletefile_test.cc
@@ -1191,7 +1004,6 @@ if(WITH_TESTS)
         db/merge_test.cc
         db/options_file_test.cc
         db/perf_context_test.cc
-        db/periodic_work_scheduler_test.cc
         db/plain_table_db_test.cc
         db/prefix_test.cc
         db/range_del_aggregator_test.cc
@@ -1202,21 +1014,18 @@ if(WITH_TESTS)
         db/version_edit_test.cc
         db/version_set_test.cc
         db/wal_manager_test.cc
-        db/wal_edit_test.cc
         db/write_batch_test.cc
         db/write_callback_test.cc
         db/write_controller_test.cc
+        env/env_basic_test.cc
         env/env_test.cc
         env/io_posix_test.cc
         env/mock_env_test.cc
         file/delete_scheduler_test.cc
-        file/prefetch_test.cc
-        file/random_access_file_reader_test.cc
         logging/auto_roll_logger_test.cc
         logging/env_logger_test.cc
         logging/event_logger_test.cc
         memory/arena_test.cc
-        memory/memkind_kmem_allocator_test.cc
         memtable/inlineskiplist_test.cc
         memtable/skiplist_test.cc
         memtable/write_buffer_manager_test.cc
@@ -1224,12 +1033,9 @@ if(WITH_TESTS)
         monitoring/iostats_context_test.cc
         monitoring/statistics_test.cc
         monitoring/stats_history_test.cc
-        options/configurable_test.cc
-        options/customizable_test.cc
         options/options_settable_test.cc
         options/options_test.cc
         table/block_based/block_based_filter_block_test.cc
-        table/block_based/block_based_table_reader_test.cc
         table/block_based/block_test.cc
         table/block_based/data_block_hash_index_test.cc
         table/block_based/full_filter_block_test.cc
@@ -1240,12 +1046,7 @@ if(WITH_TESTS)
         table/merger_test.cc
         table/sst_file_reader_test.cc
         table/table_test.cc
-        table/block_fetcher_test.cc
-        test_util/testutil_test.cc
-        trace_replay/block_cache_tracer_test.cc
-        trace_replay/io_tracer_test.cc
         tools/block_cache_analyzer/block_cache_trace_analyzer_test.cc
-        tools/io_tracer_parser_test.cc
         tools/ldb_cmd_test.cc
         tools/reduce_levels_test.cc
         tools/sst_dump_test.cc
@@ -1263,14 +1064,11 @@ if(WITH_TESTS)
         util/random_test.cc
         util/rate_limiter_test.cc
         util/repeatable_thread_test.cc
-        util/ribbon_test.cc
         util/slice_test.cc
         util/slice_transform_test.cc
         util/timer_queue_test.cc
-        util/timer_test.cc
         util/thread_list_test.cc
         util/thread_local_test.cc
-        util/work_queue_test.cc
         utilities/backupable/backupable_db_test.cc
         utilities/blob_db/blob_db_test.cc
         utilities/cassandra/cassandra_functional_test.cc
@@ -1290,14 +1088,11 @@ if(WITH_TESTS)
         utilities/table_properties_collectors/compact_on_deletion_collector_test.cc
         utilities/transactions/optimistic_transaction_test.cc
         utilities/transactions/transaction_test.cc
-        utilities/transactions/lock/point/point_lock_manager_test.cc
         utilities/transactions/write_prepared_transaction_test.cc
         utilities/transactions/write_unprepared_transaction_test.cc
-        utilities/transactions/lock/range/range_locking_test.cc
         utilities/ttl/ttl_test.cc
         utilities/write_batch_with_index/write_batch_with_index_test.cc
-    )
-  endif()
+  )
   if(WITH_LIBRADOS)
     list(APPEND TESTS utilities/env_librados_test.cc)
   endif()
@@ -1310,6 +1105,8 @@ if(WITH_TESTS)
       db/db_test_util.cc
       monitoring/thread_status_updater_debug.cc
       table/mock_table.cc
+      test_util/fault_injection_test_env.cc
+      test_util/fault_injection_test_fs.cc
       utilities/cassandra/test_utils.cc
   )
   enable_testing()
@@ -1324,25 +1121,21 @@ if(WITH_TESTS)
         PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
-  )
+        )
 
   foreach(sourcefile ${TESTS})
       get_filename_component(exename ${sourcefile} NAME_WE)
-      add_executable(${exename}${ARTIFACT_SUFFIX} ${sourcefile})
-      set_target_properties(${exename}${ARTIFACT_SUFFIX}
+      add_executable(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} ${sourcefile})
+      set_target_properties(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX}
         PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD_RELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_MINRELEASE 1
         EXCLUDE_FROM_DEFAULT_BUILD_RELWITHDEBINFO 1
         OUTPUT_NAME ${exename}${ARTIFACT_SUFFIX}
-      )
-      target_link_libraries(${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${THIRDPARTY_LIBS} ${ROCKSDB_LIB})
+        )
+      target_link_libraries(${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX} testutillib${ARTIFACT_SUFFIX} testharness gtest ${ROCKSDB_LIB})
       if(NOT "${exename}" MATCHES "db_sanity_test")
-        gtest_discover_tests(${exename} DISCOVERY_TIMEOUT 120)
-        add_dependencies(check ${exename}${ARTIFACT_SUFFIX})
-      endif()
-      if("${exename}" MATCHES "env_librados_test")
-        # env_librados_test.cc uses librados directly
-        target_link_libraries(${exename}${ARTIFACT_SUFFIX} rados)
+        add_test(NAME ${exename} COMMAND ${exename}${ARTIFACT_SUFFIX})
+        add_dependencies(check ${CMAKE_PROJECT_NAME}_${exename}${ARTIFACT_SUFFIX})
       endif()
   endforeach(sourcefile ${TESTS})
 
@@ -1359,53 +1152,55 @@ if(WITH_TESTS)
 
   if(ROCKSDB_LIB_FOR_C)
     set(C_TESTS db/c_test.c)
+    # C executables must link to a shared object
     add_executable(c_test db/c_test.c)
-    target_link_libraries(c_test ${ROCKSDB_LIB_FOR_C} testharness)
+    target_link_libraries(c_test ${ROCKSDB_SHARED_LIB} testharness)
     add_test(NAME c_test COMMAND c_test${ARTIFACT_SUFFIX})
     add_dependencies(check c_test)
   endif()
 endif()
 
+option(WITH_BENCHMARK_TOOLS "build with benchmarks" ON)
 if(WITH_BENCHMARK_TOOLS)
-  add_executable(db_bench${ARTIFACT_SUFFIX}
-    tools/simulated_hybrid_file_system.cc
+  add_executable(db_bench
     tools/db_bench.cc
     tools/db_bench_tool.cc)
-  target_link_libraries(db_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${THIRDPARTY_LIBS})
+  target_link_libraries(db_bench
+    ${ROCKSDB_LIB})
 
-  add_executable(cache_bench${ARTIFACT_SUFFIX}
-    cache/cache_bench.cc
-    cache/cache_bench_tool.cc)
-  target_link_libraries(cache_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+  add_executable(cache_bench
+    cache/cache_bench.cc)
+  target_link_libraries(cache_bench
+    ${ROCKSDB_LIB})
 
-  add_executable(memtablerep_bench${ARTIFACT_SUFFIX}
+  add_executable(memtablerep_bench
     memtable/memtablerep_bench.cc)
-  target_link_libraries(memtablerep_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+  target_link_libraries(memtablerep_bench
+    ${ROCKSDB_LIB})
 
-  add_executable(range_del_aggregator_bench${ARTIFACT_SUFFIX}
+  add_executable(range_del_aggregator_bench
     db/range_del_aggregator_bench.cc)
-  target_link_libraries(range_del_aggregator_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+  target_link_libraries(range_del_aggregator_bench
+    ${ROCKSDB_LIB})
 
-  add_executable(table_reader_bench${ARTIFACT_SUFFIX}
+  add_executable(table_reader_bench
     table/table_reader_bench.cc)
-  target_link_libraries(table_reader_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} testharness ${GFLAGS_LIB})
+  target_link_libraries(table_reader_bench
+    ${ROCKSDB_LIB} testharness)
 
-  add_executable(filter_bench${ARTIFACT_SUFFIX}
+  add_executable(filter_bench
     util/filter_bench.cc)
-  target_link_libraries(filter_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+  target_link_libraries(filter_bench
+    ${ROCKSDB_LIB})
 
-  add_executable(hash_table_bench${ARTIFACT_SUFFIX}
+  add_executable(hash_table_bench
     utilities/persistent_cache/hash_table_bench.cc)
-  target_link_libraries(hash_table_bench${ARTIFACT_SUFFIX}
-    ${ROCKSDB_LIB} ${GFLAGS_LIB})
+  target_link_libraries(hash_table_bench
+    ${ROCKSDB_LIB})
 endif()
 
+option(WITH_CORE_TOOLS "build with ldb and sst_dump" ON)
+option(WITH_TOOLS "build with tools" ON)
 if(WITH_CORE_TOOLS OR WITH_TOOLS)
   add_subdirectory(tools)
   add_custom_target(core_tools
@@ -1417,13 +1212,3 @@ if(WITH_TOOLS)
   add_custom_target(tools
     DEPENDS ${tool_deps})
 endif()
-
-option(WITH_EXAMPLES "build with examples" OFF)
-if(WITH_EXAMPLES)
-  add_subdirectory(examples)
-endif()
-
-option(WITH_BENCHMARK "build benchmark tests" OFF)
-if(WITH_BENCHMARK)
-  add_subdirectory(${PROJECT_SOURCE_DIR}/microbench/)
-endif()

HISTORY.md

@@ -1,462 +1,20 @@
 # Rocksdb Change Log
-## 6.23.4 (2021-08-31)
-### Bug Fixes
-* Fix a race in item ref counting in LRUCache when promoting an item from the SecondaryCache.
-
-## 6.23.3 (2021-08-09)
-### Bug Fixes
-* Removed a call to `RenameFile()` on a non-existent info log file ("LOG") when opening a new DB. Such a call was guaranteed to fail though did not impact applications since we swallowed the error. Now we also stopped swallowing errors in renaming "LOG" file.
-* Fixed a bug affecting the batched `MultiGet` API when used with keys spanning multiple column families and `sorted_input == false`.
-
-## 6.23.2 (2021-08-04)
-### Bug Fixes
-* Fixed a race related to the destruction of `ColumnFamilyData` objects. The earlier logic unlocked the DB mutex before destroying the thread-local `SuperVersion` pointers, which could result in a process crash if another thread managed to get a reference to the `ColumnFamilyData` object.
-* Fixed an issue where `OnFlushCompleted` was not called for atomic flush.
-
-## 6.23.1 (2021-07-22)
-### Bug Fixes
-* Fix a race condition during multiple DB instances opening.
-
-## 6.23.0 (2021-07-16)
-### Behavior Changes
-* Obsolete keys in the bottommost level that were preserved for a snapshot will now be cleaned upon snapshot release in all cases. This form of compaction (snapshot release triggered compaction) previously had an artificial limitation that multiple tombstones needed to be present.
-
-### Bug Fixes
-* Blob file checksums are now printed in hexadecimal format when using the `manifest_dump` `ldb` command.
-* `GetLiveFilesMetaData()` now populates the `temperature`, `oldest_ancester_time`, and `file_creation_time` fields of its `LiveFileMetaData` results when the information is available. Previously these fields always contained zero indicating unknown.
-* Fix mismatches of OnCompaction{Begin,Completed} in case of DisableManualCompaction().
-* Fix continuous logging of an existing background error on every user write
-* Fix a bug that `Get()` return Status::OK() and an empty value for non-existent key when `read_options.read_tier = kBlockCacheTier`. 
-* Fix a bug that stat in `get_context` didn't accumulate to statistics when query is failed.
-
-### New Features
-* ldb has a new feature, `list_live_files_metadata`, that shows the live SST files, as well as their LSM storage level and the column family they belong to.
-* The new BlobDB implementation now tracks the amount of garbage in each blob file in the MANIFEST.
-* Integrated BlobDB now supports Merge with base values (Put/Delete etc.).
-* RemoteCompaction supports sub-compaction, the job_id in the user interface is changed from `int` to `uint64_t` to support sub-compaction id.
-* Expose statistics option in RemoteCompaction worker.
-
-### Public API change
-* Added APIs to the Customizable class to allow developers to create their own Customizable classes.  Created the utilities/customizable_util.h file to contain helper methods for developing new Customizable classes.
-* Change signature of SecondaryCache::Name().  Make SecondaryCache customizable and add SecondaryCache::CreateFromString method.
-## 6.22.0 (2021-06-18)
-### Behavior Changes
-* Added two additional tickers, MEMTABLE_PAYLOAD_BYTES_AT_FLUSH and MEMTABLE_GARBAGE_BYTES_AT_FLUSH. These stats can be used to estimate the ratio of "garbage" (outdated) bytes in the memtable that are discarded at flush time.
-* Added API comments clarifying safe usage of Disable/EnableManualCompaction and EventListener callbacks for compaction.
-### Bug Fixes
-* fs_posix.cc GetFreeSpace() always report disk space available to root even when running as non-root.  Linux defaults often have disk mounts with 5 to 10 percent of total space reserved only for root.  Out of space could result for non-root users.
-* Subcompactions are now disabled when user-defined timestamps are used, since the subcompaction boundary picking logic is currently not timestamp-aware, which could lead to incorrect results when different subcompactions process keys that only differ by timestamp.
-* Fix an issue that `DeleteFilesInRange()` may cause ongoing compaction reports corruption exception, or ASSERT for debug build. There's no actual data loss or corruption that we find.
-* Fixed confusingly duplicated output in LOG for periodic stats ("DUMPING STATS"), including "Compaction Stats" and "File Read Latency Histogram By Level".
-* Fixed performance bugs in background gathering of block cache entry statistics, that could consume a lot of CPU when there are many column families with a shared block cache.
-
-### New Features
-* Marked the Ribbon filter and optimize_filters_for_memory features as production-ready, each enabling memory savings for Bloom-like filters. Use `NewRibbonFilterPolicy` in place of `NewBloomFilterPolicy` to use Ribbon filters instead of Bloom, or `ribbonfilter` in place of `bloomfilter` in configuration string.
-* Allow `DBWithTTL` to use `DeleteRange` api just like other DBs. `DeleteRangeCF()` which executes `WriteBatchInternal::DeleteRange()` has been added to the handler in `DBWithTTLImpl::Write()` to implement it.
-* Add BlockBasedTableOptions.prepopulate_block_cache.  If enabled, it prepopulate warm/hot data blocks which are already in memory into block cache at the time of flush. On a flush, the data block that is in memory (in memtables) get flushed to the device. If using Direct IO, additional IO is incurred to read this data back into memory again, which is avoided by enabling this option and it also helps with Distributed FileSystem. More details in include/rocksdb/table.h.
-* Added a `cancel` field to `CompactRangeOptions`, allowing individual in-process manual range compactions to be cancelled.
-
-### New Features
-* Added BlobMetaData to the ColumnFamilyMetaData to return information about blob files
-
-### Public API change
-* Added GetAllColumnFamilyMetaData API to retrieve the ColumnFamilyMetaData about all column families.
-
-## 6.21.0 (2021-05-21)
-### Bug Fixes
-* Fixed a bug in handling file rename error in distributed/network file systems when the server succeeds but client returns error. The bug can cause CURRENT file to point to non-existing MANIFEST file, thus DB cannot be opened.
-* Fixed a bug where ingested files were written with incorrect boundary key metadata. In rare cases this could have led to a level's files being wrongly ordered and queries for the boundary keys returning wrong results.
-* Fixed a data race between insertion into memtables and the retrieval of the DB properties `rocksdb.cur-size-active-mem-table`, `rocksdb.cur-size-all-mem-tables`, and `rocksdb.size-all-mem-tables`.
-* Fixed the false-positive alert when recovering from the WAL file. Avoid reporting "SST file is ahead of WAL" on a newly created empty column family, if the previous WAL file is corrupted.
-* Fixed a bug where `GetLiveFiles()` output included a non-existent file called "OPTIONS-000000". Backups and checkpoints, which use `GetLiveFiles()`, failed on DBs impacted by this bug. Read-write DBs were impacted when the latest OPTIONS file failed to write and `fail_if_options_file_error == false`. Read-only DBs were impacted when no OPTIONS files existed.
-* Handle return code by io_uring_submit_and_wait() and io_uring_wait_cqe().
-* In the IngestExternalFile() API, only try to sync the ingested file if the file is linked and the FileSystem/Env supports reopening a writable file.
-* Fixed a bug that `AdvancedColumnFamilyOptions.max_compaction_bytes` is under-calculated for manual compaction (`CompactRange()`). Manual compaction is split to multiple compactions if the compaction size exceed the `max_compaction_bytes`. The bug creates much larger compaction which size exceed the user setting. On the other hand, larger manual compaction size can increase the subcompaction parallelism, you can tune that by setting `max_compaction_bytes`.
-
-### Behavior Changes
-* Due to the fix of false-postive alert of "SST file is ahead of WAL", all the CFs with no SST file (CF empty) will bypass the consistency check. We fixed a false-positive, but introduced a very rare true-negative which will be triggered in the following conditions: A CF with some delete operations in the last a few queries which will result in an empty CF (those are flushed to SST file and a compaction triggered which combines this file and all other SST files and generates an empty CF, or there is another reason to write a manifest entry for this CF after a flush that generates no SST file from an empty CF). The deletion entries are logged in a WAL and this WAL was corrupted, while the CF's log number points to the next WAL (due to the flush). Therefore, the DB can only recover to the point without these trailing deletions and cause the inconsistent DB status.
-
-### New Features
-* Add new option allow_stall passed during instance creation of WriteBufferManager. When allow_stall is set, WriteBufferManager will stall all writers shared across multiple DBs and columns if memory usage goes beyond specified WriteBufferManager::buffer_size (soft limit). Stall will be cleared when memory is freed after flush and memory usage goes down below buffer_size.
-* Allow `CompactionFilter`s to apply in more table file creation scenarios such as flush and recovery. For compatibility, `CompactionFilter`s by default apply during compaction. Users can customize this behavior by overriding `CompactionFilterFactory::ShouldFilterTableFileCreation()`.
-* Added more fields to FilterBuildingContext with LSM details, for custom filter policies that vary behavior based on where they are in the LSM-tree.
-* Added DB::Properties::kBlockCacheEntryStats for querying statistics on what percentage of block cache is used by various kinds of blocks, etc. using DB::GetProperty and DB::GetMapProperty. The same information is now dumped to info LOG periodically according to `stats_dump_period_sec`.
-* Add an experimental Remote Compaction feature, which allows the user to run Compaction on a different host or process. The feature is still under development, currently only works on some basic use cases. The interface will be changed without backward/forward compatibility support.
-* RocksDB would validate total entries read in flush, and compare with counter inserted into it. If flush_verify_memtable_count = true (default), flush will fail. Otherwise, only log to info logs.
-* Add `TableProperties::num_filter_entries`, which can be used with `TableProperties::filter_size` to calculate the effective bits per filter entry (unique user key or prefix) for a table file.
-
-### Performance Improvements
-* BlockPrefetcher is used by iterators to prefetch data if they anticipate more data to be used in future. It is enabled implicitly by rocksdb. Added change to take in account read pattern if reads are sequential. This would disable prefetching for random reads in MultiGet and iterators as readahead_size is increased exponential doing large prefetches.
-
-### Public API change
-* Removed a parameter from TableFactory::NewTableBuilder, which should not be called by user code because TableBuilder is not a public API.
-* Removed unused structure `CompactionFilterContext`.
-* The `skip_filters` parameter to SstFileWriter is now considered deprecated. Use `BlockBasedTableOptions::filter_policy` to control generation of filters.
-* ClockCache is known to have bugs that could lead to crash or corruption, so should not be used until fixed. Use NewLRUCache instead.
-* Added a new pure virtual function `ApplyToAllEntries` to `Cache`, to replace `ApplyToAllCacheEntries`. Custom `Cache` implementations must add an implementation. Because this function is for gathering statistics, an empty implementation could be acceptable for some applications.
-* Added the ObjectRegistry to the ConfigOptions class.  This registry instance will be used to find any customizable loadable objects during initialization.
-* Expanded the ObjectRegistry functionality to allow nested ObjectRegistry instances.  Added methods to register a set of functions with the registry/library as a group.
-* Deprecated backupable_db.h and BackupableDBOptions in favor of new versions with appropriate names: backup_engine.h and BackupEngineOptions. Old API compatibility is preserved.
-
-### Default Option Change
-* When options.arena_block_size <= 0 (default value 0), still use writer_buffer_size / 8 but cap to 1MB. Too large alloation size might not be friendly to allocator and might cause performance issues in extreme cases.
-
-### Build
-* By default, try to build with liburing. For make, if ROCKSDB_USE_IO_URING is not set, treat as enable, which means RocksDB will try to build with liburing. Users can disable it with ROCKSDB_USE_IO_URING=0. For cmake, add WITH_LIBURING to control it, with default on.
-
-## 6.20.0 (2021-04-16)
-### Behavior Changes
-* `ColumnFamilyOptions::sample_for_compression` now takes effect for creation of all block-based tables. Previously it only took effect for block-based tables created by flush.
-* `CompactFiles()` can no longer compact files from lower level to up level, which has the risk to corrupt DB (details: #8063). The validation is also added to all compactions.
-* Fixed some cases in which DB::OpenForReadOnly() could write to the filesystem. If you want a Logger with a read-only DB, you must now set DBOptions::info_log yourself, such as using CreateLoggerFromOptions().
-* get_iostats_context() will never return nullptr. If thread-local support is not available, and user does not opt-out iostats context, then compilation will fail. The same applies to perf context as well.
-* Added support for WriteBatchWithIndex::NewIteratorWithBase when overwrite_key=false.  Previously, this combination was not supported and would assert or return nullptr.
-* Improve the behavior of WriteBatchWithIndex for Merge operations.  Now more operations may be stored in order to return the correct merged result.
-
-### Bug Fixes
-* Use thread-safe `strerror_r()` to get error messages.
-* Fixed a potential hang in shutdown for a DB whose `Env` has high-pri thread pool disabled (`Env::GetBackgroundThreads(Env::Priority::HIGH) == 0`)
-* Made BackupEngine thread-safe and added documentation comments to clarify what is safe for multiple BackupEngine objects accessing the same backup directory.
-* Fixed crash (divide by zero) when compression dictionary is applied to a file containing only range tombstones.
-* Fixed a backward iteration bug with partitioned filter enabled: not including the prefix of the last key of the previous filter partition in current filter partition can cause wrong iteration result.
-* Fixed a bug that allowed `DBOptions::max_open_files` to be set with a non-negative integer with `ColumnFamilyOptions::compaction_style = kCompactionStyleFIFO`.
-
-### Performance Improvements
-* On ARM platform, use `yield` instead of `wfe` to relax cpu to gain better performance.
-
-### Public API change
-* Added `TableProperties::slow_compression_estimated_data_size` and `TableProperties::fast_compression_estimated_data_size`. When `ColumnFamilyOptions::sample_for_compression > 0`, they estimate what `TableProperties::data_size` would have been if the "fast" or "slow" (see `ColumnFamilyOptions::sample_for_compression` API doc for definitions) compression had been used instead.
-* Update DB::StartIOTrace and remove Env object from the arguments as its redundant and DB already has Env object that is passed down to IOTracer::StartIOTrace
-* Added `FlushReason::kWalFull`, which is reported when a memtable is flushed due to the WAL reaching its size limit; those flushes were previously reported as `FlushReason::kWriteBufferManager`. Also, changed the reason for flushes triggered by the write buffer manager to `FlushReason::kWriteBufferManager`; they were previously reported as `FlushReason::kWriteBufferFull`.
-* Extend file_checksum_dump ldb command and DB::GetLiveFilesChecksumInfo API for IntegratedBlobDB and get checksum of blob files along with SST files.
-
-### New Features
-* Added the ability to open BackupEngine backups as read-only DBs, using BackupInfo::name_for_open and env_for_open provided by BackupEngine::GetBackupInfo() with include_file_details=true.
-* Added BackupEngine support for integrated BlobDB, with blob files shared between backups when table files are shared. Because of current limitations, blob files always use the kLegacyCrc32cAndFileSize naming scheme, and incremental backups must read and checksum all blob files in a DB, even for files that are already backed up.
-* Added an optional output parameter to BackupEngine::CreateNewBackup(WithMetadata) to return the BackupID of the new backup.
-* Added BackupEngine::GetBackupInfo / GetLatestBackupInfo for querying individual backups.
-* Made the Ribbon filter a long-term supported feature in terms of the SST schema(compatible with version >= 6.15.0) though the API for enabling it is expected to change.
-
-## 6.19.0 (2021-03-21)
-### Bug Fixes
-* Fixed the truncation error found in APIs/tools when dumping block-based SST files in a human-readable format. After fix, the block-based table can be fully dumped as a readable file.
-* When hitting a write slowdown condition, no write delay (previously 1 millisecond) is imposed until `delayed_write_rate` is actually exceeded, with an initial burst allowance of 1 millisecond worth of bytes. Also, beyond the initial burst allowance, `delayed_write_rate` is now more strictly enforced, especially with multiple column families.
-
-### Public API change
-* Changed default `BackupableDBOptions::share_files_with_checksum` to `true` and deprecated `false` because of potential for data loss. Note that accepting this change in behavior can temporarily increase backup data usage because files are not shared between backups using the two different settings. Also removed obsolete option kFlagMatchInterimNaming.
-* Add a new option BlockBasedTableOptions::max_auto_readahead_size. RocksDB does auto-readahead for iterators on noticing more than two reads for a table file if user doesn't provide readahead_size. The readahead starts at 8KB and doubles on every additional read upto max_auto_readahead_size and now max_auto_readahead_size can be configured dynamically as well. Found that 256 KB readahead size provides the best performance, based on experiments, for auto readahead. Experiment data is in PR #3282. If value is set 0 then no automatic prefetching will be done by rocksdb. Also changing the value will only affect files opened after the change.
-* Add suppport to extend DB::VerifyFileChecksums API to also verify blob files checksum.
-* When using the new BlobDB, the amount of data written by flushes/compactions is now broken down into table files and blob files in the compaction statistics; namely, Write(GB) denotes the amount of data written to table files, while Wblob(GB) means the amount of data written to blob files.
-* New default BlockBasedTableOptions::format_version=5 to enable new Bloom filter implementation by default, compatible with RocksDB versions >= 6.6.0.
-* Add new SetBufferSize API to WriteBufferManager to allow dynamic management of memory allotted to all write buffers.  This allows user code to adjust memory monitoring provided by WriteBufferManager as process memory needs change datasets grow and shrink.
-* Clarified the required semantics of Read() functions in FileSystem and Env APIs. Please ensure any custom implementations are compliant.
-* For the new integrated BlobDB implementation, compaction statistics now include the amount of data read from blob files during compaction (due to garbage collection or compaction filters). Write amplification metrics have also been extended to account for data read from blob files.
-* Add EqualWithoutTimestamp() to Comparator.
-* Extend support to track blob files in SSTFileManager whenever a blob file is created/deleted. Blob files will be scheduled to delete via SSTFileManager and SStFileManager will now take blob files in account while calculating size and space limits along with SST files.
-* Add new Append and PositionedAppend API with checksum handoff to legacy Env.
-
-### New Features
-* Support compaction filters for the new implementation of BlobDB. Add `FilterBlobByKey()` to `CompactionFilter`. Subclasses can override this method so that compaction filters can determine whether the actual blob value has to be read during compaction. Use a new `kUndetermined` in `CompactionFilter::Decision` to indicated that further action is necessary for compaction filter to make a decision.
-* Add support to extend retrieval of checksums for blob files from the MANIFEST when checkpointing. During backup, rocksdb can detect corruption in blob files  during file copies.
-* Add new options for db_bench --benchmarks: flush, waitforcompaction, compact0, compact1.
-* Add an option to BackupEngine::GetBackupInfo to include the name and size of each backed-up file. Especially in the presence of file sharing among backups, this offers detailed insight into backup space usage.
-* Enable backward iteration on keys with user-defined timestamps.
-* Add statistics and info log for error handler: counters for bg error, bg io error, bg retryable io error, auto resume count, auto resume total retry number, and auto resume sucess; Histogram for auto resume retry count in each recovery call. Note that, each auto resume attempt will have one or multiple retries.
-
-### Behavior Changes
-* During flush, only WAL sync retryable IO error is mapped to hard error, which will stall the writes. When WAL is used but only SST file write has retryable IO error, it will be mapped to soft error and write will not be affected.
-
-## 6.18.0 (2021-02-19)
-### Behavior Changes
-* When retryable IO error occurs during compaction, it is mapped to soft error and set the BG error. However, auto resume is not called to clean the soft error since compaction will reschedule by itself. In this change, When retryable IO error occurs during compaction, BG error is not set. User will be informed the error via EventHelper.
-* Introduce a new trace file format for query tracing and replay and trace file version is bump up to 0.2. A payload map is added as the first portion of the payload. We will not have backward compatible issues when adding new entries to trace records. Added the iterator_upper_bound and iterator_lower_bound in Seek and SeekForPrev tracing function. Added them as the new payload member for iterator tracing.
-
-### New Features
-* Add support for key-value integrity protection in live updates from the user buffers provided to `WriteBatch` through the write to RocksDB's in-memory update buffer (memtable). This is intended to detect some cases of in-memory data corruption, due to either software or hardware errors. Users can enable protection by constructing their `WriteBatch` with `protection_bytes_per_key == 8`.
-* Add support for updating `full_history_ts_low` option in manual compaction, which is for old timestamp data GC.
-* Add a mechanism for using Makefile to build external plugin code into the RocksDB libraries/binaries. This intends to simplify compatibility and distribution for plugins (e.g., special-purpose `FileSystem`s) whose source code resides outside the RocksDB repo. See "plugin/README.md" for developer details, and "PLUGINS.md" for a listing of available plugins.
-* Added memory pre-fetching for experimental Ribbon filter, which especially optimizes performance with batched MultiGet.
-* A new, experimental version of BlobDB (key-value separation) is now available. The new implementation is integrated into the RocksDB core, i.e. it is accessible via the usual `rocksdb::DB` API, as opposed to the separate `rocksdb::blob_db::BlobDB` interface used by the earlier version, and can be configured on a per-column family basis using the configuration options `enable_blob_files`, `min_blob_size`, `blob_file_size`, `blob_compression_type`, `enable_blob_garbage_collection`, and `blob_garbage_collection_age_cutoff`. It extends RocksDB's consistency guarantees to blobs, and offers more features and better performance. Note that some features, most notably `Merge`, compaction filters, and backup/restore are not yet supported, and there is no support for migrating a database created by the old implementation.
-
-### Bug Fixes
-* Since 6.15.0, `TransactionDB` returns error `Status`es from calls to `DeleteRange()` and calls to `Write()` where the `WriteBatch` contains a range deletion. Previously such operations may have succeeded while not providing the expected transactional guarantees. There are certain cases where range deletion can still be used on such DBs; see the API doc on `TransactionDB::DeleteRange()` for details.
-* `OptimisticTransactionDB` now returns error `Status`es from calls to `DeleteRange()` and calls to `Write()` where the `WriteBatch` contains a range deletion. Previously such operations may have succeeded while not providing the expected transactional guarantees.
-* Fix `WRITE_PREPARED`, `WRITE_UNPREPARED` TransactionDB `MultiGet()` may return uncommitted data with snapshot.
-* In DB::OpenForReadOnly, if any error happens while checking Manifest file path, it was overridden by Status::NotFound. It has been fixed and now actual error is returned.
-
-### Public API Change
-* Added a "only_mutable_options" flag to the ConfigOptions.  When this flag is "true", the Configurable functions and convenience methods (such as GetDBOptionsFromString) will only deal with options that are marked as mutable.  When this flag is true, only options marked as mutable can be configured (a Status::InvalidArgument will be returned) and options not marked as mutable will not be returned or compared.  The default is "false", meaning to compare all options.
-* Add new Append and PositionedAppend APIs to FileSystem to bring the data verification information (data checksum information) from upper layer (e.g., WritableFileWriter) to the storage layer. In this way, the customized FileSystem is able to verify the correctness of data being written to the storage on time. Add checksum_handoff_file_types to DBOptions. User can use this option to control which file types (Currently supported file tyes: kWALFile, kTableFile, kDescriptorFile.) should use the new Append and PositionedAppend APIs to handoff the verification information. Currently, RocksDB only use crc32c to calculate the checksum for write handoff.
-* Add an option, `CompressionOptions::max_dict_buffer_bytes`, to limit the in-memory buffering for selecting samples for generating/training a dictionary. The limit is currently loosely adhered to.
-
-
-## 6.17.0 (2021-01-15)
-### Behavior Changes
-* When verifying full file checksum with `DB::VerifyFileChecksums()`, we now fail with `Status::InvalidArgument` if the name of the checksum generator used for verification does not match the name of the checksum generator used for protecting the file when it was created.
-* Since RocksDB does not continue write the same file if a file write fails for any reason, the file scope write IO error is treated the same as retryable IO error. More information about error handling of file scope IO error is included in `ErrorHandler::SetBGError`.
-
-### Bug Fixes
-* Version older than 6.15 cannot decode VersionEdits `WalAddition` and `WalDeletion`, fixed this by changing the encoded format of them to be ignorable by older versions.
-* Fix a race condition between DB startups and shutdowns in managing the periodic background worker threads. One effect of this race condition could be the process being terminated.
-
-### Public API Change
-* Add a public API WriteBufferManager::dummy_entries_in_cache_usage() which reports the size of dummy entries stored in cache (passed to WriteBufferManager). Dummy entries are used to account for DataBlocks.
-* Add a SystemClock class that contains the time-related methods from Env.  The original methods in Env may be deprecated in a future release.  This class will allow easier testing, development, and expansion of time-related features.
-* Add a public API GetRocksBuildProperties and GetRocksBuildInfoAsString to get properties about the current build.  These properties may include settings related to the GIT settings (branch, timestamp).  This change also sets the "build date" based on the GIT properties, rather than the actual build time, thereby enabling more reproducible builds.
-
-## 6.16.0 (2020-12-18)
-### Behavior Changes
-* Attempting to write a merge operand without explicitly configuring `merge_operator` now fails immediately, causing the DB to enter read-only mode. Previously, failure was deferred until the `merge_operator` was needed by a user read or a background operation.
-
-### Bug Fixes
-* Truncated WALs ending in incomplete records can no longer produce gaps in the recovered data when `WALRecoveryMode::kPointInTimeRecovery` is used. Gaps are still possible when WALs are truncated exactly on record boundaries; for complete protection, users should enable `track_and_verify_wals_in_manifest`.
-* Fix a bug where compressed blocks read by MultiGet are not inserted into the compressed block cache when use_direct_reads = true.
-* Fixed the issue of full scanning on obsolete files when there are too many outstanding compactions with ConcurrentTaskLimiter enabled.
-* Fixed the logic of populating native data structure for `read_amp_bytes_per_bit` during OPTIONS file parsing on big-endian architecture. Without this fix, original code introduced in PR7659, when running on big-endian machine, can mistakenly store read_amp_bytes_per_bit (an uint32) in little endian format. Future access to `read_amp_bytes_per_bit` will give wrong values. Little endian architecture is not affected.
-* Fixed prefix extractor with timestamp issues.
-* Fixed a bug in atomic flush: in two-phase commit mode, the minimum WAL log number to keep is incorrect.
-* Fixed a bug related to checkpoint in PR7789: if there are multiple column families, and the checkpoint is not opened as read only, then in rare cases, data loss may happen in the checkpoint. Since backup engine relies on checkpoint, it may also be affected.
-* When ldb --try_load_options is used with the --column_family option, the ColumnFamilyOptions for the specified column family was not loaded from the OPTIONS file. Fix it so its loaded from OPTIONS and then overridden with command line overrides.
-
-### New Features
-* User defined timestamp feature supports `CompactRange` and `GetApproximateSizes`.
-* Support getting aggregated table properties (kAggregatedTableProperties and kAggregatedTablePropertiesAtLevel) with DB::GetMapProperty, for easier access to the data in a structured format.
-* Experimental option BlockBasedTableOptions::optimize_filters_for_memory now works with experimental Ribbon filter (as well as Bloom filter).
-
-### Public API Change
-* Deprecated public but rarely-used FilterBitsBuilder::CalculateNumEntry, which is replaced with ApproximateNumEntries taking a size_t parameter and returning size_t.
-* To improve portability the functions `Env::GetChildren` and `Env::GetChildrenFileAttributes` will no longer return entries for the special directories `.` or `..`.
-* Added a new option `track_and_verify_wals_in_manifest`. If `true`, the log numbers and sizes of the synced WALs are tracked in MANIFEST, then during DB recovery, if a synced WAL is missing from disk, or the WAL's size does not match the recorded size in MANIFEST, an error will be reported and the recovery will be aborted. Note that this option does not work with secondary instance.
-* `rocksdb_approximate_sizes` and `rocksdb_approximate_sizes_cf` in the C API now requires an error pointer (`char** errptr`) for receiving any error.
-* All overloads of DB::GetApproximateSizes now return Status, so that any failure to obtain the sizes is indicated to the caller.
-
-## 6.15.0 (2020-11-13)
-### Bug Fixes
-* Fixed a bug in the following combination of features: indexes with user keys (`format_version >= 3`), indexes are partitioned (`index_type == kTwoLevelIndexSearch`), and some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`). The bug could cause keys to be truncated when read from the index leading to wrong read results or other unexpected behavior.
-* Fixed a bug when indexes are partitioned (`index_type == kTwoLevelIndexSearch`), some index partitions are pinned in memory (`BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache`), and partitions reads could be mixed between block cache and directly from the file (e.g., with `enable_index_compression == 1` and `mmap_read == 1`, partitions that were stored uncompressed due to poor compression ratio would be read directly from the file via mmap, while partitions that were stored compressed would be read from block cache). The bug could cause index partitions to be mistakenly considered empty during reads leading to wrong read results.
-* Since 6.12, memtable lookup should report unrecognized value_type as corruption (#7121).
-* Since 6.14, fix false positive flush/compaction `Status::Corruption` failure when `paranoid_file_checks == true` and range tombstones were written to the compaction output files.
-* Since 6.14, fix a bug that could cause a stalled write to crash with mixed of slowdown and no_slowdown writes (`WriteOptions.no_slowdown=true`).
-* Fixed a bug which causes hang in closing DB when refit level is set in opt build. It was because ContinueBackgroundWork() was called in assert statement which is a no op. It was introduced in 6.14.
-* Fixed a bug which causes Get() to return incorrect result when a key's merge operand is applied twice. This can occur if the thread performing Get() runs concurrently with a background flush thread and another thread writing to the MANIFEST file (PR6069).
-* Reverted a behavior change silently introduced in 6.14.2, in which the effects of the `ignore_unknown_options` flag (used in option parsing/loading functions) changed.
-* Reverted a behavior change silently introduced in 6.14, in which options parsing/loading functions began returning `NotFound` instead of `InvalidArgument` for option names not available in the present version.
-* Fixed MultiGet bugs it doesn't return valid data with user defined timestamp.
-* Fixed a potential bug caused by evaluating `TableBuilder::NeedCompact()` before `TableBuilder::Finish()` in compaction job. For example, the `NeedCompact()` method of `CompactOnDeletionCollector` returned by built-in `CompactOnDeletionCollectorFactory` requires `BlockBasedTable::Finish()` to return the correct result. The bug can cause a compaction-generated file not to be marked for future compaction based on deletion ratio.
-* Fixed a seek issue with prefix extractor and timestamp.
-* Fixed a bug of encoding and parsing BlockBasedTableOptions::read_amp_bytes_per_bit as a 64-bit integer.
-* Fixed a bug of a recovery corner case, details in PR7621.
-
-### Public API Change
-* Deprecate `BlockBasedTableOptions::pin_l0_filter_and_index_blocks_in_cache` and `BlockBasedTableOptions::pin_top_level_index_and_filter`. These options still take effect until users migrate to the replacement APIs in `BlockBasedTableOptions::metadata_cache_options`. Migration guidance can be found in the API comments on the deprecated options.
-* Add new API `DB::VerifyFileChecksums` to verify SST file checksum with corresponding entries in the MANIFEST if present. Current implementation requires scanning and recomputing file checksums.
-
-### Behavior Changes
-* The dictionary compression settings specified in `ColumnFamilyOptions::compression_opts` now additionally affect files generated by flush and compaction to non-bottommost level. Previously those settings at most affected files generated by compaction to bottommost level, depending on whether `ColumnFamilyOptions::bottommost_compression_opts` overrode them. Users who relied on dictionary compression settings in `ColumnFamilyOptions::compression_opts` affecting only the bottommost level can keep the behavior by moving their dictionary settings to `ColumnFamilyOptions::bottommost_compression_opts` and setting its `enabled` flag.
-* When the `enabled` flag is set in `ColumnFamilyOptions::bottommost_compression_opts`, those compression options now take effect regardless of the value in `ColumnFamilyOptions::bottommost_compression`. Previously, those compression options only took effect when `ColumnFamilyOptions::bottommost_compression != kDisableCompressionOption`. Now, they additionally take effect when `ColumnFamilyOptions::bottommost_compression == kDisableCompressionOption` (such a setting causes bottommost compression type to fall back to `ColumnFamilyOptions::compression_per_level` if configured, and otherwise fall back to `ColumnFamilyOptions::compression`).
-
-### New Features
-* An EXPERIMENTAL new Bloom alternative that saves about 30% space compared to Bloom filters, with about 3-4x construction time and similar query times is available using NewExperimentalRibbonFilterPolicy.
-
-## 6.14 (2020-10-09)
-### Bug fixes
-* Fixed a bug after a `CompactRange()` with `CompactRangeOptions::change_level` set fails due to a conflict in the level change step, which caused all subsequent calls to `CompactRange()` with `CompactRangeOptions::change_level` set to incorrectly fail with a `Status::NotSupported("another thread is refitting")` error.
-* Fixed a bug that the bottom most level compaction could still be a trivial move even if `BottommostLevelCompaction.kForce` or `kForceOptimized` is set.
-
-### Public API Change
-* The methods to create and manage EncrypedEnv have been changed.  The EncryptionProvider is now passed to NewEncryptedEnv as a shared pointer, rather than a raw pointer.  Comparably, the CTREncryptedProvider now takes a shared pointer, rather than a reference, to a BlockCipher.  CreateFromString methods have been added to BlockCipher and EncryptionProvider to provide a single API by which different ciphers and providers can be created, respectively.
-* The internal classes (CTREncryptionProvider, ROT13BlockCipher, CTRCipherStream) associated with the EncryptedEnv have been moved out of the public API.  To create a CTREncryptionProvider, one can either use EncryptionProvider::NewCTRProvider, or EncryptionProvider::CreateFromString("CTR").  To create a new ROT13BlockCipher, one can either use BlockCipher::NewROT13Cipher or BlockCipher::CreateFromString("ROT13").
-* The EncryptionProvider::AddCipher method has been added to allow keys to be added to an EncryptionProvider.  This API will allow future providers to support multiple cipher keys.
-* Add a new option "allow_data_in_errors". When this new option is set by users, it allows users to opt-in to get error messages containing corrupted keys/values. Corrupt keys, values will be logged in the messages, logs, status etc. that will help users with the useful information regarding affected data. By default value of this option is set false to prevent users data to be exposed in the messages so currently, data will be redacted from logs, messages, status by default.
-* AdvancedColumnFamilyOptions::force_consistency_checks is now true by default, for more proactive DB corruption detection at virtually no cost (estimated two extra CPU cycles per million on a major production workload). Corruptions reported by these checks now mention "force_consistency_checks" in case a false positive corruption report is suspected and the option needs to be disabled (unlikely). Since existing column families have a saved setting for force_consistency_checks, only new column families will pick up the new default.
-
-### General Improvements
-* The settings of the DBOptions and ColumnFamilyOptions are now managed by Configurable objects (see New Features).  The same convenience methods to configure these options still exist but the backend implementation has been unified under a common implementation.
-
-### New Features
-
-* Methods to configure serialize, and compare -- such as TableFactory -- are exposed directly through the Configurable base class (from which these objects inherit).  This change will allow for better and more thorough configuration management and retrieval in the future.  The options for a Configurable object can be set via the ConfigureFromMap, ConfigureFromString, or ConfigureOption method.  The serialized version of the options of an object can be retrieved via the GetOptionString, ToString, or GetOption methods.  The list of options supported by an object can be obtained via the GetOptionNames method.  The "raw" object (such as the BlockBasedTableOption) for an option may be retrieved via the GetOptions method.  Configurable options can be compared via the AreEquivalent method.  The settings within a Configurable object may be validated via the ValidateOptions method.  The object may be intialized (at which point only mutable options may be updated) via the PrepareOptions method.
-* Introduce options.check_flush_compaction_key_order with default value to be true. With this option, during flush and compaction, key order will be checked when writing to each SST file. If the order is violated, the flush or compaction will fail.
-* Added is_full_compaction to CompactionJobStats, so that the information is available through the EventListener interface.
-* Add more stats for MultiGet in Histogram to get number of data blocks, index blocks, filter blocks and sst files read from file system per level.
-* SST files have a new table property called db_host_id, which is set to the hostname by default. A new option in DBOptions, db_host_id, allows the property value to be overridden with a user specified string, or disable it completely by making the option string empty.
-* Methods to create customizable extensions -- such as TableFactory -- are exposed directly through the Customizable base class (from which these objects inherit).  This change will allow these Customizable classes to be loaded and configured in a standard way (via CreateFromString).  More information on how to write and use Customizable classes is in the customizable.h header file.
-
-## 6.13 (2020-09-12)
-### Bug fixes
-* Fix a performance regression introduced in 6.4 that makes a upper bound check for every Next() even if keys are within a data block that is within the upper bound.
-* Fix a possible corruption to the LSM state (overlapping files within a level) when a `CompactRange()` for refitting levels (`CompactRangeOptions::change_level == true`) and another manual compaction are executed in parallel.
-* Sanitize `recycle_log_file_num` to zero when the user attempts to enable it in combination with `WALRecoveryMode::kTolerateCorruptedTailRecords`. Previously the two features were allowed together, which compromised the user's configured crash-recovery guarantees.
-* Fix a bug where a level refitting in CompactRange() might race with an automatic compaction that puts the data to the target level of the refitting. The bug has been there for years.
-* Fixed a bug in version 6.12 in which BackupEngine::CreateNewBackup could fail intermittently with non-OK status when backing up a read-write DB configured with a DBOptions::file_checksum_gen_factory.
-* Fix useless no-op compactions scheduled upon snapshot release when options.disable-auto-compactions = true.
-* Fix a bug when max_write_buffer_size_to_maintain is set, immutable flushed memtable destruction is delayed until the next super version is installed. A memtable is not added to delete list because of its reference hold by super version and super version doesn't switch because of empt delete list. So memory usage keeps on increasing beyond write_buffer_size + max_write_buffer_size_to_maintain.
-* Avoid converting MERGES to PUTS when allow_ingest_behind is true.
-* Fix compression dictionary sampling together with `SstFileWriter`. Previously, the dictionary would be trained/finalized immediately with zero samples. Now, the whole `SstFileWriter` file is buffered in memory and then sampled.
-* Fix a bug with `avoid_unnecessary_blocking_io=1` and creating backups (BackupEngine::CreateNewBackup) or checkpoints (Checkpoint::Create). With this setting and WAL enabled, these operations could randomly fail with non-OK status.
-* Fix a bug in which bottommost compaction continues to advance the underlying InternalIterator to skip tombstones even after shutdown.
-
-### New Features
-* A new field `std::string requested_checksum_func_name` is added to `FileChecksumGenContext`, which enables the checksum factory to create generators for a suite of different functions.
-* Added a new subcommand, `ldb unsafe_remove_sst_file`, which removes a lost or corrupt SST file from a DB's metadata. This command involves data loss and must not be used on a live DB.
-
-### Performance Improvements
-* Reduce thread number for multiple DB instances by re-using one global thread for statistics dumping and persisting.
-* Reduce write-amp in heavy write bursts in `kCompactionStyleLevel` compaction style with `level_compaction_dynamic_level_bytes` set.
-* BackupEngine incremental backups no longer read DB table files that are already saved to a shared part of the backup directory, unless `share_files_with_checksum` is used with `kLegacyCrc32cAndFileSize` naming (discouraged).
-  * For `share_files_with_checksum`, we are confident there is no regression (vs. pre-6.12) in detecting DB or backup corruption at backup creation time, mostly because the old design did not leverage this extra checksum computation for detecting inconsistencies at backup creation time.
-  * For `share_table_files` without "checksum" (not recommended), there is a regression in detecting fundamentally unsafe use of the option, greatly mitigated by file size checking (under "Behavior Changes"). Almost no reason to use `share_files_with_checksum=false` should remain.
-  * `DB::VerifyChecksum` and `BackupEngine::VerifyBackup` with checksum checking are still able to catch corruptions that `CreateNewBackup` does not.
-
-### Public API Change
-* Expose kTypeDeleteWithTimestamp in EntryType and update GetEntryType() accordingly.
-* Added file_checksum and file_checksum_func_name to TableFileCreationInfo, which can pass the table file checksum information through the OnTableFileCreated callback during flush and compaction.
-* A warning is added to `DB::DeleteFile()` API describing its known problems and deprecation plan.
-* Add a new stats level, i.e. StatsLevel::kExceptTickers (PR7329) to exclude tickers even if application passes a non-null Statistics object.
-* Added a new status code IOStatus::IOFenced() for the Env/FileSystem to indicate that writes from this instance are fenced off. Like any other background error, this error is returned to the user in Put/Merge/Delete/Flush calls and can be checked using Status::IsIOFenced().
-
-### Behavior Changes
-* File abstraction `FSRandomAccessFile.Prefetch()` default return status is changed from `OK` to `NotSupported`. If the user inherited file doesn't implement prefetch, RocksDB will create internal prefetch buffer to improve read performance.
-* When retryabel IO error happens during Flush (manifest write error is excluded) and WAL is disabled, originally it is mapped to kHardError. Now,it is mapped to soft error. So DB will not stall the writes unless the memtable is full. At the same time, when auto resume is triggered to recover the retryable IO error during Flush, SwitchMemtable is not called to avoid generating to many small immutable memtables. If WAL is enabled, no behavior changes.
-* When considering whether a table file is already backed up in a shared part of backup directory, BackupEngine would already query the sizes of source (DB) and pre-existing destination (backup) files. BackupEngine now uses these file sizes to detect corruption, as at least one of (a) old backup, (b) backup in progress, or (c) current DB is corrupt if there's a size mismatch.
-
-### Others
-* Error in prefetching partitioned index blocks will not be swallowed. It will fail the query and return the IOError users.
-
-## 6.12 (2020-07-28)
-### Public API Change
-* Encryption file classes now exposed for inheritance in env_encryption.h
-* File I/O listener is extended to cover more I/O operations. Now class `EventListener` in listener.h contains new callback functions: `OnFileFlushFinish()`, `OnFileSyncFinish()`, `OnFileRangeSyncFinish()`, `OnFileTruncateFinish()`, and ``OnFileCloseFinish()``.
-* `FileOperationInfo` now reports `duration` measured by `std::chrono::steady_clock` and `start_ts` measured by `std::chrono::system_clock` instead of start and finish timestamps measured by `system_clock`. Note that `system_clock` is called before `steady_clock` in program order at operation starts.
-* `DB::GetDbSessionId(std::string& session_id)` is added. `session_id` stores a unique identifier that gets reset every time the DB is opened. This DB session ID should be unique among all open DB instances on all hosts, and should be unique among re-openings of the same or other DBs. This identifier is recorded in the LOG file on the line starting with "DB Session ID:".
-* `DB::OpenForReadOnly()` now returns `Status::NotFound` when the specified DB directory does not exist. Previously the error returned depended on the underlying `Env`. This change is available in all 6.11 releases as well.
-* A parameter `verify_with_checksum` is added to `BackupEngine::VerifyBackup`, which is false by default. If it is ture, `BackupEngine::VerifyBackup` verifies checksums and file sizes of backup files. Pass `false` for `verify_with_checksum` to maintain the previous behavior and performance of `BackupEngine::VerifyBackup`, by only verifying sizes of backup files.
-
-### Behavior Changes
-* Best-efforts recovery ignores CURRENT file completely. If CURRENT file is missing during recovery, best-efforts recovery still proceeds with MANIFEST file(s).
-* In best-efforts recovery, an error that is not Corruption or IOError::kNotFound or IOError::kPathNotFound will be overwritten silently. Fix this by checking all non-ok cases and return early.
-* When `file_checksum_gen_factory` is set to `GetFileChecksumGenCrc32cFactory()`, BackupEngine will compare the crc32c checksums of table files computed when creating a backup to the expected checksums stored in the DB manifest, and will fail `CreateNewBackup()` on mismatch (corruption). If the `file_checksum_gen_factory` is not set or set to any other customized factory, there is no checksum verification to detect if SST files in a DB are corrupt when read, copied, and independently checksummed by BackupEngine.
-* When a DB sets `stats_dump_period_sec > 0`, either as the initial value for DB open or as a dynamic option change, the first stats dump is staggered in the following X seconds, where X is an integer in `[0, stats_dump_period_sec)`. Subsequent stats dumps are still spaced `stats_dump_period_sec` seconds apart.
-* When the paranoid_file_checks option is true, a hash is generated of all keys and values are generated when the SST file is written, and then the values are read back in to validate the file.  A corruption is signaled if the two hashes do not match.
-
-### Bug fixes
-* Compressed block cache was automatically disabled with read-only DBs by mistake. Now it is fixed: compressed block cache will be in effective with read-only DB too.
-* Fix a bug of wrong iterator result if another thread finishes an update and a DB flush between two statement.
-* Disable file deletion after MANIFEST write/sync failure until db re-open or Resume() so that subsequent re-open will not see MANIFEST referencing deleted SSTs.
-* Fix a bug when index_type == kTwoLevelIndexSearch in PartitionedIndexBuilder to update FlushPolicy to point to internal key partitioner when it changes from user-key mode to internal-key mode in index partition.
-* Make compaction report InternalKey corruption while iterating over the input.
-* Fix a bug which may cause MultiGet to be slow because it may read more data than requested, but this won't affect correctness. The bug was introduced in 6.10 release.
-* Fail recovery and report once hitting a physical log record checksum mismatch, while reading MANIFEST. RocksDB should not continue processing the MANIFEST any further.
-* Fixed a bug in size-amp-triggered and periodic-triggered universal compaction, where the compression settings for the first input level were used rather than the compression settings for the output (bottom) level.
-
-### New Features
-* DB identity (`db_id`) and DB session identity (`db_session_id`) are added to table properties and stored in SST files. SST files generated from SstFileWriter and Repairer have DB identity “SST Writer” and “DB Repairer”, respectively. Their DB session IDs are generated in the same way as `DB::GetDbSessionId`. The session ID for SstFileWriter (resp., Repairer) resets every time `SstFileWriter::Open` (resp., `Repairer::Run`) is called.
-* Added experimental option BlockBasedTableOptions::optimize_filters_for_memory for reducing allocated memory size of Bloom filters (~10% savings with Jemalloc) while preserving the same general accuracy. To have an effect, the option requires format_version=5 and malloc_usable_size. Enabling this option is forward and backward compatible with existing format_version=5.
-* `BackupableDBOptions::share_files_with_checksum_naming` is added with new default behavior for naming backup files with `share_files_with_checksum`, to address performance and backup integrity issues. See API comments for details.
-* Added auto resume function to automatically recover the DB from background Retryable IO Error. When retryable IOError happens during flush and WAL write, the error is mapped to Hard Error and DB will be in read mode. When retryable IO Error happens during compaction, the error will be mapped to Soft Error. DB is still in write/read mode. Autoresume function will create a thread for a DB to call DB->ResumeImpl() to try the recover for Retryable IO Error during flush and WAL write. Compaction will be rescheduled by itself if retryable IO Error happens. Auto resume may also cause other Retryable IO Error during the recovery, so the recovery will fail. Retry the auto resume may solve the issue, so we use max_bgerror_resume_count to decide how many resume cycles will be tried in total. If it is <=0, auto resume retryable IO Error is disabled. Default is INT_MAX, which will lead to a infinit auto resume. bgerror_resume_retry_interval decides the time interval between two auto resumes.
-* Option `max_subcompactions` can be set dynamically using DB::SetDBOptions().
-* Added experimental ColumnFamilyOptions::sst_partitioner_factory to define determine the partitioning of sst files. This helps compaction to split the files on interesting boundaries (key prefixes) to make propagation of sst files less write amplifying (covering the whole key space).
-
-### Performance Improvements
-* Eliminate key copies for internal comparisons while accessing ingested block-based tables.
-* Reduce key comparisons during random access in all block-based tables.
-* BackupEngine avoids unnecessary repeated checksum computation for backing up a table file to the `shared_checksum` directory when using `share_files_with_checksum_naming = kUseDbSessionId` (new default), except on SST files generated before this version of RocksDB, which fall back on using `kLegacyCrc32cAndFileSize`.
-
-## 6.11 (2020-06-12)
-### Bug Fixes
-* Fix consistency checking error swallowing in some cases when options.force_consistency_checks = true.
-* Fix possible false NotFound status from batched MultiGet using index type kHashSearch.
-* Fix corruption caused by enabling delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode, along with parallel compactions. The bug can result in two parallel compactions picking the same input files, resulting in the DB resurrecting older and deleted versions of some keys.
-* Fix a use-after-free bug in best-efforts recovery. column_family_memtables_ needs to point to valid ColumnFamilySet.
-* Let best-efforts recovery ignore corrupted files during table loading.
-* Fix corrupt key read from ingested file when iterator direction switches from reverse to forward at a key that is a prefix of another key in the same file. It is only possible in files with a non-zero global seqno.
-* Fix abnormally large estimate from GetApproximateSizes when a range starts near the end of one SST file and near the beginning of another. Now GetApproximateSizes consistently and fairly includes the size of SST metadata in addition to data blocks, attributing metadata proportionally among the data blocks based on their size.
-* Fix potential file descriptor leakage in PosixEnv's IsDirectory() and NewRandomAccessFile().
-* Fix false negative from the VerifyChecksum() API when there is a checksum mismatch in an index partition block in a BlockBasedTable format table file (index_type is kTwoLevelIndexSearch).
-* Fix sst_dump to return non-zero exit code if the specified file is not a recognized SST file or fails requested checks.
-* Fix incorrect results from batched MultiGet for duplicate keys, when the duplicate key matches the largest key of an SST file and the value type for the key in the file is a merge value.
-
-### Public API Change
-* Flush(..., column_family) may return Status::ColumnFamilyDropped() instead of Status::InvalidArgument() if column_family is dropped while processing the flush request.
-* BlobDB now explicitly disallows using the default column family's storage directories as blob directory.
-* DeleteRange now returns `Status::InvalidArgument` if the range's end key comes before its start key according to the user comparator. Previously the behavior was undefined.
-* ldb now uses options.force_consistency_checks = true by default and "--disable_consistency_checks" is added to disable it.
-* DB::OpenForReadOnly no longer creates files or directories if the named DB does not exist, unless create_if_missing is set to true.
-* The consistency checks that validate LSM state changes (table file additions/deletions during flushes and compactions) are now stricter, more efficient, and no longer optional, i.e. they are performed even if `force_consistency_checks` is `false`.
-* Disable delete triggered compaction (NewCompactOnDeletionCollectorFactory) in universal compaction mode and num_levels = 1 in order to avoid a corruption bug.
-* `pin_l0_filter_and_index_blocks_in_cache` no longer applies to L0 files larger than `1.5 * write_buffer_size` to give more predictable memory usage. Such L0 files may exist due to intra-L0 compaction, external file ingestion, or user dynamically changing `write_buffer_size` (note, however, that files that are already pinned will continue being pinned, even after such a dynamic change).
-* In point-in-time wal recovery mode, fail database recovery in case of IOError while reading the WAL to avoid data loss.
-* A new method `Env::LowerThreadPoolCPUPriority(Priority, CpuPriority)` is added to `Env` to be able to lower to a specific priority such as `CpuPriority::kIdle`.
-
-### New Features
-* sst_dump to add a new --readahead_size argument. Users can specify read size when scanning the data. Sst_dump also tries to prefetch tail part of the SST files so usually some number of I/Os are saved there too.
-* Generate file checksum in SstFileWriter if Options.file_checksum_gen_factory is set. The checksum and checksum function name are stored in ExternalSstFileInfo after the sst file write is finished.
-* Add a value_size_soft_limit in read options which limits the cumulative value size of keys read in batches in MultiGet. Once the cumulative value size of found keys exceeds read_options.value_size_soft_limit, all the remaining keys are returned with status Abort without further finding their values. By default the value_size_soft_limit is std::numeric_limits<uint64_t>::max().
-* Enable SST file ingestion with file checksum information when calling IngestExternalFiles(const std::vector<IngestExternalFileArg>& args). Added files_checksums and files_checksum_func_names to IngestExternalFileArg such that user can ingest the sst files with their file checksum information. Added verify_file_checksum to IngestExternalFileOptions (default is True). To be backward compatible, if DB does not enable file checksum or user does not provide checksum information (vectors of files_checksums and files_checksum_func_names are both empty), verification of file checksum is always sucessful. If DB enables file checksum, DB will always generate the checksum for each ingested SST file during Prepare stage of ingestion and store the checksum in Manifest, unless verify_file_checksum is False and checksum information is provided by the application. In this case, we only verify the checksum function name and directly store the ingested checksum in Manifest. If verify_file_checksum is set to True, DB will verify the ingested checksum and function name with the genrated ones. Any mismatch will fail the ingestion. Note that, if IngestExternalFileOptions::write_global_seqno is True, the seqno will be changed in the ingested file. Therefore, the checksum of the file will be changed. In this case, a new checksum will be generated after the seqno is updated and be stored in the Manifest.
-
-### Performance Improvements
-* Eliminate redundant key comparisons during random access in block-based tables.
-
-## 6.10 (2020-05-02)
-### Bug Fixes
-* Fix wrong result being read from ingested file. May happen when a key in the file happen to be prefix of another key also in the file. The issue can further cause more data corruption. The issue exists with rocksdb >= 5.0.0 since DB::IngestExternalFile() was introduced.
-* Finish implementation of BlockBasedTableOptions::IndexType::kBinarySearchWithFirstKey. It's now ready for use. Significantly reduces read amplification in some setups, especially for iterator seeks.
-* Fix a bug by updating CURRENT file so that it points to the correct MANIFEST file after best-efforts recovery.
-* Fixed a bug where ColumnFamilyHandle objects were not cleaned up in case an error happened during BlobDB's open after the base DB had been opened.
-* Fix a potential undefined behavior caused by trying to dereference nullable pointer (timestamp argument) in DB::MultiGet.
-* Fix a bug caused by not including user timestamp in MultiGet LookupKey construction. This can lead to wrong query result since the trailing bytes of a user key, if not shorter than timestamp, will be mistaken for user timestamp.
-* Fix a bug caused by using wrong compare function when sorting the input keys of MultiGet with timestamps.
-* Upgraded version of bzip library (1.0.6 -> 1.0.8) used with RocksJava to address potential vulnerabilities if an attacker can manipulate compressed data saved and loaded by RocksDB (not normal). See issue #6703.
-
-### Public API Change
-* Add a ConfigOptions argument to the APIs dealing with converting options to and from strings and files.  The ConfigOptions is meant to replace some of the options (such as input_strings_escaped and ignore_unknown_options) and allow for more parameters to be passed in the future without changing the function signature.
-* Add NewFileChecksumGenCrc32cFactory to the file checksum public API, such that the builtin Crc32c based file checksum generator factory can be used by applications.
-* Add IsDirectory to Env and FS to indicate if a path is a directory.
-
-### New Features
-* Added support for pipelined & parallel compression optimization for `BlockBasedTableBuilder`. This optimization makes block building, block compression and block appending a pipeline, and uses multiple threads to accelerate block compression. Users can set `CompressionOptions::parallel_threads` greater than 1 to enable compression parallelism. This feature is experimental for now.
-* Provide an allocator for memkind to be used with block cache. This is to work with memory technologies (Intel DCPMM is one such technology currently available) that require different libraries for allocation and management (such as PMDK and memkind). The high capacities available make it possible to provision large caches (up to several TBs in size) beyond what is achievable with DRAM.
-* Option `max_background_flushes` can be set dynamically using DB::SetDBOptions().
-* Added functionality in sst_dump tool to check the compressed file size for different compression levels and print the time spent on compressing files with each compression type. Added arguments `--compression_level_from` and `--compression_level_to` to report size of all compression levels and one compression_type must be specified with it so that it will report compressed sizes of one compression type with different levels.
-* Added statistics for redundant insertions into block cache: rocksdb.block.cache.*add.redundant. (There is currently no coordination to ensure that only one thread loads a table block when many threads are trying to access that same table block.)
-
-### Bug Fixes
-* Fix a bug when making options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts dynamically changeable: the modified values are not written to option files or returned back to users when being queried.
-* Fix a bug where index key comparisons were unaccounted in `PerfContext::user_key_comparison_count` for lookups in files written with `format_version >= 3`.
-* Fix many bloom.filter statistics not being updated in batch MultiGet.
-
-### Performance Improvements
-* Improve performance of batch MultiGet with partitioned filters, by sharing block cache lookups to applicable filter blocks.
-* Reduced memory copies when fetching and uncompressing compressed blocks from sst files.
-
-## 6.9.0 (2020-03-29)
-### Behavior changes
-* Since RocksDB 6.8, ttl-based FIFO compaction can drop a file whose oldest key becomes older than options.ttl while others have not. This fix reverts this and makes ttl-based FIFO compaction use the file's flush time as the criterion. This fix also requires that max_open_files = -1 and compaction_options_fifo.allow_compaction = false to function properly.
-
+## Unreleased
 ### Public API Change
 * Fix spelling so that API now has correctly spelled transaction state name `COMMITTED`, while the old misspelled `COMMITED` is still available as an alias.
-* Updated default format_version in BlockBasedTableOptions from 2 to 4. SST files generated with the new default can be read by RocksDB versions 5.16 and newer, and use more efficient encoding of keys in index blocks.
-* A new parameter `CreateBackupOptions` is added to both `BackupEngine::CreateNewBackup` and `BackupEngine::CreateNewBackupWithMetadata`, you can decrease CPU priority of `BackupEngine`'s background threads by setting `decrease_background_thread_cpu_priority` and `background_thread_cpu_priority` in `CreateBackupOptions`.
-* Updated the public API of SST file checksum. Introduce the FileChecksumGenFactory to create the FileChecksumGenerator for each SST file, such that the FileChecksumGenerator is not shared and it can be more general for checksum implementations. Changed the FileChecksumGenerator interface from Value, Extend, and GetChecksum to Update, Finalize, and GetChecksum. Finalize should be only called once after all data is processed to generate the final checksum. Temproal data should be maintained by the FileChecksumGenerator object itself and finally it can return the checksum string.
 
 ### Bug Fixes
 * Fix a bug where range tombstone blocks in ingested files were cached incorrectly during ingestion. If range tombstones were read from those incorrectly cached blocks, the keys they covered would be exposed.
 * Fix a data race that might cause crash when calling DB::GetCreationTimeOfOldestFile() by a small chance. The bug was introduced in 6.6 Release.
-* Fix a bug where a boolean value optimize_filters_for_hits was for max threads when calling load table handles after a flush or compaction. The value is correct to 1. The bug should not cause user visible problems.
-* Fix a bug which might crash the service when write buffer manager fails to insert the dummy handle to the block cache.
 
 ### Performance Improvements
 * In CompactRange, for levels starting from 0, if the level does not have any file with any key falling in the specified range, the level is skipped. So instead of always compacting from level 0, the compaction starts from the first level with keys in the specified range until the last such level.
-* Reduced memory copy when reading sst footer and blobdb in direct IO mode.
-* When restarting a database with large numbers of sst files, large amount of CPU time is spent on getting logical block size of the sst files, which slows down the starting progress, this inefficiency is optimized away with an internal cache for the logical block sizes.
 
 ### New Features
 * Basic support for user timestamp in iterator. Seek/SeekToFirst/Next and lower/upper bounds are supported. Reverse iteration is not supported. Merge is not considered.
 * When file lock failure when the lock is held by the current process, return acquiring time and thread ID in the error message.
-* Added a new option, best_efforts_recovery (default: false), to allow database to open in a db dir with missing table files. During best efforts recovery, missing table files are ignored, and database recovers to the most recent state without missing table file. Cross-column-family consistency is not guaranteed even if WAL is enabled.
-* options.bottommost_compression, options.compression_opts and options.bottommost_compression_opts are now dynamically changeable.
 
-## 6.8.0 (2020-02-24)
+## 6.8.0 (02/24/2020)
 ### Java API Changes
 * Major breaking changes to Java comparators, toward standardizing on ByteBuffer for performant, locale-neutral operations on keys (#6252).
 * Added overloads of common API methods using direct ByteBuffers for keys and values (#2283).
@@ -483,7 +41,7 @@
 * `db_bench` now supports `value_size_distribution_type`, `value_size_min`, `value_size_max` options for generating random variable sized value. Added `blob_db_compression_type` option for BlobDB to enable blob compression.
 * Replace RocksDB namespace "rocksdb" with flag "ROCKSDB_NAMESPACE" which if is not defined, defined as "rocksdb" in header file rocksdb_namespace.h.
 
-## 6.7.0 (2020-01-21)
+## 6.7.0 (01/21/2020)
 ### Public API Change
 * Added a rocksdb::FileSystem class in include/rocksdb/file_system.h to encapsulate file creation/read/write operations, and an option DBOptions::file_system to allow a user to pass in an instance of rocksdb::FileSystem. If its a non-null value, this will take precendence over DBOptions::env for file operations. A new API rocksdb::FileSystem::Default() returns a platform default object. The DBOptions::env option and Env::Default() API will continue to be used for threading and other OS related functions, and where DBOptions::file_system is not specified, for file operations. For storage developers who are accustomed to rocksdb::Env, the interface in rocksdb::FileSystem is new and will probably undergo some changes as more storage systems are ported to it from rocksdb::Env. As of now, no env other than Posix has been ported to the new interface.
 * A new rocksdb::NewSstFileManager() API that allows the caller to pass in separate Env and FileSystem objects.
@@ -508,11 +66,11 @@
 * Introduce ReadOptions.auto_prefix_mode. When set to true, iterator will return the same result as total order seek, but may choose to use prefix seek internally based on seek key and iterator upper bound.
 * MultiGet() can use IO Uring to parallelize read from the same SST file. This featuer is by default disabled. It can be enabled with environment variable ROCKSDB_USE_IO_URING.
 
-## 6.6.2 (2020-01-13)
+## 6.6.2 (01/13/2020)
 ### Bug Fixes
 * Fixed a bug where non-L0 compaction input files were not considered to compute the `creation_time` of new compaction outputs.
 
-## 6.6.1 (2020-01-02)
+## 6.6.1 (01/02/2020)
 ### Bug Fixes
 * Fix a bug in WriteBatchWithIndex::MultiGetFromBatchAndDB, which is called by Transaction::MultiGet, that causes due to stale pointer access when the number of keys is > 32
 * Fixed two performance issues related to memtable history trimming. First, a new SuperVersion is now created only if some memtables were actually trimmed. Second, trimming is only scheduled if there is at least one flushed memtable that is kept in memory for the purposes of transaction conflict checking.
@@ -522,7 +80,7 @@
 * Delete superversions in BackgroundCallPurge.
 * Fix use-after-free and double-deleting files in BackgroundCallPurge().
 
-## 6.6.0 (2019-11-25)
+## 6.6.0 (11/25/2019)
 ### Bug Fixes
 * Fix data corruption caused by output of intra-L0 compaction on ingested file not being placed in correct order in L0.
 * Fix a data race between Version::GetColumnFamilyMetaData() and Compaction::MarkFilesBeingCompacted() for access to being_compacted (#6056). The current fix acquires the db mutex during Version::GetColumnFamilyMetaData(), which may cause regression.
@@ -575,19 +133,19 @@
 * For 64-bit hashing, RocksDB is standardizing on a slightly modified preview version of XXH3. This function is now used for many non-persisted hashes, along with fastrange64() in place of the modulus operator, and some benchmarks show a slight improvement.
 * Level iterator to invlidate the iterator more often in prefix seek and the level is filtered out by prefix bloom.
 
-## 6.5.2 (2019-11-15)
+## 6.5.2 (11/15/2019)
 ### Bug Fixes
 * Fix a assertion failure in MultiGet() when BlockBasedTableOptions::no_block_cache is true and there is no compressed block cache
 * Fix a buffer overrun problem in BlockBasedTable::MultiGet() when compression is enabled and no compressed block cache is configured.
 * If a call to BackupEngine::PurgeOldBackups or BackupEngine::DeleteBackup suffered a crash, power failure, or I/O error, files could be left over from old backups that could only be purged with a call to GarbageCollect. Any call to PurgeOldBackups, DeleteBackup, or GarbageCollect should now suffice to purge such files.
 
-## 6.5.1 (2019-10-16)
+## 6.5.1 (10/16/2019)
 ### Bug Fixes
 * Revert the feature "Merging iterator to avoid child iterator reseek for some cases (#5286)" since it might cause strange results when reseek happens with a different iterator upper bound.
 * Fix a bug in BlockBasedTableIterator that might return incorrect results when reseek happens with a different iterator upper bound.
 * Fix a bug when partitioned filters and prefix search are used in conjunction, ::SeekForPrev could return invalid for an existing prefix. ::SeekForPrev might be called by the user, or internally on ::Prev, or within ::Seek if the return value involves Delete or a Merge operand.
 
-## 6.5.0 (2019-09-13)
+## 6.5.0 (9/13/2019)
 ### Bug Fixes
 * Fixed a number of data races in BlobDB.
 * Fix a bug where the compaction snapshot refresh feature is not disabled as advertised when `snap_refresh_nanos` is set to 0..
@@ -608,7 +166,7 @@
 ### Performance Improvements
 * Improve the speed of the MemTable Bloom filter, reducing the write overhead of enabling it by 1/3 to 1/2, with similar benefit to read performance.
 
-## 6.4.0 (2019-07-30)
+## 6.4.0 (7/30/2019)
 ### Default Option Change
 * LRUCacheOptions.high_pri_pool_ratio is set to 0.5 (previously 0.0) by default, which means that by default midpoint insertion is enabled. The same change is made for the default value of high_pri_pool_ratio argument in NewLRUCache(). When block cache is not explicitly created, the small block cache created by BlockBasedTable will still has this option to be 0.0.
 * Change BlockBasedTableOptions.cache_index_and_filter_blocks_with_high_priority's default value from false to true.
@@ -644,7 +202,7 @@
 * Fixed a regression where the fill_cache read option also affected index blocks.
 * Fixed an issue where using cache_index_and_filter_blocks==false affected partitions of partitioned indexes/filters as well.
 
-## 6.3.2 (2019-08-15)
+## 6.3.2 (8/15/2019)
 ### Public API Change
 * The semantics of the per-block-type block read counts in the performance context now match those of the generic block_read_count.
 
@@ -652,11 +210,11 @@
 * Fixed a regression where the fill_cache read option also affected index blocks.
 * Fixed an issue where using cache_index_and_filter_blocks==false affected partitions of partitioned indexes as well.
 
-## 6.3.1 (2019-07-24)
+## 6.3.1 (7/24/2019)
 ### Bug Fixes
 * Fix auto rolling bug introduced in 6.3.0, which causes segfault if log file creation fails.
 
-## 6.3.0 (2019-06-18)
+## 6.3.0 (6/18/2019)
 ### Public API Change
 * Now DB::Close() will return Aborted() error when there is unreleased snapshot. Users can retry after all snapshots are released.
 * Index blocks are now handled similarly to data blocks with regards to the block cache: instead of storing objects in the cache, only the blocks themselves are cached. In addition, index blocks no longer get evicted from the cache when a table is closed, can now use the compressed block cache (if any), and can be shared among multiple table readers.
@@ -693,7 +251,7 @@
 * Fix a bug caused by secondary not skipping the beginning of new MANIFEST.
 * On DB open, delete WAL trash files left behind in wal_dir
 
-## 6.2.0 (2019-04-30)
+## 6.2.0 (4/30/2019)
 ### New Features
 * Add an option `strict_bytes_per_sync` that causes a file-writing thread to block rather than exceed the limit on bytes pending writeback specified by `bytes_per_sync` or `wal_bytes_per_sync`.
 * Improve range scan performance by avoiding per-key upper bound check in BlockBasedTableIterator.
@@ -715,7 +273,7 @@
 * Close a WAL file before another thread deletes it.
 * Fix an assertion failure `IsFlushPending() == true` caused by one bg thread releasing the db mutex in ~ColumnFamilyData and another thread clearing `flush_requested_` flag.
 
-## 6.1.1 (2019-04-09)
+## 6.1.1 (4/9/2019)
 ### New Features
 * When reading from option file/string/map, customized comparators and/or merge operators can be filled according to object registry.
 
@@ -725,7 +283,7 @@
 * Fix a bug in 2PC where a sequence of txn prepare, memtable flush, and crash could result in losing the prepared transaction.
 * Fix a bug in Encryption Env which could cause encrypted files to be read beyond file boundaries.
 
-## 6.1.0 (2019-03-27)
+## 6.1.0 (3/27/2019)
 ### New Features
 * Introduce two more stats levels, kExceptHistogramOrTimers and kExceptTimers.
 * Added a feature to perform data-block sampling for compressibility, and report stats to user.
@@ -743,7 +301,7 @@
 * Fix JEMALLOC_CXX_THROW macro missing from older Jemalloc versions, causing build failures on some platforms.
 * Fix SstFileReader not able to open file ingested with write_glbal_seqno=true.
 
-## 6.0.0 (2019-02-19)
+## 6.0.0 (2/19/2019)
 ### New Features
 * Enabled checkpoint on readonly db (DBImplReadOnly).
 * Make DB ignore dropped column families while committing results of atomic flush.
@@ -785,7 +343,7 @@
 ### Change Default Options
 * Change options.compaction_pri's default to kMinOverlappingRatio
 
-## 5.18.0 (2018-11-30)
+## 5.18.0 (11/30/2018)
 ### New Features
 * Introduced `JemallocNodumpAllocator` memory allocator. When being use, block cache will be excluded from core dump.
 * Introduced `PerfContextByLevel` as part of `PerfContext` which allows storing perf context at each level. Also replaced `__thread` with `thread_local` keyword for perf_context. Added per-level perf context for bloom filter and `Get` query.
@@ -813,7 +371,7 @@
 * Start populating `NO_FILE_CLOSES` ticker statistic, which was always zero previously.
 * The default value of NewBloomFilterPolicy()'s argument use_block_based_builder is changed to false. Note that this new default may cause large temp memory usage when building very large SST files.
 
-## 5.17.0 (2018-10-05)
+## 5.17.0 (10/05/2018)
 ### Public API Change
 * `OnTableFileCreated` will now be called for empty files generated during compaction. In that case, `TableFileCreationInfo::file_path` will be "(nil)" and `TableFileCreationInfo::file_size` will be zero.
 * Add `FlushOptions::allow_write_stall`, which controls whether Flush calls start working immediately, even if it causes user writes to stall, or will wait until flush can be performed without causing write stall (similar to `CompactRangeOptions::allow_write_stall`). Note that the default value is false, meaning we add delay to Flush calls until stalling can be avoided when possible. This is behavior change compared to previous RocksDB versions, where Flush calls didn't check if they might cause stall or not.
@@ -827,21 +385,21 @@
 * Avoid creating empty SSTs and subsequently deleting them in certain cases during compaction.
 * Sync CURRENT file contents during checkpoint.
 
-## 5.16.3 (2018-10-01)
+## 5.16.3 (10/1/2018)
 ### Bug Fixes
 * Fix crash caused when `CompactFiles` run with `CompactionOptions::compression == CompressionType::kDisableCompressionOption`. Now that setting causes the compression type to be chosen according to the column family-wide compression options.
 
-## 5.16.2 (2018-09-21)
+## 5.16.2 (9/21/2018)
 ### Bug Fixes
 * Fix bug in partition filters with format_version=4.
 
-## 5.16.1 (2018-09-17)
+## 5.16.1 (9/17/2018)
 ### Bug Fixes
 * Remove trace_analyzer_tool from rocksdb_lib target in TARGETS file.
 * Fix RocksDB Java build and tests.
 * Remove sync point in Block destructor.
 
-## 5.16.0 (2018-08-21)
+## 5.16.0 (8/21/2018)
 ### Public API Change
 * The merge operands are passed to `MergeOperator::ShouldMerge` in the reversed order relative to how they were merged (passed to FullMerge or FullMergeV2) for performance reasons
 * GetAllKeyVersions() to take an extra argument of `max_num_ikeys`.
@@ -855,7 +413,7 @@
 ### Bug Fixes
 * Fix a bug in misreporting the estimated partition index size in properties block.
 
-## 5.15.0 (2018-07-17)
+## 5.15.0 (7/17/2018)
 ### Public API Change
 * Remove managed iterator. ReadOptions.managed is not effective anymore.
 * For bottommost_compression, a compatible CompressionOptions is added via `bottommost_compression_opts`. To keep backward compatible, a new boolean `enabled` is added to CompressionOptions. For compression_opts, it will be always used no matter what value of `enabled` is. For bottommost_compression_opts, it will only be used when user set `enabled=true`, otherwise, compression_opts will be used for bottommost_compression as default.
@@ -881,7 +439,7 @@
 * Fix a bug caused by not copying the block trailer with compressed SST file, direct IO, prefetcher and no compressed block cache.
 * Fix write can stuck indefinitely if enable_pipelined_write=true. The issue exists since pipelined write was introduced in 5.5.0.
 
-## 5.14.0 (2018-05-16)
+## 5.14.0 (5/16/2018)
 ### Public API Change
 * Add a BlockBasedTableOption to align uncompressed data blocks on the smaller of block size or page size boundary, to reduce flash reads by avoiding reads spanning 4K pages.
 * The background thread naming convention changed (on supporting platforms) to "rocksdb:<thread pool priority><thread number>", e.g., "rocksdb:low0".
@@ -914,7 +472,7 @@
 * Add `BlockBasedTableConfig.setBlockCache` to allow sharing a block cache across DB instances.
 * Added SstFileManager to the Java API to allow managing SST files across DB instances.
 
-## 5.13.0 (2018-03-20)
+## 5.13.0 (3/20/2018)
 ### Public API Change
 * RocksDBOptionsParser::Parse()'s `ignore_unknown_options` argument will only be effective if the option file shows it is generated using a higher version of RocksDB than the current version.
 * Remove CompactionEventListener.
@@ -930,7 +488,7 @@
 * Fix a leak in prepared_section_completed_ where the zeroed entries would not removed from the map.
 * Fix WAL corruption caused by race condition between user write thread and backup/checkpoint thread.
 
-## 5.12.0 (2018-02-14)
+## 5.12.0 (2/14/2018)
 ### Public API Change
 * Iterator::SeekForPrev is now a pure virtual method. This is to prevent user who implement the Iterator interface fail to implement SeekForPrev by mistake.
 * Add `include_end` option to make the range end exclusive when `include_end == false` in `DeleteFilesInRange()`.
@@ -952,7 +510,7 @@
 * Fix advance reservation of arena block addresses.
 * Fix handling of empty string as checkpoint directory.
 
-## 5.11.0 (2018-01-08)
+## 5.11.0 (01/08/2018)
 ### Public API Change
 * Add `autoTune` and `getBytesPerSecond()` to RocksJava RateLimiter
 
@@ -969,7 +527,7 @@
 * Fix a mislabel bug for bottom-pri compaction threads.
 * Fix DB::Flush() keep waiting after flush finish under certain condition.
 
-## 5.10.0 (2017-12-11)
+## 5.10.0 (12/11/2017)
 ### Public API Change
 * When running `make` with environment variable `USE_SSE` set and `PORTABLE` unset, will use all machine features available locally. Previously this combination only compiled SSE-related features.
 
@@ -984,7 +542,7 @@
 * Fix performance issue in `IngestExternalFile()` affecting databases with large number of SST files.
 * Fix possible corruption to LSM structure when `DeleteFilesInRange()` deletes a subset of files spanned by a `DeleteRange()` marker.
 
-## 5.9.0 (2017-11-01)
+## 5.9.0 (11/1/2017)
 ### Public API Change
 * `BackupableDBOptions::max_valid_backups_to_open == 0` now means no backups will be opened during BackupEngine initialization. Previously this condition disabled limiting backups opened.
 * `DBOptions::preserve_deletes` is a new option that allows one to specify that DB should not drop tombstones for regular deletes if they have sequence number larger than what was set by the new API call `DB::SetPreserveDeletesSequenceNumber(SequenceNumber seqnum)`. Disabled by default.
@@ -1011,7 +569,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Fix a potential data inconsistency issue during point-in-time recovery. `DB:Open()` will abort if column family inconsistency is found during PIT recovery.
 * Fix possible metadata corruption in databases using `DeleteRange()`.
 
-## 5.8.0 (2017-08-30)
+## 5.8.0 (08/30/2017)
 ### Public API Change
 * Users of `Statistics::getHistogramString()` will see fewer histogram buckets and different bucket endpoints.
 * `Slice::compare` and BytewiseComparator `Compare` no longer accept `Slice`s containing nullptr.
@@ -1031,7 +589,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Fix transient reappearance of keys covered by range deletions when memtable prefix bloom filter is enabled.
 * Fix potentially wrong file smallest key when range deletions separated by snapshot are written together.
 
-## 5.7.0 (2017-07-13)
+## 5.7.0 (07/13/2017)
 ### Public API Change
 * DB property "rocksdb.sstables" now prints keys in hex form.
 
@@ -1046,7 +604,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Fix discarding empty compaction output files when `DeleteRange()` is used together with subcompactions.
 
-## 5.6.0 (2017-06-06)
+## 5.6.0 (06/06/2017)
 ### Public API Change
 * Scheduling flushes and compactions in the same thread pool is no longer supported by setting `max_background_flushes=0`. Instead, users can achieve this by configuring their high-pri thread pool to have zero threads.
 * Replace `Options::max_background_flushes`, `Options::max_background_compactions`, and `Options::base_background_compactions` all with `Options::max_background_jobs`, which automatically decides how many threads to allocate towards flush/compaction.
@@ -1063,7 +621,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Shouldn't ignore return value of fsync() in flush.
 
-## 5.5.0 (2017-05-17)
+## 5.5.0 (05/17/2017)
 ### New Features
 * FIFO compaction to support Intra L0 compaction too with CompactionOptionsFIFO.allow_compaction=true.
 * DB::ResetStats() to reset internal stats.
@@ -1080,7 +638,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Fix the bug that Direct I/O uses direct reads for non-SST file
 
-## 5.4.0 (2017-04-11)
+## 5.4.0 (04/11/2017)
 ### Public API Change
 * random_access_max_buffer_size no longer has any effect
 * Removed Env::EnableReadAhead(), Env::ShouldForwardRawRequest()
@@ -1097,7 +655,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Introduce level-based L0->L0 compactions to reduce file count, so write delays are incurred less often.
 * (Experimental) Partitioning filters which creates an index on the partitions. The feature can be enabled by setting partition_filters when using kFullFilter. Currently the feature also requires two-level indexing to be enabled. Number of partitions is the same as the number of partitions for indexes, which is controlled by metadata_block_size.
 
-## 5.3.0 (2017-03-08)
+## 5.3.0 (03/08/2017)
 ### Public API Change
 * Remove disableDataSync option.
 * Remove timeout_hint_us option from WriteOptions. The option has been deprecated and has no effect since 3.13.0.
@@ -1107,7 +665,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Bug Fixes
 * Fix the bug that iterator may skip keys
 
-## 5.2.0 (2017-02-08)
+## 5.2.0 (02/08/2017)
 ### Public API Change
 * NewLRUCache() will determine number of shard bits automatically based on capacity, if the user doesn't pass one. This also impacts the default block cache when the user doesn't explicit provide one.
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
@@ -1125,7 +683,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Some fixes related to 2PC.
 * Fix bugs of data corruption in direct I/O
 
-## 5.1.0 (2017-01-13)
+## 5.1.0 (01/13/2017)
 * Support dynamically change `delete_obsolete_files_period_micros` option via SetDBOptions().
 * Added EventListener::OnExternalFileIngested which will be called when IngestExternalFile() add a file successfully.
 * BackupEngine::Open and BackupEngineReadOnly::Open now always return error statuses matching those of the backup Env.
@@ -1134,7 +692,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Fix the bug that if 2PC is enabled, checkpoints may loss some recent transactions.
 * When file copying is needed when creating checkpoints or bulk loading files, fsync the file after the file copying.
 
-## 5.0.0 (2016-11-17)
+## 5.0.0 (11/17/2016)
 ### Public API Change
 * Options::max_bytes_for_level_multiplier is now a double along with all getters and setters.
 * Support dynamically change `delayed_write_rate` and `max_total_wal_size` options via SetDBOptions().
@@ -1153,7 +711,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add LuaCompactionFilter in utilities.  This allows developers to write compaction filters in Lua.  To use this feature, LUA_PATH needs to be set to the root directory of Lua.
 * No longer populate "LATEST_BACKUP" file in backup directory, which formerly contained the number of the latest backup. The latest backup can be determined by finding the highest numbered file in the "meta/" subdirectory.
 
-## 4.13.0 (2016-10-18)
+## 4.13.0 (10/18/2016)
 ### Public API Change
 * DB::GetOptions() reflect dynamic changed options (i.e. through DB::SetOptions()) and return copy of options instead of reference.
 * Added Statistics::getAndResetTickerCount().
@@ -1162,7 +720,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add DB::SetDBOptions() to dynamic change base_background_compactions and max_background_compactions.
 * Added Iterator::SeekForPrev(). This new API will seek to the last key that less than or equal to the target key.
 
-## 4.12.0 (2016-09-12)
+## 4.12.0 (9/12/2016)
 ### Public API Change
 * CancelAllBackgroundWork() flushes all memtables for databases containing writes that have bypassed the WAL (writes issued with WriteOptions::disableWAL=true) before shutting down background threads.
 * Merge options source_compaction_factor, max_grandparent_overlap_bytes and expanded_compaction_factor into max_compaction_bytes.
@@ -1174,7 +732,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Change ticker/histogram statistics implementations to accumulate data in thread-local storage, which improves CPU performance by reducing cache coherency costs. Callers of CreateDBStatistics do not need to change anything to use this feature.
 * Block cache mid-point insertion, where index and filter block are inserted into LRU block cache with higher priority. The feature can be enabled by setting BlockBasedTableOptions::cache_index_and_filter_blocks_with_high_priority to true and high_pri_pool_ratio > 0 when creating NewLRUCache.
 
-## 4.11.0 (2016-08-01)
+## 4.11.0 (8/1/2016)
 ### Public API Change
 * options.memtable_prefix_bloom_huge_page_tlb_size => memtable_huge_page_size. When it is set, RocksDB will try to allocate memory from huge page for memtable too, rather than just memtable bloom filter.
 
@@ -1182,7 +740,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * A tool to migrate DB after options change. See include/rocksdb/utilities/option_change_migration.h.
 * Add ReadOptions.background_purge_on_iterator_cleanup. If true, we avoid file deletion when destroying iterators.
 
-## 4.10.0 (2016-07-05)
+## 4.10.0 (7/5/2016)
 ### Public API Change
 * options.memtable_prefix_bloom_bits changes to options.memtable_prefix_bloom_bits_ratio and deprecate options.memtable_prefix_bloom_probes
 * enum type CompressionType and PerfLevel changes from char to unsigned char. Value of all PerfLevel shift by one.
@@ -1194,7 +752,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * RepairDB support for column families. RepairDB now associates data with non-default column families using information embedded in the SST/WAL files (4.7 or later). For data written by 4.6 or earlier, RepairDB associates it with the default column family.
 * Add options.write_buffer_manager which allows users to control total memtable sizes across multiple DB instances.
 
-## 4.9.0 (2016-06-09)
+## 4.9.0 (6/9/2016)
 ### Public API changes
 * Add bottommost_compression option, This option can be used to set a specific compression algorithm for the bottommost level (Last level containing files in the DB).
 * Introduce CompactionJobInfo::compression, This field state the compression algorithm used to generate the output files of the compaction.
@@ -1204,7 +762,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### New Features
 * Introduce NewSimCache() in rocksdb/utilities/sim_cache.h. This function creates a block cache that is able to give simulation results (mainly hit rate) of simulating block behavior with a configurable cache size.
 
-## 4.8.0 (2016-05-02)
+## 4.8.0 (5/2/2016)
 ### Public API Change
 * Allow preset compression dictionary for improved compression of block-based tables. This is supported for zlib, zstd, and lz4. The compression dictionary's size is configurable via CompressionOptions::max_dict_bytes.
 * Delete deprecated classes for creating backups (BackupableDB) and restoring from backups (RestoreBackupableDB). Now, BackupEngine should be used for creating backups, and BackupEngineReadOnly should be used for restorations. For more details, see https://github.com/facebook/rocksdb/wiki/How-to-backup-RocksDB%3F
@@ -1214,12 +772,12 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### New Features
 * Add ReadOptions::readahead_size. If non-zero, NewIterator will create a new table reader which performs reads of the given size.
 
-## 4.7.0 (2016-04-08)
+## 4.7.0 (4/8/2016)
 ### Public API Change
 * rename options compaction_measure_io_stats to report_bg_io_stats and include flush too.
 * Change some default options. Now default options will optimize for server-workloads. Also enable slowdown and full stop triggers for pending compaction bytes. These changes may cause sub-optimal performance or significant increase of resource usage. To avoid these risks, users can open existing RocksDB with options extracted from RocksDB option files. See https://github.com/facebook/rocksdb/wiki/RocksDB-Options-File for how to use RocksDB option files. Or you can call Options.OldDefaults() to recover old defaults. DEFAULT_OPTIONS_HISTORY.md will track change history of default options.
 
-## 4.6.0 (2016-03-10)
+## 4.6.0 (3/10/2016)
 ### Public API Changes
 * Change default of BlockBasedTableOptions.format_version to 2. It means default DB created by 4.6 or up cannot be opened by RocksDB version 3.9 or earlier.
 * Added strict_capacity_limit option to NewLRUCache. If the flag is set to true, insert to cache will fail if no enough capacity can be free. Signature of Cache::Insert() is updated accordingly.
@@ -1230,7 +788,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add CompactionPri::kMinOverlappingRatio, a compaction picking mode friendly to write amplification.
 * Deprecate Iterator::IsKeyPinned() and replace it with Iterator::GetProperty() with prop_name="rocksdb.iterator.is.key.pinned"
 
-## 4.5.0 (2016-02-05)
+## 4.5.0 (2/5/2016)
 ### Public API Changes
 * Add a new perf context level between kEnableCount and kEnableTime. Level 2 now does not include timers for mutexes.
 * Statistics of mutex operation durations will not be measured by default. If you want to have them enabled, you need to set Statistics::stats_level_ to kAll.
@@ -1241,7 +799,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add kPersistedTier to ReadTier.  This option allows Get and MultiGet to read only the persited data and skip mem-tables if writes were done with disableWAL = true.
 * Add DBOptions::sst_file_manager. Use NewSstFileManager() in include/rocksdb/sst_file_manager.h to create a SstFileManager that can be used to track the total size of SST files and control the SST files deletion rate.
 
-## 4.4.0 (2016-01-14)
+## 4.4.0 (1/14/2016)
 ### Public API Changes
 * Change names in CompactionPri and add a new one.
 * Deprecate options.soft_rate_limit and add options.soft_pending_compaction_bytes_limit.
@@ -1251,7 +809,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Increase default options.delayed_write_rate to 2MB/s.
 * Added a new parameter --path to ldb tool. --path accepts the name of either MANIFEST, SST or a WAL file. Either --db or --path can be used when calling ldb.
 
-## 4.3.0 (2015-12-08)
+## 4.3.0 (12/8/2015)
 ### New Features
 * CompactionFilter has new member function called IgnoreSnapshots which allows CompactionFilter to be called even if there are snapshots later than the key.
 * RocksDB will now persist options under the same directory as the RocksDB database on successful DB::Open, CreateColumnFamily, DropColumnFamily, and SetOptions.
@@ -1261,7 +819,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Public API Changes
 * When options.db_write_buffer_size triggers, only the column family with the largest column family size will be flushed, not all the column families.
 
-## 4.2.0 (2015-11-09)
+## 4.2.0 (11/9/2015)
 ### New Features
 * Introduce CreateLoggerFromOptions(), this function create a Logger for provided DBOptions.
 * Add GetAggregatedIntProperty(), which returns the sum of the GetIntProperty of all the column families.
@@ -1274,7 +832,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Remove DefaultCompactionFilterFactory.
 
 
-## 4.1.0 (2015-10-08)
+## 4.1.0 (10/8/2015)
 ### New Features
 * Added single delete operation as a more efficient way to delete keys that have not been overwritten.
 * Added experimental AddFile() to DB interface that allow users to add files created by SstFileWriter into an empty Database, see include/rocksdb/sst_file_writer.h and DB::AddFile() for more info.
@@ -1288,7 +846,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * CompactionFilter has a new method FilterMergeOperand() that RocksDB applies to every merge operand during compaction to decide whether to filter the operand.
 * We removed CompactionFilterV2 interfaces from include/rocksdb/compaction_filter.h. The functionality was deprecated already in version 3.13.
 
-## 4.0.0 (2015-09-09)
+## 4.0.0 (9/9/2015)
 ### New Features
 * Added support for transactions.  See include/rocksdb/utilities/transaction.h for more info.
 * DB::GetProperty() now accepts "rocksdb.aggregated-table-properties" and "rocksdb.aggregated-table-properties-at-levelN", in which case it returns aggregated table properties of the target column family, or the aggregated table properties of the specified level N if the "at-level" version is used.
@@ -1301,7 +859,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Added Equal() method to the Comparator interface that can optionally be overwritten in cases where equality comparisons can be done more efficiently than three-way comparisons.
 * Previous 'experimental' OptimisticTransaction class has been replaced by Transaction class.
 
-## 3.13.0 (2015-08-06)
+## 3.13.0 (8/6/2015)
 ### New Features
 * RollbackToSavePoint() in WriteBatch/WriteBatchWithIndex
 * Add NewCompactOnDeletionCollectorFactory() in utilities/table_properties_collectors, which allows rocksdb to mark a SST file as need-compaction when it observes at least D deletion entries in any N consecutive entries in that SST file.  Note that this feature depends on an experimental NeedCompact() API --- the result of this API will not persist after DB restart.
@@ -1316,7 +874,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add statistics::getHistogramString() to print detailed distribution of a histogram metric.
 * Add DBOptions::skip_stats_update_on_db_open.  When it is on, DB::Open() will run faster as it skips the random reads required for loading necessary stats from SST files to optimize compaction.
 
-## 3.12.0 (2015-07-02)
+## 3.12.0 (7/2/2015)
 ### New Features
 * Added experimental support for optimistic transactions.  See include/rocksdb/utilities/optimistic_transaction.h for more info.
 * Added a new way to report QPS from db_bench (check out --report_file and --report_interval_seconds)
@@ -1346,7 +904,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add BackupEngineImpl.options_.max_background_operations to specify the maximum number of operations that may be performed in parallel. Add support for parallelized backup and restore.
 * Add DB::SyncWAL() that does a WAL sync without blocking writers.
 
-## 3.11.0 (2015-05-19)
+## 3.11.0 (5/19/2015)
 ### New Features
 * Added a new API Cache::SetCapacity(size_t capacity) to dynamically change the maximum configured capacity of the cache. If the new capacity is less than the existing cache usage, the implementation will try to lower the usage by evicting the necessary number of elements following a strict LRU policy.
 * Added an experimental API for handling flashcache devices (blacklists background threads from caching their reads) -- NewFlashcacheAwareEnv
@@ -1357,7 +915,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * TablePropertiesCollector::AddUserKey() is added to replace TablePropertiesCollector::Add(). AddUserKey() exposes key type, sequence number and file size up to now to users.
 * DBOptions::bytes_per_sync used to apply to both WAL and table files. As of 3.11 it applies only to table files. If you want to use this option to sync WAL in the background, please use wal_bytes_per_sync
 
-## 3.10.0 (2015-03-24)
+## 3.10.0 (3/24/2015)
 ### New Features
 * GetThreadStatus() is now able to report detailed thread status, including:
  - Thread Operation including flush and compaction.
@@ -1392,7 +950,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * lz4 compression is now included in rocksjava static library when running `make rocksdbjavastatic`.
 * Overflowing a size_t when setting rocksdb options now throws an IllegalArgumentException, which removes the necessity for a developer to catch these Exceptions explicitly.
 
-## 3.9.0 (2014-12-08)
+## 3.9.0 (12/8/2014)
 
 ### New Features
 * Add rocksdb::GetThreadList(), which in the future will return the current status of all
@@ -1411,7 +969,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Improvements
 * RocksDBLite library now becomes smaller and will be compiled with -fno-exceptions flag.
 
-## 3.8.0 (2014-11-14)
+## 3.8.0 (11/14/2014)
 
 ### Public API changes
 * BackupEngine::NewBackupEngine() was deprecated; please use BackupEngine::Open() from now on.
@@ -1425,14 +983,14 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * CompactFiles and EventListener, although they are still in experimental state
 * Full ColumnFamily support in RocksJava.
 
-## 3.7.0 (2014-11-06)
+## 3.7.0 (11/6/2014)
 ### Public API changes
 * Introduce SetOptions() API to allow adjusting a subset of options dynamically online
 * Introduce 4 new convenient functions for converting Options from string: GetColumnFamilyOptionsFromMap(), GetColumnFamilyOptionsFromString(), GetDBOptionsFromMap(), GetDBOptionsFromString()
 * Remove WriteBatchWithIndex.Delete() overloads using SliceParts
 * When opening a DB, if options.max_background_compactions is larger than the existing low pri pool of options.env, it will enlarge it. Similarly, options.max_background_flushes is larger than the existing high pri pool of options.env, it will enlarge it.
 
-## 3.6.0 (2014-10-07)
+## 3.6.0 (10/7/2014)
 ### Disk format changes
 * If you're using RocksDB on ARM platforms and you're using default bloom filter, there is a disk format change you need to be aware of. There are three steps you need to do when you convert to new release: 1. turn off filter policy, 2. compact the whole database, 3. turn on filter policy
 
@@ -1445,7 +1003,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Change target_file_size_base type to uint64_t from int.
 * Remove allow_thread_local. This feature was proved to be stable, so we are turning it always-on.
 
-## 3.5.0 (2014-09-03)
+## 3.5.0 (9/3/2014)
 ### New Features
 * Add include/utilities/write_batch_with_index.h, providing a utility class to query data out of WriteBatch when building it.
 * Move BlockBasedTable related options to BlockBasedTableOptions from Options. Change corresponding JNI interface. Options affected include:
@@ -1456,7 +1014,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Public API changes
 * The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.
 
-## 3.4.0 (2014-08-18)
+## 3.4.0 (8/18/2014)
 ### New Features
 * Support Multiple DB paths in universal style compactions
 * Add feature of storing plain table index and bloom filter in SST file.
@@ -1472,7 +1030,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 * Add DB::GetIntProperty(), which returns DB properties that are integer as uint64_t.
 * The Prefix Extractor used with V2 compaction filters is now passed user key to SliceTransform::Transform instead of unparsed RocksDB key.
 
-## 3.3.0 (2014-07-10)
+## 3.3.0 (7/10/2014)
 ### New Features
 * Added JSON API prototype.
 * HashLinklist reduces performance outlier caused by skewed bucket by switching data in the bucket from linked list to skip list. Add parameter threshold_use_skiplist in NewHashLinkListRepFactory().
@@ -1483,7 +1041,7 @@ if set to something > 0 user will see 2 changes in iterators behavior 1) only ke
 ### Public API changes
 * Removed NewTotalOrderPlainTableFactory because it is not used and implemented semantically incorrect.
 
-## 3.2.0 (2014-06-20)
+## 3.2.0 (06/20/2014)
 
 ### Public API changes
 * We removed seek compaction as a concept from RocksDB because:
@@ -1501,7 +1059,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 ### Performance Improvements
 * Tailing Iterator re-implemeted with ForwardIterator + Cascading Search Hint , see ~20% throughput improvement.
 
-## 3.1.0 (2014-05-21)
+## 3.1.0 (05/21/2014)
 
 ### Public API changes
 * Replaced ColumnFamilyOptions::table_properties_collectors with ColumnFamilyOptions::table_properties_collector_factories
@@ -1510,7 +1068,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 * Hash index for block-based table will be materialized and reconstructed more efficiently. Previously hash index is constructed by scanning the whole table during every table open.
 * FIFO compaction style
 
-## 3.0.0 (2014-05-05)
+## 3.0.0 (05/05/2014)
 
 ### Public API changes
 * Added _LEVEL to all InfoLogLevel enums
@@ -1522,7 +1080,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 * Added an option to use different checksum functions in BlockBasedTableOptions
 * Added ApplyToAllCacheEntries() function to Cache
 
-## 2.8.0 (2014-04-04)
+## 2.8.0 (04/04/2014)
 
 * Removed arena.h from public header files.
 * By default, checksums are verified on every read from database
@@ -1551,7 +1109,7 @@ Because of that, Options::disable_seek_compaction is now obsolete. It is still a
 * Now compaction filter has a V2 interface. It buffers the kv-pairs sharing the same key prefix, process them in batches, and return the batched results back to DB. The new interface uses a new structure CompactionFilterContext for the same purpose as CompactionFilter::Context in V1.
 * Geo-spatial support for locations and radial-search.
 
-## 2.7.0 (2014-01-28)
+## 2.7.0 (01/28/2014)
 
 ### Public API changes
 

INSTALL.md

@@ -43,8 +43,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       command line flags processing. You can compile rocksdb library even
       if you don't have gflags installed.
 
-* `make check` will also check code formatting, which requires [clang-format](https://clang.llvm.org/docs/ClangFormat.html)
-
 * If you wish to build the RocksJava static target, then cmake is required for building Snappy.
 
 ## Supported platforms
@@ -96,21 +94,12 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
               sudo yum install libasan
 
     * Install zstandard:
-        * With [EPEL](https://fedoraproject.org/wiki/EPEL):
 
-              sudo yum install libzstd-devel
-
-        * With CentOS 8:
-
-              sudo dnf install libzstd-devel
-
-        * From source:
-
-              wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
-              mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
-              tar zxvf zstd-1.1.3.tar.gz
-              cd zstd-1.1.3
-              make && sudo make install
+             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+             mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
+             tar zxvf zstd-1.1.3.tar.gz
+             cd zstd-1.1.3
+             make && sudo make install
 
 * **OS X**:
     * Install latest C++ compiler that supports C++ 11:

LANGUAGE-BINDINGS.md

@@ -10,9 +10,7 @@ This is the list of all known third-party language bindings for RocksDB. If some
 * Ruby - http://rubygems.org/gems/rocksdb-ruby
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
-* C#
-    * https://github.com/warrenfalk/rocksdb-sharp
-    * https://github.com/curiosity-ai/rocksdb-sharp
+* C# - https://github.com/warrenfalk/rocksdb-sharp
 * Rust
     * https://github.com/pingcap/rust-rocksdb (used in production fork of https://github.com/spacejam/rust-rocksdb)
     * https://github.com/spacejam/rust-rocksdb

PLUGINS.md

@@ -1,4 +0,0 @@
-This is the list of all known third-party plugins for RocksDB. If something is missing, please open a pull request to add it.
-
-* [Dedupfs](https://github.com/ajkr/dedupfs): an example for plugin developers to reference
-* [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices

README.md

@@ -1,9 +1,8 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
 
-[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb)
-[![TravisCI Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
-[![Appveyor Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
-[![PPC64le Build Status](http://140-211-168-68-openstack.osuosl.org:8080/buildStatus/icon?job=rocksdb&style=plastic)](http://140-211-168-68-openstack.osuosl.org:8080/job/rocksdb)
+[![Linux/Mac Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
+[![Windows Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
+[![PPC64le Build Status](http://140.211.168.68:8080/buildStatus/icon?job=Rocksdb)](http://140.211.168.68:8080/job/Rocksdb)
 
 RocksDB is developed and maintained by Facebook Database Engineering Team.
 It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com)
@@ -25,7 +24,7 @@ The public interface is in `include/`.  Callers should not include or
 rely on the details of any other header files in this package.  Those
 internal APIs may be changed without warning.
 
-Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/ and https://rocksdb.slack.com/
+Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/
 
 ## License
 

USERS.md

@@ -26,9 +26,6 @@ Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasu
 ## Yahoo
 Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
 
-## Baidu
-[Apache Doris](http://doris.apache.org/master/en/) is a MPP analytical database engine released by Baidu. It [uses RocksDB](http://doris.apache.org/master/en/administrator-guide/operation/tablet-meta-tool.html) to manage its tablet's metadata.
-
 ## CockroachDB
 CockroachDB is an open-source geo-replicated transactional database. They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
 
@@ -47,7 +44,7 @@ Tango is using RocksDB as a graph storage to store all users' connection data an
 Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
 Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
 
-## Santander UK/Cloudera Profession Services
+## Santanader UK/Cloudera Profession Services
 Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/
 
 ## Airbnb
@@ -70,7 +67,7 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 [VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
 
 ## quasardb
-[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
 quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## Netflix
@@ -89,7 +86,7 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 [Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
 
 ## 360 Pika
-[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been used in many companies.
+[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
 
 ## LzLabs
 LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
@@ -99,19 +96,13 @@ LzLabs is using RocksDB as a storage engine in their multi-database distributed
 
 ## IOTA Foundation
  [IOTA Foundation](https://www.iota.org/) is using RocksDB in the [IOTA Reference Implementation (IRI)](https://github.com/iotaledger/iri) to store the local state of the Tangle. The Tangle is the first open-source distributed ledger powering the future of the Internet of Things.
-
+ 
 ## Avrio Project
  [Avrio Project](http://avrio-project.github.io/avrio.network/) is using RocksDB in [Avrio ](https://github.com/avrio-project/avrio) to store blocks, account balances and data and other blockchain-releated data. Avrio is a multiblockchain decentralized cryptocurrency empowering monetary transactions.
-
+ 
 ## Crux
 [Crux](https://github.com/juxt/crux) is a document database that uses RocksDB for local [EAV](https://en.wikipedia.org/wiki/Entity%E2%80%93attribute%E2%80%93value_model) index storage to enable point-in-time bitemporal Datalog queries. The "unbundled" architecture uses Kafka to provide horizontal scalability.
 
 ## Nebula Graph
+
 [Nebula Graph](https://github.com/vesoft-inc/nebula) is a distributed, scalable, lightning-fast, open source graph database capable of hosting super large scale graphs with dozens of billions of vertices (nodes) and trillions of edges, with milliseconds of latency.
-
-## YugabyteDB
-[YugabyteDB](https://www.yugabyte.com/) is an open source, high performance, distributed SQL database that uses RocksDB as its storage layer. For more information, please see https://github.com/yugabyte/yugabyte-db/.
-
-## ArangoDB
-[ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its sotrage engine.
-

appveyor.yml

@@ -1,6 +1,6 @@
 version: 1.0.{build}
 
-image: Visual Studio 2019
+image: Visual Studio 2017
 
 environment:
   JAVA_HOME: C:\Program Files\Java\jdk1.8.0
@@ -34,8 +34,7 @@ install:
   - cd snappy-1.1.7
   - mkdir build
   - cd build
-  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT%
+  - cmake -G "%CMAKE_GENERATOR%" ..
   - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
   - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
   - echo "Building LZ4 dependency..."
@@ -58,8 +57,7 @@ install:
 before_build:
   - md %APPVEYOR_BUILD_FOLDER%\build
   - cd %APPVEYOR_BUILD_FOLDER%\build
-  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT% %CMAKE_OPT% -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 -DWITH_ALL_TESTS=0
+  - cmake -G "%CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 ..
   - cd ..
 
 build:
@@ -70,7 +68,7 @@ build:
 test:
 
 test_script:
-  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,env_basic_test -Concurrency 8
+  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_with_timestamp_basic_test,db_test2,db_test,env_basic_test,env_test,db_merge_operand_test -Concurrency 8
 
 on_failure:
   - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip

buckifier/buckify_rocksdb.py

@@ -20,14 +20,14 @@ from util import ColorString
 # User can pass extra dependencies as a JSON object via command line, and this
 # script can include these dependencies in the generate TARGETS file.
 # Usage:
-# $python3 buckifier/buckify_rocksdb.py
+# $python buckifier/buckify_rocksdb.py
 # (This generates a TARGET file without user-specified dependency for unit
 # tests.)
-# $python3 buckifier/buckify_rocksdb.py \
-#        '{"fake": {
-#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],
-#                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]
-#                  }
+# $python buckifier/buckify_rocksdb.py \
+#        '{"fake": { \
+#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],  \
+#                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"], \
+#                  } \
 #         }'
 # (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
 # unit tests, and will use the extra_compiler_flags to compile the unit test
@@ -48,8 +48,8 @@ def parse_src_mk(repo_path):
         if '=' in line:
             current_src = line.split('=')[0].strip()
             src_files[current_src] = []
-        elif '.c' in line:
-            src_path = line.split('\\')[0].strip()
+        elif '.cc' in line:
+            src_path = line.split('.cc')[0].strip() + '.cc'
             src_files[current_src].append(src_path)
     return src_files
 
@@ -69,28 +69,45 @@ def get_cc_files(repo_path):
     return cc_files
 
 
-# Get non_parallel tests from Makefile
-def get_non_parallel_tests(repo_path):
+# Get tests from Makefile
+def get_tests(repo_path):
     Makefile = repo_path + "/Makefile"
 
-    s = set({})
+    # Dictionary TEST_NAME => IS_PARALLEL
+    tests = {}
 
-    found_non_parallel_tests = False
+    found_tests = False
     for line in open(Makefile):
         line = line.strip()
-        if line.startswith("NON_PARALLEL_TEST ="):
-            found_non_parallel_tests = True
-        elif found_non_parallel_tests:
+        if line.startswith("TESTS ="):
+            found_tests = True
+        elif found_tests:
             if line.endswith("\\"):
                 # remove the trailing \
                 line = line[:-1]
                 line = line.strip()
-                s.add(line)
+                tests[line] = False
             else:
-                # we consumed all the non_parallel tests
+                # we consumed all the tests
                 break
 
-    return s
+    found_parallel_tests = False
+    for line in open(Makefile):
+        line = line.strip()
+        if line.startswith("PARALLEL_TEST ="):
+            found_parallel_tests = True
+        elif found_parallel_tests:
+            if line.endswith("\\"):
+                # remove the trailing \
+                line = line[:-1]
+                line = line.strip()
+                tests[line] = True
+            else:
+                # we consumed all the parallel tests
+                break
+
+    return tests
+
 
 # Parse extra dependencies passed by user from command line
 def get_dependencies():
@@ -123,38 +140,18 @@ def generate_targets(repo_path, deps_map):
     src_mk = parse_src_mk(repo_path)
     # get all .cc files
     cc_files = get_cc_files(repo_path)
-    # get non_parallel tests from Makefile
-    non_parallel_tests = get_non_parallel_tests(repo_path)
+    # get tests from Makefile
+    tests = get_tests(repo_path)
 
-    if src_mk is None or cc_files is None or non_parallel_tests is None:
+    if src_mk is None or cc_files is None or tests is None:
         return False
 
-    extra_argv = ""
-    if len(sys.argv) >= 2:
-        # Heuristically quote and canonicalize whitespace for inclusion
-        # in how the file was generated.
-        extra_argv = " '{0}'".format(" ".join(sys.argv[1].split()))
-
-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, extra_argv)
-
+    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
     # rocksdb_lib
     TARGETS.add_library(
         "rocksdb_lib",
         src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] +
         src_mk["TOOL_LIB_SOURCES"])
-    # rocksdb_whole_archive_lib
-    TARGETS.add_library(
-        "rocksdb_whole_archive_lib",
-        src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] +
-        src_mk["TOOL_LIB_SOURCES"],
-        deps=None,
-        headers=None,
-        extra_external_deps="",
-        link_whole=True)
     # rocksdb_test_lib
     TARGETS.add_library(
         "rocksdb_test_lib",
@@ -162,10 +159,7 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("TEST_LIB_SOURCES", []) +
         src_mk.get("EXP_LIB_SOURCES", []) +
         src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-        extra_external_deps=""" + [
-        ("googletest", None, "gtest"),
-    ]""")
+        [":rocksdb_lib"])
     # rocksdb_tools_lib
     TARGETS.add_library(
         "rocksdb_tools_lib",
@@ -173,56 +167,41 @@ def generate_targets(repo_path, deps_map):
         src_mk.get("ANALYZER_LIB_SOURCES", []) +
         ["test_util/testutil.cc"],
         [":rocksdb_lib"])
-    # rocksdb_cache_bench_tools_lib
-    TARGETS.add_library(
-        "rocksdb_cache_bench_tools_lib",
-        src_mk.get("CACHE_BENCH_LIB_SOURCES", []),
-        [":rocksdb_lib"])
     # rocksdb_stress_lib
-    TARGETS.add_rocksdb_library(
+    TARGETS.add_library(
         "rocksdb_stress_lib",
         src_mk.get("ANALYZER_LIB_SOURCES", [])
         + src_mk.get('STRESS_LIB_SOURCES', [])
-        + ["test_util/testutil.cc"])
-
-    print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
-
-    # Dictionary test executable name -> relative source file path
-    test_source_map = {}
-    print(src_mk)
-
-    # c_test.c is added through TARGETS.add_c_test(). If there
-    # are more than one .c test file, we need to extend
-    # TARGETS.add_c_test() to include other C tests too.
-    for test_src in src_mk.get("TEST_MAIN_SOURCES_C", []):
-        if test_src != 'db/c_test.c':
-            print("Don't know how to deal with " + test_src)
-            return False
-    TARGETS.add_c_test()
-
-    for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
-        test = test_src.split('.c')[0].strip().split('/')[-1].strip()
-        test_source_map[test] = test_src
-        print("" + test + " " + test_src)
+        + ["test_util/testutil.cc"],
+        [":rocksdb_lib"])
 
+    print("Extra dependencies:\n{0}".format(str(deps_map)))
+    # test for every test we found in the Makefile
     for target_alias, deps in deps_map.items():
-        for test, test_src in sorted(test_source_map.items()):
-            if len(test) == 0:
-                print(ColorString.warning("Failed to get test name for %s" % test_src))
+        for test in sorted(tests):
+            match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+            if len(match_src) == 0:
+                print(ColorString.warning("Cannot find .cc file for %s" % test))
+                continue
+            elif len(match_src) > 1:
+                print(ColorString.warning("Found more than one .cc for %s" % test))
+                print(match_src)
                 continue
 
+            assert(len(match_src) == 1)
+            is_parallel = tests[test]
             test_target_name = \
                 test if not target_alias else test + "_" + target_alias
             TARGETS.register_test(
                 test_target_name,
-                test_src,
-                test not in non_parallel_tests,
-                json.dumps(deps['extra_deps']),
-                json.dumps(deps['extra_compiler_flags']))
+                match_src[0],
+                is_parallel,
+                deps['extra_deps'],
+                deps['extra_compiler_flags'])
 
             if test in _EXPORTED_TEST_LIBS:
                 test_library = "%s_lib" % test_target_name
-                TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
+                TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
     TARGETS.flush_tests()
 
     print(ColorString.info("Generated TARGETS Summary:"))
@@ -241,7 +220,6 @@ def get_rocksdb_path():
 
     return rocksdb_path
 
-
 def exit_with_error(msg):
     print(ColorString.error(msg))
     sys.exit(1)

buckifier/check_buck_targets.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# If clang_format_diff.py command is not specfied, we assume we are able to
-# access directly without any path.
-
-TGT_DIFF=`git diff TARGETS | head -n 1`
-
-if [ ! -z "$TGT_DIFF" ]
-then
-  echo "TARGETS file has uncommitted changes. Skip this check."
-  exit 0
-fi
-
-echo Backup original TARGETS file.
-
-cp TARGETS TARGETS.bkp
-
-${PYTHON:-python3} buckifier/buckify_rocksdb.py
-
-TGT_DIFF=`git diff TARGETS | head -n 1`
-
-if [ -z "$TGT_DIFF" ]
-then
-  mv TARGETS.bkp TARGETS
-  exit 0
-else
-  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update TARGETS file."
-  echo "Do not manually update TARGETS file."
-  ${PYTHON:-python3} --version
-  mv TARGETS.bkp TARGETS
-  exit 1
-fi

buckifier/targets_builder.py

@@ -25,12 +25,10 @@ def pretty_list(lst, indent=8):
 
 
 class TARGETSBuilder(object):
-    def __init__(self, path, extra_argv):
+    def __init__(self, path):
         self.path = path
-        self.targets_file = open(path, 'wb')
-        header = targets_cfg.rocksdb_target_header_template.format(
-            extra_argv=extra_argv)
-        self.targets_file.write(header.encode("utf-8"))
+        self.targets_file = open(path, 'w')
+        self.targets_file.write(targets_cfg.rocksdb_target_header)
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
@@ -39,68 +37,26 @@ class TARGETSBuilder(object):
     def __del__(self):
         self.targets_file.close()
 
-    def add_library(self, name, srcs, deps=None, headers=None,
-                    extra_external_deps="", link_whole=False):
+    def add_library(self, name, srcs, deps=None, headers=None):
         headers_attr_prefix = ""
         if headers is None:
             headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
         self.targets_file.write(targets_cfg.library_template.format(
             name=name,
             srcs=pretty_list(srcs),
             headers_attr_prefix=headers_attr_prefix,
             headers=headers,
-            deps=pretty_list(deps),
-            extra_external_deps=extra_external_deps,
-            link_whole=link_whole).encode("utf-8"))
-        self.total_lib = self.total_lib + 1
-
-    def add_rocksdb_library(self, name, srcs, headers=None):
-        headers_attr_prefix = ""
-        if headers is None:
-            headers_attr_prefix = "auto_"
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
-        self.targets_file.write(targets_cfg.rocksdb_library_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
-            headers=headers).encode("utf-8"))
+            deps=pretty_list(deps)))
         self.total_lib = self.total_lib + 1
 
     def add_binary(self, name, srcs, deps=None):
-        self.targets_file.write(targets_cfg.binary_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            deps=pretty_list(deps)).encode("utf-8"))
+        self.targets_file.write(targets_cfg.binary_template % (
+            name,
+            pretty_list(srcs),
+            pretty_list(deps)))
         self.total_bin = self.total_bin + 1
 
-    def add_c_test(self):
-        self.targets_file.write(b"""
-cpp_binary(
-    name = "c_test_bin",
-    srcs = ["db/c_test.c"],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [":rocksdb_test_lib"],
-) if not is_opt_mode else None
-
-custom_unittest(
-    name = "c_test",
-    command = [
-        native.package_name() + "/buckifier/rocks_test_runner.sh",
-        "$(location :{})".format("c_test_bin"),
-    ],
-    type = "simple",
-) if not is_opt_mode else None
-""")
-
     def register_test(self,
                       test_name,
                       src,
@@ -120,5 +76,5 @@ custom_unittest(
         self.total_test = self.total_test + 1
 
     def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
+        self.targets_file.write(targets_cfg.unittests_template % self.tests_cfg)
         self.tests_cfg = ""

buckifier/targets_cfg.py

@@ -4,9 +4,7 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-rocksdb_target_header_template = \
-    """# This file \100generated by:
-#$ python3 buckifier/buckify_rocksdb.py{extra_argv}
+rocksdb_target_header = """# This file \100generated by `python buckifier/buckify_rocksdb.py`
 # --> DO NOT EDIT MANUALLY <--
 # This file is a Facebook-specific integration for buck builds, so can
 # only be validated by Facebook employees.
@@ -17,7 +15,7 @@ load(":defs.bzl", "test_binary")
 
 REPO_PATH = package_name() + "/"
 
-ROCKSDB_COMPILER_FLAGS_0 = [
+ROCKSDB_COMPILER_FLAGS = [
     "-fno-builtin-memcmp",
     # Needed to compile in fbcode
     "-Wno-expansion-to-defined",
@@ -32,21 +30,19 @@ ROCKSDB_EXTERNAL_DEPS = [
     ("zlib", None, "z"),
     ("gflags", None, "gflags"),
     ("lz4", None, "lz4"),
-    ("zstd", None, "zstd"),
+    ("zstd", None),
+    ("tbb", None),
+    ("googletest", None, "gtest"),
 ]
 
-ROCKSDB_OS_DEPS_0 = [
+ROCKSDB_OS_DEPS = [
     (
         "linux",
-        ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
-    ),
-    (
-        "macos",
-        ["third-party//tbb:tbb"],
+        ["third-party//numa:numa", "third-party//liburing:uring"],
     ),
 ]
 
-ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
+ROCKSDB_OS_PREPROCESSOR_FLAGS = [
     (
         "linux",
         [
@@ -60,27 +56,17 @@ ROCKSDB_OS_PREPROCESSOR_FLAGS_0 = [
             "-DHAVE_SSE42",
             "-DLIBURING",
             "-DNUMA",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
         ],
     ),
     (
         "macos",
-        [
-            "-DOS_MACOSX",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "windows",
-        [ "-DOS_WIN", "-DWIN32", "-D_MBCS", "-DWIN64", "-DNOMINMAX" ]
+        ["-DOS_MACOSX"],
     ),
 ]
 
 ROCKSDB_PREPROCESSOR_FLAGS = [
+    "-DROCKSDB_PLATFORM_POSIX",
+    "-DROCKSDB_LIB_IO_POSIX",
     "-DROCKSDB_SUPPORT_THREAD_LOCAL",
 
     # Flags to enable libs we include
@@ -91,22 +77,21 @@ ROCKSDB_PREPROCESSOR_FLAGS = [
     "-DZSTD",
     "-DZSTD_STATIC_LINKING_ONLY",
     "-DGFLAGS=gflags",
+    "-DTBB",
 
     # Added missing flags from output of build_detect_platform
     "-DROCKSDB_BACKTRACE",
+
+    # Directories with files for #include
+    "-I" + REPO_PATH + "include/",
+    "-I" + REPO_PATH,
 ]
 
-# Directories with files for #include
-ROCKSDB_INCLUDE_PATHS = [
-    "",
-    "include",
-]
-
-ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
+ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {
     "x86_64": [
         "-DHAVE_PCLMUL",
     ],
-}}
+}
 
 build_mode = read_config("fbcode", "build_mode")
 
@@ -114,26 +99,21 @@ is_opt_mode = build_mode.startswith("opt")
 
 # -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
 # doesn't harm and avoid forgetting to add it.
-ROCKSDB_COMPILER_FLAGS = ROCKSDB_COMPILER_FLAGS_0 + (["-DNDEBUG"] if is_opt_mode else [])
+ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
 
 sanitizer = read_config("fbcode", "sanitizer")
 
 # Do not enable jemalloc if sanitizer presents. RocksDB will further detect
 # whether the binary is linked with jemalloc at runtime.
-ROCKSDB_OS_PREPROCESSOR_FLAGS = ROCKSDB_OS_PREPROCESSOR_FLAGS_0 + ([(
+ROCKSDB_OS_PREPROCESSOR_FLAGS += ([(
     "linux",
     ["-DROCKSDB_JEMALLOC"],
 )] if sanitizer == "" else [])
 
-ROCKSDB_OS_DEPS = ROCKSDB_OS_DEPS_0 + ([(
+ROCKSDB_OS_DEPS += ([(
     "linux",
     ["third-party//jemalloc:headers"],
 )] if sanitizer == "" else [])
-
-ROCKSDB_LIB_DEPS = [
-    ":rocksdb_lib",
-    ":rocksdb_test_lib",
-] if not is_opt_mode else [":rocksdb_lib"]
 """
 
 
@@ -147,38 +127,19 @@ cpp_library(
     os_deps = ROCKSDB_OS_DEPS,
     os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
     preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
     deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
-    link_whole = {link_whole},
-)
-"""
-
-rocksdb_library_template = """
-cpp_library(
-    name = "{name}",
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = ROCKSDB_LIB_DEPS,
     external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 """
 
 binary_template = """
 cpp_binary(
-    name = "{name}",
-    srcs = [{srcs}],
+    name = "%s",
+    srcs = [%s],
     arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
     compiler_flags = ROCKSDB_COMPILER_FLAGS,
     preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [{deps}],
+    deps = [%s],
     external_deps = ROCKSDB_EXTERNAL_DEPS,
 )
 """
@@ -195,24 +156,24 @@ test_cfg_template = """    [
 unittests_template = """
 # [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
 ROCKS_TESTS = [
-{tests}]
+%s]
 
 # Generate a test rule for each entry in ROCKS_TESTS
 # Do not build the tests in opt mode, since SyncPoint and other test code
 # will not be included.
 [
-    cpp_unittest(
-        name = test_name,
-        srcs = [test_cc],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS + extra_compiler_flags,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        include_paths = ROCKSDB_INCLUDE_PATHS,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        external_deps = ROCKSDB_EXTERNAL_DEPS + [
-            ("googletest", None, "gtest"),
-        ],
+    test_binary(
+        extra_compiler_flags = extra_compiler_flags,
+        extra_deps = extra_deps,
+        parallelism = parallelism,
+        rocksdb_arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
+        rocksdb_compiler_flags = ROCKSDB_COMPILER_FLAGS,
+        rocksdb_external_deps = ROCKSDB_EXTERNAL_DEPS,
+        rocksdb_os_deps = ROCKSDB_OS_DEPS,
+        rocksdb_os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
+        rocksdb_preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
+        test_cc = test_cc,
+        test_name = test_name,
     )
     for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
     if not is_opt_mode

build_tools/build_detect_platform

@@ -28,7 +28,6 @@
 #       -DZSTD                      if the ZSTD library is present
 #       -DNUMA                      if the NUMA library is present
 #       -DTBB                       if the TBB library is present
-#       -DMEMKIND                   if the memkind library is present
 #
 # Using gflags in rocksdb:
 # Our project depends on gflags, which requires users to take some extra steps
@@ -45,13 +44,8 @@ if test -z "$OUTPUT"; then
   exit 1
 fi
 
-# we depend on C++11, but should be compatible with newer standards
-if [ "$ROCKSDB_CXX_STANDARD" ]; then
-  PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
-else
-  PLATFORM_CXXFLAGS="-std=c++11"
-fi
-
+# we depend on C++11
+PLATFORM_CXXFLAGS="-std=c++11"
 # we currently depend on POSIX platform
 COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
 
@@ -65,12 +59,8 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
       source "$PWD/build_tools/fbcode_config4.8.1.sh"
     elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
       source "$PWD/build_tools/fbcode_config.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then
-      source "$PWD/build_tools/fbcode_config_platform009.sh"
     else
-      source "$PWD/build_tools/fbcode_config_platform009.sh"
+      source "$PWD/build_tools/fbcode_config_platform007.sh"
     fi
 fi
 
@@ -98,16 +88,6 @@ if test -z "$CXX"; then
     fi
 fi
 
-if test -z "$AR"; then
-    if [ -x "$(command -v gcc-ar)" ]; then
-        AR=gcc-ar
-    elif [ -x "$(command -v llvm-ar)" ]; then
-        AR=llvm-ar
-    else
-        AR=ar
-    fi
-fi
-
 # Detect OS
 if test -z "$TARGET_OS"; then
     TARGET_OS=`uname -s`
@@ -170,13 +150,10 @@ case "$TARGET_OS" in
         else
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
-        if test -z "$ROCKSDB_USE_IO_URING"; then
-            ROCKSDB_USE_IO_URING=1
-        fi
-        if test "$ROCKSDB_USE_IO_URING" -ne 0; then
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
+        if test $ROCKSDB_USE_IO_URING; then
             # check for liburing
-            $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
+            $CXX $CFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
               #include <liburing.h>
               int main() {
                 struct io_uring ring;
@@ -214,17 +191,6 @@ EOF
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         # PORT_FILES=port/freebsd/freebsd_specific.cc
         ;;
-    GNU/kFreeBSD)
-        PLATFORM=OS_GNU_KFREEBSD
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_GNU_KFREEBSD"
-        if [ -z "$USE_CLANG" ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
-        fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
-        # PORT_FILES=port/gnu_kfreebsd/gnu_kfreebsd_specific.cc
-        ;;
     NetBSD)
         PLATFORM=OS_NETBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
@@ -279,15 +245,11 @@ JAVAC_ARGS="-source 7"
 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Cross-compiling; do not try any compilation tests.
     # Also don't need any compilation tests if compiling on fbcode
-    if [ "$FBCODE_BUILD" = "true" ]; then
-      # Enable backtrace on fbcode since the necessary libraries are present
-      COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-    fi
     true
 else
     if ! test $ROCKSDB_DISABLE_FALLOCATE; then
         # Test whether fallocate is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <fcntl.h>
           #include <linux/falloc.h>
           int main() {
@@ -303,7 +265,7 @@ EOF
     if ! test $ROCKSDB_DISABLE_SNAPPY; then
         # Test whether Snappy library is installed
         # http://code.google.com/p/snappy/
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <snappy.h>
           int main() {}
 EOF
@@ -318,38 +280,30 @@ EOF
         # Test whether gflags library is installed
         # http://gflags.github.io/gflags/
         # check if the namespace is gflags
-        if $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
           #include <gflags/gflags.h>
-          using namespace GFLAGS_NAMESPACE;
           int main() {}
 EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is gflags
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace gflags;
-            int main() {}
-EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is google
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+        else
+          # check if namespace is google
+          $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
             #include <gflags/gflags.h>
             using namespace google;
             int main() {}
 EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+          if [ "$?" = 0 ]; then
+              COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
+              PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+          fi
         fi
     fi
 
     if ! test $ROCKSDB_DISABLE_ZLIB; then
         # Test whether zlib library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <zlib.h>
           int main() {}
 EOF
@@ -362,7 +316,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_BZIP; then
         # Test whether bzip library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <bzlib.h>
           int main() {}
 EOF
@@ -375,7 +329,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_LZ4; then
         # Test whether lz4 library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <lz4.h>
           #include <lz4hc.h>
           int main() {}
@@ -389,7 +343,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_ZSTD; then
         # Test whether zstd library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <zstd.h>
           int main() {}
 EOF
@@ -402,7 +356,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_NUMA; then
         # Test whether numa is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
           #include <numa.h>
           #include <numaif.h>
           int main() {}
@@ -416,7 +370,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_TBB; then
         # Test whether tbb is available
-        $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
+        $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
           #include <tbb/tbb.h>
           int main() {}
 EOF
@@ -429,7 +383,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_JEMALLOC; then
         # Test whether jemalloc is available
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -ljemalloc \
+        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
           2>/dev/null; then
             # This will enable some preprocessor identifiers in the Makefile
             JEMALLOC=1
@@ -450,7 +404,7 @@ EOF
     fi
     if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
         # jemalloc is not available. Let's try tcmalloc
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null \
+        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
           -ltcmalloc 2>/dev/null; then
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
             JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
@@ -459,7 +413,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then
         # Test whether malloc_usable_size is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <malloc.h>
           int main() {
             size_t res = malloc_usable_size(0);
@@ -472,25 +426,9 @@ EOF
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_MEMKIND; then
-        # Test whether memkind library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -lmemkind -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <memkind.h>
-          int main() {
-            memkind_malloc(MEMKIND_DAX_KMEM, 1024);
-            return 0;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DMEMKIND"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lmemkind"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lmemkind"
-        fi
-    fi
-
     if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
         # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <pthread.h>
           int main() {
             int x = PTHREAD_MUTEX_ADAPTIVE_NP;
@@ -505,7 +443,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_BACKTRACE; then
         # Test whether backtrace is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <execinfo.h>
           int main() {
             void* frames[1];
@@ -517,7 +455,7 @@ EOF
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
         else
             # Test whether execinfo library is installed
-            $CXX $PLATFORM_CXXFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
+            $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
               #include <execinfo.h>
               int main() {
                 void* frames[1];
@@ -534,7 +472,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_PG; then
         # Test if -pg is supported
-        $CXX $PLATFORM_CXXFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
           int main() {
             return 0;
           }
@@ -546,7 +484,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_SYNC_FILE_RANGE; then
         # Test whether sync_file_range is supported for compatibility with an old glibc
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <fcntl.h>
           int main() {
             int fd = open("/dev/null", 0);
@@ -560,7 +498,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then
         # Test whether sched_getcpu is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <sched.h>
           int main() {
             int cpuid = sched_getcpu();
@@ -574,7 +512,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_AUXV_GETAUXVAL; then
         # Test whether getauxval is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <sys/auxv.h>
           int main() {
             uint64_t auxv = getauxval(AT_HWCAP);
@@ -596,23 +534,13 @@ EOF
             PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
         fi
     fi
-    if ! test $ROCKSDB_DISABLE_BENCHMARK; then
-        # Test whether google benchmark is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lbenchmark 2>/dev/null  <<EOF
-          #include <benchmark/benchmark.h>
-          int main() {}
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark"
-        fi
-    fi
 fi
 
 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
 # -Wshorten-64-to-32 breaks compilation on FreeBSD i386
 if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
   # Test whether -Wshorten-64-to-32 is available
-  $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
+  $CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
     int main() {}
 EOF
   if [ "$?" = 0 ]; then
@@ -663,35 +591,6 @@ else
   if test "$USE_SSE"; then
     TRY_SSE_ETC="1"
   fi
-
-  if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
-    # For portability compile for macOS 10.12 (2016) or newer
-    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_SHARED_LDFLAGS="$PLATFORM_SHARED_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.12"
-    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.12"
-    JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-  fi
-fi
-
-if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
-  # check for GNU libc on ppc64
-  $CXX -x c++ - -o /dev/null 2>/dev/null <<EOF
-    #include <stdio.h>
-    #include <stdlib.h>
-    #include <gnu/libc-version.h>
-
-    int main(int argc, char *argv[]) {
-      printf("GNU libc version: %s\n", gnu_get_libc_version());
-      return 0;
-    }
-EOF
-  if [ "$?" != 0 ]; then
-      PPC_LIBC_IS_GNU=0
-  fi
 fi
 
 if test "$TRY_SSE_ETC"; then
@@ -701,18 +600,11 @@ if test "$TRY_SSE_ETC"; then
   # It doesn't even really check that your current CPU is compatible.
   #
   # SSE4.2 available since nehalem, ca. 2008-2010
-  # Includes POPCNT for BitsSetToOne, BitParity
   TRY_SSE42="-msse4.2"
   # PCLMUL available since westmere, ca. 2010-2011
   TRY_PCLMUL="-mpclmul"
   # AVX2 available since haswell, ca. 2013-2015
   TRY_AVX2="-mavx2"
-  # BMI available since haswell, ca. 2013-2015
-  # Primarily for TZCNT for CountTrailingZeroBits
-  TRY_BMI="-mbmi"
-  # LZCNT available since haswell, ca. 2013-2015
-  # For FloorLog2
-  TRY_LZCNT="-mlzcnt"
 fi
 
 $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_SSE42 -x c++ - -o /dev/null 2>/dev/null <<EOF
@@ -761,34 +653,6 @@ elif test "$USE_SSE"; then
   echo "warning: USE_SSE specified but compiler could not use AVX2 intrinsics, disabling" >&2
 fi
 
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_BMI -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main(int argc, char *argv[]) {
-    (void)argv;
-    return (int)_tzcnt_u64((uint64_t)argc);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_BMI -DHAVE_BMI"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use BMI intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_LZCNT -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main(int argc, char *argv[]) {
-    (void)argv;
-    return (int)_lzcnt_u64((uint64_t)argc);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_LZCNT -DHAVE_LZCNT"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use LZCNT intrinsics, disabling" >&2
-fi
-
 $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
   #include <cstdint>
   int main() {
@@ -820,7 +684,6 @@ EOF
   fi
 fi
 
-
 if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then
   $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null <<EOF
   void dummy_func() {}
@@ -845,15 +708,10 @@ ROCKSDB_PATCH=`build_tools/version.sh patch`
 
 echo "CC=$CC" >> "$OUTPUT"
 echo "CXX=$CXX" >> "$OUTPUT"
-echo "AR=$AR" >> "$OUTPUT"
 echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
-echo "PLATFORM_CMAKE_FLAGS=$PLATFORM_CMAKE_FLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CCFLAGS=$JAVA_STATIC_DEPS_CCFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CXXFLAGS=$JAVA_STATIC_DEPS_CXXFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_LDFLAGS=$JAVA_STATIC_DEPS_LDFLAGS" >> "$OUTPUT"
 echo "JAVAC_ARGS=$JAVAC_ARGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
@@ -887,6 +745,3 @@ echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
 if test -n "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
   echo "USE_FOLLY_DISTRIBUTED_MUTEX=$USE_FOLLY_DISTRIBUTED_MUTEX" >> "$OUTPUT"
 fi
-if test -n "$PPC_LIBC_IS_GNU"; then
-  echo "PPC_LIBC_IS_GNU=$PPC_LIBC_IS_GNU" >> "$OUTPUT"
-fi

build_tools/dependencies_platform009.sh

@@ -1,20 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/1795efe5f06778c15a92c8f9a2aba5dc496d9d4d/9.x/centos7-native/3bed279
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/7318eaac22659b6ff2fe43918e4b69fd0772a8a7/9.0.0/platform009/651ee30
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/4959b39cfbe5965a37c861c4c327fa7c5c759b87/9.x/platform009/9202ce7
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/45ce3375cdc77ecb2520bbf8f0ecddd3f98efd7a/2.30/platform009/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/be4de3205e029101b18aa8103daa696c2bef3b19/1.1.3/platform009/7f3b187
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/3c160ac5c67e257501e24c6c1d00ad5e01d73db6/1.2.8/platform009/7f3b187
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/73a237ac5bc0a5f5d67b39b8d253cfebaab88684/1.0.6/platform009/7f3b187
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/ec6573523b0ce55ef6373a4801189027cf07bb2c/1.9.1/platform009/7f3b187
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/64c58a207d2495e83abc57a500a956df09b79a7c/1.4.x/platform009/ba86d1f
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/824d0a8a5abb5b121afd1b35fc3896407ea50092/2.2.0/platform009/7f3b187
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9aef9feb850b168a68736420f217b01cce11a89/master/platform009/c305944
-NUMA_BASE=/mnt/gvfs/third-party2/numa/0af65f71e23a67bf65dc91b11f95caa39325c432/2.0.11/platform009/7f3b187
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/02486dac347645d31dce116f44e1de3177315be2/1.4/platform009/5191652
-TBB_BASE=/mnt/gvfs/third-party2/tbb/2e0ec671e550bfca347300bf3f789d9c0fff24ad/2018_U5/platform009/7f3b187
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/70dbd9cfee63a25611417d09433a86d7711b3990/20200729/platform009/7f3b187
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/32b8a2407b634df3f8f948ba373fc4acc6a18296/fb/platform009/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/08634589372fa5f237bfd374e8c644a8364e78c1/2.32/platform009/ba86d1f/
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/6ae525939ad02e5e676855082fbbc7828dbafeac/3.15.0/platform009/7f3b187
-LUA_BASE=/mnt/gvfs/third-party2/lua/162efd9561a3d21f6869f4814011e9cf1b3ff4dc/5.3.4/platform009/a6271c4

build_tools/fbcode_config.sh

@@ -109,7 +109,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS+=" -B$BINUTILS/gold"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -120,7 +119,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 

build_tools/fbcode_config4.8.1.sh

@@ -69,7 +69,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  CXX="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -82,7 +81,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
 

build_tools/fbcode_config_platform007.sh

@@ -118,7 +118,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS+=" -B$BINUTILS/gold"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
@@ -129,7 +128,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 

build_tools/fbcode_config_platform009.sh

@@ -1,173 +0,0 @@
-#!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform009.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/9.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-if test -z $PIC_BUILD; then
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
-else
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
-fi
-CFLAGS+=" -DLIBURING"
-
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
-
-  CFLAGS+=" -B$BINUTILS"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform009/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform009/lib"
-EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-USE_FOLLY_DISTRIBUTED_MUTEX=1
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/format-diff.sh

@@ -2,99 +2,41 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
+if [ -z $CLANG_FORMAT_DIFF ]
+then
+CLANG_FORMAT_DIFF="clang-format-diff.py"
+fi
 
-print_usage () {
-  echo "Usage:"
-  echo "format-diff.sh [OPTIONS]"
-  echo "-c: check only."
-  echo "-h: print this message."
-}
+# Check clang-format-diff.py
+if ! which $CLANG_FORMAT_DIFF &> /dev/null
+then
+  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
+  echo "You can download clang-format-diff.py by running: "
+  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
+  echo "You can download clang-format by running:"
+  echo "    brew install clang-format"
+  echo "  Or"
+  echo "    apt install clang-format"
+  echo "  This might work too:"
+  echo "    yum install git-clang-format"
+  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  echo "and make sure ${CLANG_FORMAT_DIFF} is executable."
+  exit 128
+fi
 
-while getopts ':ch' OPTION; do
-  case "$OPTION" in
-    c)
-      CHECK_ONLY=1
-      ;;
-    h)
-      print_usage
-      exit 1
-      ;;
-    ?)
-      print_usage
-      exit 1
-      ;;
-  esac
-done
+# Check argparse, a library that clang-format-diff.py requires.
+python 2>/dev/null << EOF
+import argparse
+EOF
 
-REPO_ROOT="$(git rev-parse --show-toplevel)"
-
-if [ "$CLANG_FORMAT_DIFF" ]; then
-  echo "Note: CLANG_FORMAT_DIFF='$CLANG_FORMAT_DIFF'"
-  # Dry run to confirm dependencies like argparse
-  if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-    true #Good
-  else
-    exit 128
-  fi
-else
-  # First try directly executing the possibilities
-  if clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=clang-format-diff.py
-  elif $REPO_ROOT/clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=$REPO_ROOT/clang-format-diff.py
-  else
-    # This probably means we need to directly invoke the interpreter.
-    # But first find clang-format-diff.py
-    if [ -f "$REPO_ROOT/clang-format-diff.py" ]; then
-      CFD_PATH="$REPO_ROOT/clang-format-diff.py"
-    elif which clang-format-diff.py &> /dev/null; then
-      CFD_PATH="$(which clang-format-diff.py)"
-    else
-      echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-      echo "You can download clang-format-diff.py by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      echo "You can download clang-format by running:"
-      echo "    brew install clang-format"
-      echo "  Or"
-      echo "    apt install clang-format"
-      echo "  This might work too:"
-      echo "    yum install git-clang-format"
-      echo "Then make sure clang-format is available and executable from \$PATH:"
-      echo "    clang-format --version"
-      exit 128
-    fi
-    # Check argparse pre-req on interpreter, or it will fail
-    if echo import argparse | ${PYTHON:-python3}; then
-      true # Good
-    else
-      echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
-      echo "installed. You can try either of the follow ways to install it:"
-      echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
-      echo "  2. easy_install argparse (if you have easy_install)"
-      echo "  3. pip install argparse (if you have pip)"
-      exit 129
-    fi
-    # Unfortunately, some machines have a Python2 clang-format-diff.py
-    # installed but only a Python3 interpreter installed. Unfortunately,
-    # automatic 2to3 migration is insufficient, so suggest downloading latest.
-    if grep -q "print '" "$CFD_PATH" && \
-       ${PYTHON:-python3} --version | grep -q 'ython 3'; then
-      echo "You have clang-format-diff.py for Python 2 but are using a Python 3"
-      echo "interpreter (${PYTHON:-python3})."
-      echo "You can download clang-format-diff.py for Python 3 by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      exit 130
-    fi
-    CLANG_FORMAT_DIFF="${PYTHON:-python3} $CFD_PATH"
-    # This had better work after all those checks
-    if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-      true #Good
-    else
-      exit 128
-    fi
-  fi
+if [ "$?" != 0 ]
+then
+  echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
+  echo "installed. You can try either of the follow ways to install it:"
+  echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
+  echo "  2. easy_install argparse (if you have easy_install)"
+  echo "  3. pip install argparse (if you have pip)"
+  exit 129
 fi
 
 # TODO(kailiu) following work is not complete since we still need to figure
@@ -136,25 +78,15 @@ then
   FORMAT_UPSTREAM_MERGE_BASE="$(git merge-base "$FORMAT_UPSTREAM" HEAD)"
   # Get the differences
   diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -p 1)
-  echo "Checking format of changes not yet in $FORMAT_UPSTREAM..."
 else
   # Check the format of uncommitted lines,
   diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
-  echo "Checking format of uncommitted changes..."
 fi
 
 if [ -z "$diffs" ]
 then
   echo "Nothing needs to be reformatted!"
   exit 0
-elif [ $CHECK_ONLY ]
-then
-  echo "Your change has unformatted code. Please run make format!"
-  if [ $VERBOSE_CHECK ]; then
-    clang-format --version
-    echo "$diffs"
-  fi
-  exit 1
 fi
 
 # Highlight the insertion/deletion from the clang-format-diff.py's output
@@ -189,7 +121,7 @@ if [ -z "$uncommitted_code" ]
 then
   git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -i -p 1
 else
-  git diff -U0 HEAD | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
 fi
 echo "Files reformatted!"
 

build_tools/make_package.sh

@@ -103,26 +103,31 @@ function main() {
   gem_install fpm
 
   make static_lib
-  LIBDIR=/usr/lib
-  if [[ $FPM_OUTPUT = "rpm" ]]; then
-      LIBDIR=$(rpm --eval '%_libdir')
-  fi
+  make install INSTALL_PATH=package
 
-  rm -rf package
-  make install DESTDIR=package PREFIX=/usr LIBDIR=$LIBDIR
+  cd package
+
+  LIB_DIR=lib
+  if [[ -z "$ARCH" ]]; then
+      ARCH=$(getconf LONG_BIT)
+  fi
+  if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
+      mv lib lib64
+      LIB_DIR=lib64
+  fi
 
   fpm \
     -s dir \
     -t $FPM_OUTPUT \
-    -C package \
     -n rocksdb \
     -v $1 \
+    --prefix /usr \
     --url http://rocksdb.org/ \
     -m rocksdb@fb.com \
     --license BSD \
     --vendor Facebook \
     --description "RocksDB is an embeddable persistent key-value store for fast storage." \
-    usr
+    include $LIB_DIR
 }
 
 # shellcheck disable=SC2068

build_tools/rocksdb-lego-determinator

@@ -3,7 +3,7 @@
 # to determine next steps to run
 
 # Usage:
-# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> WORKINGDIR=<working_dir> rocksdb-lego-determinator <test-name>
+# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
 #
 # Input         Value
 # -------------------------------------------------------------------------
@@ -11,7 +11,7 @@
 # ONCALL        Email address to raise a task on failure
 # TRIGGER       Trigger conditions for email. Valid values are fail, warn, all
 # SUBSCRIBER    Email addresss to add as subscriber for task
-# WORKINGDIR    Working directory
+#
 
 #
 # Report configuration
@@ -53,19 +53,13 @@ if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
   ]"
 fi
 
-# Working directory for the following command, default to current directory
-WORKING_DIR=.
-if [ ! -z $WORKINGDIR ]; then
-  WORKING_DIR=$WORKINGDIR
-fi
-
 #
 # Helper variables
 #
 CLEANUP_ENV="
 {
     'name':'Cleanup environment',
-    'shell':'cd $WORKING_DIR; rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean',
+    'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true)  && make clean',
     'user':'root'
 }"
 
@@ -106,7 +100,9 @@ NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
 GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
 ASAN="COMPILE_WITH_ASAN=1"
 CLANG="USE_CLANG=1"
-TSAN="COMPILE_WITH_TSAN=1"
+# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090.
+# using platform007 gives us gcc-8 or higher which has that bug fixed.
+TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1"
 UBSAN="COMPILE_WITH_UBSAN=1"
 TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"'
 NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd"
@@ -116,7 +112,6 @@ SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; ex
 PARSER="'parser':'python build_tools/error_filter.py $1'"
 
 CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
-SKIP_FORMAT_CHECKS="SKIP_FORMAT_BUCK_CHECKS=1"
 
 # This code is getting called under various scenarios. What we care about is to
 # understand when it's called from nightly contruns because in that case we'll
@@ -158,7 +153,7 @@ UNIT_TEST_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build and test RocksDB debug version',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -181,7 +176,7 @@ UNIT_TEST_NON_SHM_COMMANDS="[
             {
                 'name':'Build and test RocksDB debug version',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $NON_SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
+                'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -202,7 +197,7 @@ RELEASE_BUILD_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build RocksDB release',
-                'shell':'cd $WORKING_DIR; make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
+                'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -223,7 +218,7 @@ UNIT_TEST_COMMANDS_481="[
             $CLEANUP_ENV,
             {
                 'name':'Build and test RocksDB debug version',
-                'shell':'cd $WORKING_DIR; $SHM $GCC_481 $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
+                'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -244,7 +239,7 @@ RELEASE_BUILD_COMMANDS_481="[
             $CLEANUP_ENV,
             {
                 'name':'Build RocksDB release on GCC 4.8.1',
-                'shell':'cd $WORKING_DIR; $GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
+                'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -265,7 +260,7 @@ CLANG_UNIT_TEST_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build and test RocksDB debug',
-                'shell':'cd $WORKING_DIR; $CLANG $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
+                'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -286,7 +281,7 @@ CLANG_RELEASE_BUILD_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build RocksDB release',
-                'shell':'cd $WORKING_DIR; $CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
+                'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -307,7 +302,7 @@ CLANG_ANALYZE_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'RocksDB build and analyze',
-                'shell':'cd $WORKING_DIR; $CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
+                'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -328,7 +323,7 @@ CODE_COV_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build, test and collect code coverage info',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -349,7 +344,7 @@ UNITY_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build, test unity test',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -370,7 +365,7 @@ LITE_BUILD_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build RocksDB debug version',
-                'shell':'cd $WORKING_DIR; $SKIP_FORMAT_CHECKS make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
+                'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -390,7 +385,7 @@ REPORT_LITE_BINARY_SIZE_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Report RocksDB Lite binary size to scuba',
-                'shell':'cd $WORKING_DIR; tools/report_lite_binary_size.sh',
+                'shell':'tools/report_lite_binary_size.sh',
                 'user':'root',
             },
         ],
@@ -409,74 +404,14 @@ STRESS_CRASH_TEST_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build and run RocksDB debug stress tests',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
             {
                 'name':'Build and run RocksDB debug crash tests',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB blackbox stress/crash test
-#
-BLACKBOX_STRESS_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Blackbox Stress and Crash Test',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug stress tests',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            {
-                'name':'Build and run RocksDB debug blackbox crash tests',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 blackbox_crash_test || $CONTRUN_NAME=blackbox_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB whitebox stress/crash test
-#
-WHITEBOX_STRESS_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Whitebox Stress and Crash Test',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug stress tests',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            {
-                'name':'Build and run RocksDB debug whitebox crash tests',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 whitebox_crash_test || $CONTRUN_NAME=whitebox_crash_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -499,14 +434,14 @@ STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build and run RocksDB debug stress tests',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
             {
                 'name':'Build and run RocksDB debug crash tests with atomic flush',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -529,44 +464,14 @@ STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build and run RocksDB debug stress tests',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
             {
                 'name':'Build and run RocksDB debug crash tests with txn',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB stress/crash test with timestamp
-#
-STRESS_CRASH_TEST_WITH_TS_COMMANDS="[
-    {
-        'name':'Rocksdb Stress and Crash Test with ts',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug stress tests',
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            {
-                'name':'Build and run RocksDB debug crash tests with ts',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_ts || $CONTRUN_NAME=crash_test_with_ts $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -588,7 +493,7 @@ WRITE_STRESS_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build and run RocksDB write stress tests',
-                'shell':'cd $WORKING_DIR; make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
+                'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             }
@@ -611,7 +516,7 @@ ASAN_TEST_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Test RocksDB debug under ASAN',
-'shell':'cd $WORKING_DIR; set -o pipefail && ($SHM $ASAN $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
+'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
                 'user':'root',
                 $PARSER
             }
@@ -634,55 +539,7 @@ ASAN_CRASH_TEST_COMMANDS="[
             {
                 'name':'Build and run RocksDB debug asan_crash_test',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB blackbox crash testing under address sanitizer
-#
-ASAN_BLACKBOX_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb blackbox crash test under ASAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug blackbox asan_crash_test',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 blackbox_asan_crash_test || $CONTRUN_NAME=blackbox_asan_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB whitebox crash testing under address sanitizer
-#
-ASAN_WHITEBOX_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb whitebox crash test under ASAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug whitebox asan_crash_test',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 whitebox_asan_crash_test || $CONTRUN_NAME=whitebox_asan_crash_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -706,7 +563,7 @@ ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
             {
                 'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -730,7 +587,7 @@ ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[
             {
                 'name':'Build and run RocksDB debug asan_crash_test_with_txn',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $SKIP_FORMAT_CHECKS make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -752,7 +609,7 @@ UBSAN_TEST_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Test RocksDB debug under UBSAN',
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $UBSAN $CLANG $DEBUG $SKIP_FORMAT_CHECKS make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $UBSAN $CLANG $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             }
@@ -762,7 +619,7 @@ UBSAN_TEST_COMMANDS="[
 ]"
 
 #
-# RocksDB crash testing under undefined behavior sanitizer
+# RocksDB crash testing under udnefined behavior sanitizer
 #
 UBSAN_CRASH_TEST_COMMANDS="[
     {
@@ -775,55 +632,7 @@ UBSAN_CRASH_TEST_COMMANDS="[
             {
                 'name':'Build and run RocksDB debug ubsan_crash_test',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB crash testing under undefined behavior sanitizer
-#
-UBSAN_BLACKBOX_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb blackbox crash test under UBSAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug blackbox ubsan_crash_test',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 blackbox_ubsan_crash_test || $CONTRUN_NAME=blackbox_ubsan_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB crash testing under undefined behavior sanitizer
-#
-UBSAN_WHITEBOX_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb whitebox crash test under UBSAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Build and run RocksDB debug whitebox ubsan_crash_test',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 whitebox_ubsan_crash_test || $CONTRUN_NAME=whitebox_ubsan_crash_test $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -847,7 +656,7 @@ UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
             {
                 'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -871,7 +680,7 @@ UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
             {
                 'name':'Build and run RocksDB debug ubsan_crash_test_with_txn',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG $NON_TSAN_CRASH $CLANG $SKIP_FORMAT_CHECKS make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -895,7 +704,7 @@ VALGRIND_TEST_COMMANDS="[
             {
                 'name':'Run RocksDB debug unit tests',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; $SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
+                'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -918,7 +727,7 @@ TSAN_UNIT_TEST_COMMANDS="[
             {
                 'name':'Run RocksDB debug unit test',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $SKIP_FORMAT_CHECKS make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -941,55 +750,7 @@ TSAN_CRASH_TEST_COMMANDS="[
             {
                 'name':'Compile and run',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB blackbox crash test under TSAN
-#
-TSAN_BLACKBOX_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Blackbox Crash Test under TSAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Compile and run',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 blackbox_crash_test || $CONTRUN_NAME=tsan_blackbox_crash_test $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            $UPLOAD_DB_DIR,
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB whitebox crash test under TSAN
-#
-TSAN_WHITEBOX_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Whitebox Crash Test under TSAN',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            $CLEANUP_ENV,
-            {
-                'name':'Compile and run',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 whitebox_crash_test || $CONTRUN_NAME=tsan_whitebox_crash_test $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1013,7 +774,7 @@ TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[
             {
                 'name':'Compile and run',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1037,7 +798,7 @@ TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[
             {
                 'name':'Compile and run',
                 'timeout': 86400,
-                'shell':'cd $WORKING_DIR; set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL',
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1057,8 +818,6 @@ run_format_compatible()
   rm -rf /dev/shm/rocksdb
   mkdir /dev/shm/rocksdb
 
-  export https_proxy="fwdproxy:8080"
-  
   tools/check_format_compatible.sh
 }
 
@@ -1071,7 +830,7 @@ FORMAT_COMPATIBLE_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Run RocksDB debug unit test',
-                'shell':'cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
+                'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1093,7 +852,6 @@ run_no_compression()
   mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
   cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
   mv .tmp.Makefile Makefile
-  export $SKIP_FORMAT_CHECKS
   make $DEBUG J=1 check
 }
 
@@ -1106,7 +864,7 @@ NO_COMPRESSION_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Run RocksDB debug unit test',
-                'shell':'cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
+                'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1137,7 +895,6 @@ run_regression()
   strip librocksdb.a
   send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
 
-  make clean
   make -j$(nproc) shared_lib
   send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
   strip `readlink -f librocksdb.so`
@@ -1150,7 +907,6 @@ run_regression()
   strip librocksdb.a
   send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
 
-  make clean
   make LITE=1 -j$(nproc) shared_lib
   send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
   strip `readlink -f librocksdb.so`
@@ -1165,7 +921,7 @@ REGRESSION_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Make and run script',
-                'shell':'cd $WORKING_DIR; build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
+                'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1186,41 +942,7 @@ JAVA_BUILD_TEST_COMMANDS="[
             $CLEANUP_ENV,
             {
                 'name':'Build RocksDB for Java',
-                'shell':'cd $WORKING_DIR; $SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-        ],
-        $REPORT
-    }
-]"
-
-#
-# RocksDB fbcode stress/crash test
-#
-FBCODE_STRESS_CRASH_TEST_COMMANDS="[
-    {
-        'name':'Rocksdb Fbcode Stress and Crash Test',
-        'oncall':'$ONCALL',
-        'executeLocal': 'true',
-        'timeout': 86400,
-        'steps': [
-            {
-                'name':'Copy RocksDB code to fbcode repo',
-                'shell':'cd internal_repo_rocksdb/repo && git init && git add * && git commit -a -m \"Make internal_repo_rocksdb/repo a git repo\" && cd ../.. && echo Y | python3 rocks/release_script/release_to_fbcode.py -u internal_repo_rocksdb/repo master || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            {
-                'name':'Build RocksDB fbcode stress tests',
-                'shell':'cd $WORKING_DIR; buck build @mode/dbg rocks/tools:rocks_db_stress || $CONTRUN_NAME=db_stress_fbcode $TASK_CREATION_TOOL',
-                'user':'root',
-                $PARSER
-            },
-            {
-                'name':'Run RocksDB whitebox crash tests',
-                'timeout': 86400,
-                'shell':'cd $WORKING_DIR; mkdir /dev/shm/rocksdb_whitebox_crash_test && TEST_TMPDIR=\$(mktemp -d --tmpdir=/dev/shm/rocksdb_whitebox_crash_test) python3 rocksdb/src/tools/db_crashtest.py --stress_cmd=buck-out/dbg/gen/rocks/tools/rocks_db_stress -secondary_cache_uri=\"$SECONDARY_CACHE_URI\" --env_uri=$ENV_URI -checkpoint_one_in=0 -backup_one_in=0 -cache_size=134217728 -cache_numshardbits=4 -logtostderr=false whitebox || $CONTRUN_NAME=whitebox_crash_test $TASK_CREATION_TOOL',
+                'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
                 'user':'root',
                 $PARSER
             },
@@ -1270,21 +992,12 @@ case $1 in
   stress_crash)
     echo $STRESS_CRASH_TEST_COMMANDS
     ;;
-  blackbox_stress_crash)
-    echo $BLACKBOX_STRESS_CRASH_TEST_COMMANDS
-    ;;
-  whitebox_stress_crash)
-    echo $WHITEBOX_STRESS_CRASH_TEST_COMMANDS
-    ;;
   stress_crash_with_atomic_flush)
     echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
   stress_crash_with_txn)
     echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS
     ;;
-  stress_crash_with_ts)
-    echo $STRESS_CRASH_TEST_WITH_TS_COMMANDS
-    ;;
   write_stress)
     echo $WRITE_STRESS_COMMANDS
     ;;
@@ -1294,12 +1007,6 @@ case $1 in
   asan_crash)
     echo $ASAN_CRASH_TEST_COMMANDS
     ;;
-  blackbox_asan_crash)
-    echo $ASAN_BLACKBOX_CRASH_TEST_COMMANDS
-    ;;
-  whitebox_asan_crash)
-    echo $ASAN_WHITEBOX_CRASH_TEST_COMMANDS
-    ;;
   asan_crash_with_atomic_flush)
     echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
@@ -1312,12 +1019,6 @@ case $1 in
   ubsan_crash)
     echo $UBSAN_CRASH_TEST_COMMANDS
     ;;
-  blackbox_ubsan_crash)
-    echo $UBSAN_BLACKBOX_CRASH_TEST_COMMANDS
-    ;;
-  whitebox_ubsan_crash)
-    echo $UBSAN_WHITEBOX_CRASH_TEST_COMMANDS
-    ;;
   ubsan_crash_with_atomic_flush)
     echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
@@ -1333,12 +1034,6 @@ case $1 in
   tsan_crash)
     echo $TSAN_CRASH_TEST_COMMANDS
     ;;
-  blackbox_tsan_crash)
-    echo $TSAN_BLACKBOX_CRASH_TEST_COMMANDS
-    ;;
-  whitebox_tsan_crash)
-    echo $TSAN_WHITEBOX_CRASH_TEST_COMMANDS
-    ;;
   tsan_crash_with_atomic_flush)
     echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS
     ;;
@@ -1366,11 +1061,6 @@ case $1 in
   java_build)
     echo $JAVA_BUILD_TEST_COMMANDS
     ;;
-  fbcode_stress_crash)
-    set -f
-    echo $FBCODE_STRESS_CRASH_TEST_COMMANDS
-    set +f
-    ;;
   *)
     echo "Invalid determinator command"
     exit 1

build_tools/run_ci_db_test.ps1

@@ -68,7 +68,7 @@ $BinariesFolder = -Join($RootFolder, "\build\Debug\")
 
 if($WorkFolder -eq "") {
 
-    # If TEST_TMPDIR is set use it
+    # If TEST_TMPDIR is set use it    
     [string]$var = $Env:TEST_TMPDIR
     if($var -eq "") {
         $WorkFolder = -Join($RootFolder, "\db_tests\")
@@ -93,7 +93,7 @@ $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeCases -ne "") {
     Write-Host "ExcludeCases: $ExcludeCases"
     $l = $ExcludeCases -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeCasesSet.Add($t) | Out-Null
     }
 }
@@ -102,7 +102,7 @@ $ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeExes -ne "") {
     Write-Host "ExcludeExe: $ExcludeExes"
     $l = $ExcludeExes -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeExesSet.Add($t) | Out-Null
     }
 }
@@ -118,10 +118,6 @@ if($ExcludeExes -ne "") {
 #   MultiThreaded/MultiThreadedDBTest.
 #     MultiThreaded/0  # GetParam() = 0
 #     MultiThreaded/1  # GetParam() = 1
-#   RibbonTypeParamTest/0.  # TypeParam = struct DefaultTypesAndSettings
-#     CompactnessAndBacktrackAndFpRate
-#     Extremes
-#     FindOccupancyForSuccessRate
 #
 # into this:
 #
@@ -129,9 +125,6 @@ if($ExcludeExes -ne "") {
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-#   RibbonTypeParamTest/0.CompactnessAndBacktrackAndFpRate
-#   RibbonTypeParamTest/0.Extremes
-#   RibbonTypeParamTest/0.FindOccupancyForSuccessRate
 #
 # Output into the parameter in a form TestName -> Log File Name
 function ExtractTestCases([string]$GTestExe, $HashTable) {
@@ -145,8 +138,6 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
     ForEach( $l in $Tests) {
 
-      # remove trailing comment if any
-      $l = $l -replace '\s+\#.*',''
       # Leading whitespace is fine
       $l = $l -replace '^\s+',''
       # Trailing dot is a test group but no whitespace
@@ -155,7 +146,8 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
       }  else {
         # Otherwise it is a test name, remove leading space
         $test = $l
-        # create a log name
+        # remove trailing comment if any and create a log name
+        $test = $test -replace '\s+\#.*',''
         $test = "$Group$test"
 
         if($ExcludeCasesSet.Contains($test)) {
@@ -261,7 +253,7 @@ if($Run -ne "") {
 
   $DiscoveredExe = @()
   dir -Path $search_path | ForEach-Object {
-     $DiscoveredExe += ($_.Name)
+     $DiscoveredExe += ($_.Name)     
   }
 
   # Remove exclusions
@@ -301,7 +293,7 @@ if($SuiteRun -ne "") {
 
   $ListOfExe = @()
   dir -Path $search_path | ForEach-Object {
-     $ListOfExe += ($_.Name)
+     $ListOfExe += ($_.Name)     
   }
 
   # Exclude those in RunOnly from running as suites
@@ -356,7 +348,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
-          ($TestCmds.Count -gt 0) -or
+          ($TestCmds.Count -gt 0) -or 
            ($Suites.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
@@ -476,8 +468,8 @@ RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
 
 $EndDate = (Get-Date)
 
-New-TimeSpan -Start $StartDate -End $EndDate |
-  ForEach-Object {
+New-TimeSpan -Start $StartDate -End $EndDate | 
+  ForEach-Object { 
     "Elapsed time: {0:g}" -f $_
   }
 
@@ -492,4 +484,4 @@ if(!$script:success) {
 
  exit 0
 
-
+ 

build_tools/setup_centos7.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-set -ex
+set -e
 
-ROCKSDB_VERSION="6.7.3"
-ZSTD_VERSION="1.4.4"
+ROCKSDB_VERSION="5.10.3"
+ZSTD_VERSION="1.1.3"
 
 echo "This script configures CentOS with everything needed to build and run RocksDB"
 
@@ -40,6 +40,5 @@ cd /usr/local/rocksdb
 chown -R vagrant:vagrant /usr/local/rocksdb/
 sudo -u vagrant make static_lib
 cd examples/
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ make all
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ ./c_simple_example
-
+sudo -u vagrant make all
+sudo -u vagrant ./c_simple_example

cache/cache.cc

@@ -1,72 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "rocksdb/cache.h"
-
-#include "cache/lru_cache.h"
-#include "rocksdb/secondary_cache.h"
-#include "rocksdb/utilities/customizable_util.h"
-#include "rocksdb/utilities/options_type.h"
-#include "util/string_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-#ifndef ROCKSDB_LITE
-static std::unordered_map<std::string, OptionTypeInfo>
-    lru_cache_options_type_info = {
-        {"capacity",
-         {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
-        {"num_shard_bits",
-         {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
-        {"strict_capacity_limit",
-         {offsetof(struct LRUCacheOptions, strict_capacity_limit),
-          OptionType::kBoolean, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable}},
-        {"high_pri_pool_ratio",
-         {offsetof(struct LRUCacheOptions, high_pri_pool_ratio),
-          OptionType::kDouble, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable}},
-};
-#endif  // ROCKSDB_LITE
-
-Status SecondaryCache::CreateFromString(
-    const ConfigOptions& config_options, const std::string& value,
-    std::shared_ptr<SecondaryCache>* result) {
-  return LoadSharedObject<SecondaryCache>(config_options, value, nullptr,
-                                          result);
-}
-
-Status Cache::CreateFromString(const ConfigOptions& config_options,
-                               const std::string& value,
-                               std::shared_ptr<Cache>* result) {
-  Status status;
-  std::shared_ptr<Cache> cache;
-  if (value.find('=') == std::string::npos) {
-    cache = NewLRUCache(ParseSizeT(value));
-  } else {
-#ifndef ROCKSDB_LITE
-    LRUCacheOptions cache_opts;
-    status = OptionTypeInfo::ParseStruct(config_options, "",
-                                         &lru_cache_options_type_info, "",
-                                         value, &cache_opts);
-    if (status.ok()) {
-      cache = NewLRUCache(cache_opts);
-    }
-#else
-    (void)config_options;
-    status = Status::NotSupported("Cannot load cache in LITE mode ", value);
-#endif  //! ROCKSDB_LITE
-  }
-  if (status.ok()) {
-    result->swap(cache);
-  }
-  return status;
-}
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_bench.cc

@@ -1,11 +1,8 @@
-//  Copyright (c) 2013-present, Facebook, Inc.  All rights reserved.
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
+
 #ifndef GFLAGS
 #include <cstdio>
 int main() {
@@ -13,8 +10,272 @@ int main() {
   return 1;
 }
 #else
-#include <rocksdb/cache_bench_tool.h>
-int main(int argc, char** argv) {
-  return ROCKSDB_NAMESPACE::cache_bench_tool(argc, argv);
+
+#include <stdio.h>
+#include <sys/types.h>
+#include <cinttypes>
+
+#include "port/port.h"
+#include "rocksdb/cache.h"
+#include "rocksdb/db.h"
+#include "rocksdb/env.h"
+#include "util/gflags_compat.h"
+#include "util/mutexlock.h"
+#include "util/random.h"
+
+using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+
+static const uint32_t KB = 1024;
+
+DEFINE_int32(threads, 16, "Number of concurrent threads to run.");
+DEFINE_int64(cache_size, 8 * KB * KB,
+             "Number of bytes to use as a cache of uncompressed data.");
+DEFINE_int32(num_shard_bits, 4, "shard_bits.");
+
+DEFINE_int64(max_key, 1 * KB * KB * KB, "Max number of key to place in cache");
+DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
+
+DEFINE_bool(populate_cache, false, "Populate cache before operations");
+DEFINE_int32(insert_percent, 40,
+             "Ratio of insert to total workload (expressed as a percentage)");
+DEFINE_int32(lookup_percent, 50,
+             "Ratio of lookup to total workload (expressed as a percentage)");
+DEFINE_int32(erase_percent, 10,
+             "Ratio of erase to total workload (expressed as a percentage)");
+
+DEFINE_bool(use_clock_cache, false, "");
+
+namespace ROCKSDB_NAMESPACE {
+
+class CacheBench;
+namespace {
+void deleter(const Slice& /*key*/, void* value) {
+    delete reinterpret_cast<char *>(value);
 }
+
+// State shared by all concurrent executions of the same benchmark.
+class SharedState {
+ public:
+  explicit SharedState(CacheBench* cache_bench)
+      : cv_(&mu_),
+        num_threads_(FLAGS_threads),
+        num_initialized_(0),
+        start_(false),
+        num_done_(0),
+        cache_bench_(cache_bench) {
+  }
+
+  ~SharedState() {}
+
+  port::Mutex* GetMutex() {
+    return &mu_;
+  }
+
+  port::CondVar* GetCondVar() {
+    return &cv_;
+  }
+
+  CacheBench* GetCacheBench() const {
+    return cache_bench_;
+  }
+
+  void IncInitialized() {
+    num_initialized_++;
+  }
+
+  void IncDone() {
+    num_done_++;
+  }
+
+  bool AllInitialized() const {
+    return num_initialized_ >= num_threads_;
+  }
+
+  bool AllDone() const {
+    return num_done_ >= num_threads_;
+  }
+
+  void SetStart() {
+    start_ = true;
+  }
+
+  bool Started() const {
+    return start_;
+  }
+
+ private:
+  port::Mutex mu_;
+  port::CondVar cv_;
+
+  const uint64_t num_threads_;
+  uint64_t num_initialized_;
+  bool start_;
+  uint64_t num_done_;
+
+  CacheBench* cache_bench_;
+};
+
+// Per-thread state for concurrent executions of the same benchmark.
+struct ThreadState {
+  uint32_t tid;
+  Random rnd;
+  SharedState* shared;
+
+  ThreadState(uint32_t index, SharedState* _shared)
+      : tid(index), rnd(1000 + index), shared(_shared) {}
+};
+}  // namespace
+
+class CacheBench {
+ public:
+  CacheBench() : num_threads_(FLAGS_threads) {
+    if (FLAGS_use_clock_cache) {
+      cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+      if (!cache_) {
+        fprintf(stderr, "Clock cache not supported.\n");
+        exit(1);
+      }
+    } else {
+      cache_ = NewLRUCache(FLAGS_cache_size, FLAGS_num_shard_bits);
+    }
+  }
+
+  ~CacheBench() {}
+
+  void PopulateCache() {
+    Random rnd(1);
+    for (int64_t i = 0; i < FLAGS_cache_size; i++) {
+      uint64_t rand_key = rnd.Next() % FLAGS_max_key;
+      // Cast uint64* to be char*, data would be copied to cache
+      Slice key(reinterpret_cast<char*>(&rand_key), 8);
+      // do insert
+      cache_->Insert(key, new char[10], 1, &deleter);
+    }
+  }
+
+  bool Run() {
+    ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
+
+    PrintEnv();
+    SharedState shared(this);
+    std::vector<ThreadState*> threads(num_threads_);
+    for (uint32_t i = 0; i < num_threads_; i++) {
+      threads[i] = new ThreadState(i, &shared);
+      env->StartThread(ThreadBody, threads[i]);
+    }
+    {
+      MutexLock l(shared.GetMutex());
+      while (!shared.AllInitialized()) {
+        shared.GetCondVar()->Wait();
+      }
+      // Record start time
+      uint64_t start_time = env->NowMicros();
+
+      // Start all threads
+      shared.SetStart();
+      shared.GetCondVar()->SignalAll();
+
+      // Wait threads to complete
+      while (!shared.AllDone()) {
+        shared.GetCondVar()->Wait();
+      }
+
+      // Record end time
+      uint64_t end_time = env->NowMicros();
+      double elapsed = static_cast<double>(end_time - start_time) * 1e-6;
+      uint32_t qps = static_cast<uint32_t>(
+          static_cast<double>(FLAGS_threads * FLAGS_ops_per_thread) / elapsed);
+      fprintf(stdout, "Complete in %.3f s; QPS = %u\n", elapsed, qps);
+    }
+    return true;
+  }
+
+ private:
+  std::shared_ptr<Cache> cache_;
+  uint32_t num_threads_;
+
+  static void ThreadBody(void* v) {
+    ThreadState* thread = reinterpret_cast<ThreadState*>(v);
+    SharedState* shared = thread->shared;
+
+    {
+      MutexLock l(shared->GetMutex());
+      shared->IncInitialized();
+      if (shared->AllInitialized()) {
+        shared->GetCondVar()->SignalAll();
+      }
+      while (!shared->Started()) {
+        shared->GetCondVar()->Wait();
+      }
+    }
+    thread->shared->GetCacheBench()->OperateCache(thread);
+
+    {
+      MutexLock l(shared->GetMutex());
+      shared->IncDone();
+      if (shared->AllDone()) {
+        shared->GetCondVar()->SignalAll();
+      }
+    }
+  }
+
+  void OperateCache(ThreadState* thread) {
+    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
+      uint64_t rand_key = thread->rnd.Next() % FLAGS_max_key;
+      // Cast uint64* to be char*, data would be copied to cache
+      Slice key(reinterpret_cast<char*>(&rand_key), 8);
+      int32_t prob_op = thread->rnd.Uniform(100);
+      if (prob_op >= 0 && prob_op < FLAGS_insert_percent) {
+        // do insert
+        cache_->Insert(key, new char[10], 1, &deleter);
+      } else if (prob_op -= FLAGS_insert_percent &&
+                 prob_op < FLAGS_lookup_percent) {
+        // do lookup
+        auto handle = cache_->Lookup(key);
+        if (handle) {
+          cache_->Release(handle);
+        }
+      } else if (prob_op -= FLAGS_lookup_percent &&
+                 prob_op < FLAGS_erase_percent) {
+        // do erase
+        cache_->Erase(key);
+      }
+    }
+  }
+
+  void PrintEnv() const {
+    printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
+    printf("Number of threads   : %d\n", FLAGS_threads);
+    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
+    printf("Cache size          : %" PRIu64 "\n", FLAGS_cache_size);
+    printf("Num shard bits      : %d\n", FLAGS_num_shard_bits);
+    printf("Max key             : %" PRIu64 "\n", FLAGS_max_key);
+    printf("Populate cache      : %d\n", FLAGS_populate_cache);
+    printf("Insert percentage   : %d%%\n", FLAGS_insert_percent);
+    printf("Lookup percentage   : %d%%\n", FLAGS_lookup_percent);
+    printf("Erase percentage    : %d%%\n", FLAGS_erase_percent);
+    printf("----------------------------\n");
+  }
+};
+}  // namespace ROCKSDB_NAMESPACE
+
+int main(int argc, char** argv) {
+  ParseCommandLineFlags(&argc, &argv, true);
+
+  if (FLAGS_threads <= 0) {
+    fprintf(stderr, "threads number <= 0\n");
+    exit(1);
+  }
+
+  ROCKSDB_NAMESPACE::CacheBench bench;
+  if (FLAGS_populate_cache) {
+    bench.PopulateCache();
+  }
+  if (bench.Run()) {
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
 #endif  // GFLAGS

cache/cache_bench_tool.cc

@@ -1,572 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#ifdef GFLAGS
-#include <cinttypes>
-#include <cstdio>
-#include <limits>
-#include <set>
-#include <sstream>
-
-#include "monitoring/histogram.h"
-#include "port/port.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/convenience.h"
-#include "rocksdb/db.h"
-#include "rocksdb/env.h"
-#include "rocksdb/secondary_cache.h"
-#include "rocksdb/system_clock.h"
-#include "table/block_based/cachable_entry.h"
-#include "util/coding.h"
-#include "util/gflags_compat.h"
-#include "util/hash.h"
-#include "util/mutexlock.h"
-#include "util/random.h"
-#include "util/stop_watch.h"
-#include "util/string_util.h"
-
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
-
-static constexpr uint32_t KiB = uint32_t{1} << 10;
-static constexpr uint32_t MiB = KiB << 10;
-static constexpr uint64_t GiB = MiB << 10;
-
-DEFINE_uint32(threads, 16, "Number of concurrent threads to run.");
-DEFINE_uint64(cache_size, 1 * GiB,
-              "Number of bytes to use as a cache of uncompressed data.");
-DEFINE_uint32(num_shard_bits, 6, "shard_bits.");
-
-DEFINE_double(resident_ratio, 0.25,
-              "Ratio of keys fitting in cache to keyspace.");
-DEFINE_uint64(ops_per_thread, 2000000U, "Number of operations per thread.");
-DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added.");
-
-DEFINE_uint32(skew, 5, "Degree of skew in key selection");
-DEFINE_bool(populate_cache, true, "Populate cache before operations");
-
-DEFINE_uint32(lookup_insert_percent, 87,
-              "Ratio of lookup (+ insert on not found) to total workload "
-              "(expressed as a percentage)");
-DEFINE_uint32(insert_percent, 2,
-              "Ratio of insert to total workload (expressed as a percentage)");
-DEFINE_uint32(lookup_percent, 10,
-              "Ratio of lookup to total workload (expressed as a percentage)");
-DEFINE_uint32(erase_percent, 1,
-              "Ratio of erase to total workload (expressed as a percentage)");
-DEFINE_bool(gather_stats, false,
-            "Whether to periodically simulate gathering block cache stats, "
-            "using one more thread.");
-DEFINE_uint32(
-    gather_stats_sleep_ms, 1000,
-    "How many milliseconds to sleep between each gathering of stats.");
-
-DEFINE_uint32(gather_stats_entries_per_lock, 256,
-              "For Cache::ApplyToAllEntries");
-DEFINE_bool(skewed, false, "If true, skew the key access distribution");
-#ifndef ROCKSDB_LITE
-DEFINE_string(secondary_cache_uri, "",
-              "Full URI for creating a custom secondary cache object");
-static class std::shared_ptr<ROCKSDB_NAMESPACE::SecondaryCache> secondary_cache;
-#endif  // ROCKSDB_LITE
-
-DEFINE_bool(use_clock_cache, false, "");
-
-namespace ROCKSDB_NAMESPACE {
-
-class CacheBench;
-namespace {
-// State shared by all concurrent executions of the same benchmark.
-class SharedState {
- public:
-  explicit SharedState(CacheBench* cache_bench)
-      : cv_(&mu_),
-        num_initialized_(0),
-        start_(false),
-        num_done_(0),
-        cache_bench_(cache_bench) {}
-
-  ~SharedState() {}
-
-  port::Mutex* GetMutex() { return &mu_; }
-
-  port::CondVar* GetCondVar() { return &cv_; }
-
-  CacheBench* GetCacheBench() const { return cache_bench_; }
-
-  void IncInitialized() { num_initialized_++; }
-
-  void IncDone() { num_done_++; }
-
-  bool AllInitialized() const { return num_initialized_ >= FLAGS_threads; }
-
-  bool AllDone() const { return num_done_ >= FLAGS_threads; }
-
-  void SetStart() { start_ = true; }
-
-  bool Started() const { return start_; }
-
- private:
-  port::Mutex mu_;
-  port::CondVar cv_;
-
-  uint64_t num_initialized_;
-  bool start_;
-  uint64_t num_done_;
-
-  CacheBench* cache_bench_;
-};
-
-// Per-thread state for concurrent executions of the same benchmark.
-struct ThreadState {
-  uint32_t tid;
-  Random64 rnd;
-  SharedState* shared;
-  HistogramImpl latency_ns_hist;
-  uint64_t duration_us = 0;
-
-  ThreadState(uint32_t index, SharedState* _shared)
-      : tid(index), rnd(1000 + index), shared(_shared) {}
-};
-
-struct KeyGen {
-  char key_data[27];
-
-  Slice GetRand(Random64& rnd, uint64_t max_key, int max_log) {
-    uint64_t key = 0;
-    if (!FLAGS_skewed) {
-      uint64_t raw = rnd.Next();
-      // Skew according to setting
-      for (uint32_t i = 0; i < FLAGS_skew; ++i) {
-        raw = std::min(raw, rnd.Next());
-      }
-      key = FastRange64(raw, max_key);
-    } else {
-      key = rnd.Skewed(max_log);
-      if (key > max_key) {
-        key -= max_key;
-      }
-    }
-    // Variable size and alignment
-    size_t off = key % 8;
-    key_data[0] = char{42};
-    EncodeFixed64(key_data + 1, key);
-    key_data[9] = char{11};
-    EncodeFixed64(key_data + 10, key);
-    key_data[18] = char{4};
-    EncodeFixed64(key_data + 19, key);
-    return Slice(&key_data[off], sizeof(key_data) - off);
-  }
-};
-
-char* createValue(Random64& rnd) {
-  char* rv = new char[FLAGS_value_bytes];
-  // Fill with some filler data, and take some CPU time
-  for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
-    EncodeFixed64(rv + i, rnd.Next());
-  }
-  return rv;
-}
-
-// Callbacks for secondary cache
-size_t SizeFn(void* /*obj*/) { return FLAGS_value_bytes; }
-
-Status SaveToFn(void* obj, size_t /*offset*/, size_t size, void* out) {
-  memcpy(out, obj, size);
-  return Status::OK();
-}
-
-// Different deleters to simulate using deleter to gather
-// stats on the code origin and kind of cache entries.
-void deleter1(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-void deleter2(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-void deleter3(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
-
-Cache::CacheItemHelper helper1(SizeFn, SaveToFn, deleter1);
-Cache::CacheItemHelper helper2(SizeFn, SaveToFn, deleter2);
-Cache::CacheItemHelper helper3(SizeFn, SaveToFn, deleter3);
-}  // namespace
-
-class CacheBench {
-  static constexpr uint64_t kHundredthUint64 =
-      std::numeric_limits<uint64_t>::max() / 100U;
-
- public:
-  CacheBench()
-      : max_key_(static_cast<uint64_t>(FLAGS_cache_size / FLAGS_resident_ratio /
-                                       FLAGS_value_bytes)),
-        lookup_insert_threshold_(kHundredthUint64 *
-                                 FLAGS_lookup_insert_percent),
-        insert_threshold_(lookup_insert_threshold_ +
-                          kHundredthUint64 * FLAGS_insert_percent),
-        lookup_threshold_(insert_threshold_ +
-                          kHundredthUint64 * FLAGS_lookup_percent),
-        erase_threshold_(lookup_threshold_ +
-                         kHundredthUint64 * FLAGS_erase_percent),
-        skewed_(FLAGS_skewed) {
-    if (erase_threshold_ != 100U * kHundredthUint64) {
-      fprintf(stderr, "Percentages must add to 100.\n");
-      exit(1);
-    }
-
-    max_log_ = 0;
-    if (skewed_) {
-      uint64_t max_key = max_key_;
-      while (max_key >>= 1) max_log_++;
-      if (max_key > (1u << max_log_)) max_log_++;
-    }
-
-    if (FLAGS_use_clock_cache) {
-      cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
-      if (!cache_) {
-        fprintf(stderr, "Clock cache not supported.\n");
-        exit(1);
-      }
-    } else {
-      LRUCacheOptions opts(FLAGS_cache_size, FLAGS_num_shard_bits, false, 0.5);
-#ifndef ROCKSDB_LITE
-      if (!FLAGS_secondary_cache_uri.empty()) {
-        Status s = SecondaryCache::CreateFromString(
-            ConfigOptions(), FLAGS_secondary_cache_uri, &secondary_cache);
-        if (secondary_cache == nullptr) {
-          fprintf(
-              stderr,
-              "No secondary cache registered matching string: %s status=%s\n",
-              FLAGS_secondary_cache_uri.c_str(), s.ToString().c_str());
-          exit(1);
-        }
-        opts.secondary_cache = secondary_cache;
-      }
-#endif  // ROCKSDB_LITE
-
-      cache_ = NewLRUCache(opts);
-    }
-  }
-
-  ~CacheBench() {}
-
-  void PopulateCache() {
-    Random64 rnd(1);
-    KeyGen keygen;
-    for (uint64_t i = 0; i < 2 * FLAGS_cache_size; i += FLAGS_value_bytes) {
-      cache_->Insert(keygen.GetRand(rnd, max_key_, max_log_), createValue(rnd),
-                     &helper1, FLAGS_value_bytes);
-    }
-  }
-
-  bool Run() {
-    const auto clock = SystemClock::Default().get();
-
-    PrintEnv();
-    SharedState shared(this);
-    std::vector<std::unique_ptr<ThreadState> > threads(FLAGS_threads);
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      threads[i].reset(new ThreadState(i, &shared));
-      std::thread(ThreadBody, threads[i].get()).detach();
-    }
-
-    HistogramImpl stats_hist;
-    std::string stats_report;
-    std::thread stats_thread(StatsBody, &shared, &stats_hist, &stats_report);
-
-    uint64_t start_time;
-    {
-      MutexLock l(shared.GetMutex());
-      while (!shared.AllInitialized()) {
-        shared.GetCondVar()->Wait();
-      }
-      // Record start time
-      start_time = clock->NowMicros();
-
-      // Start all threads
-      shared.SetStart();
-      shared.GetCondVar()->SignalAll();
-
-      // Wait threads to complete
-      while (!shared.AllDone()) {
-        shared.GetCondVar()->Wait();
-      }
-    }
-
-    // Stats gathering is considered background work. This time measurement
-    // is for foreground work, and not really ideal for that. See below.
-    uint64_t end_time = clock->NowMicros();
-    stats_thread.join();
-
-    // Wall clock time - includes idle time if threads
-    // finish at different times (not ideal).
-    double elapsed_secs = static_cast<double>(end_time - start_time) * 1e-6;
-    uint32_t ops_per_sec = static_cast<uint32_t>(
-        1.0 * FLAGS_threads * FLAGS_ops_per_thread / elapsed_secs);
-    printf("Complete in %.3f s; Rough parallel ops/sec = %u\n", elapsed_secs,
-           ops_per_sec);
-
-    // Total time in each thread (more accurate throughput measure)
-    elapsed_secs = 0;
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      elapsed_secs += threads[i]->duration_us * 1e-6;
-    }
-    ops_per_sec = static_cast<uint32_t>(1.0 * FLAGS_threads *
-                                        FLAGS_ops_per_thread / elapsed_secs);
-    printf("Thread ops/sec = %u\n", ops_per_sec);
-
-    printf("\nOperation latency (ns):\n");
-    HistogramImpl combined;
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      combined.Merge(threads[i]->latency_ns_hist);
-    }
-    printf("%s", combined.ToString().c_str());
-
-    if (FLAGS_gather_stats) {
-      printf("\nGather stats latency (us):\n");
-      printf("%s", stats_hist.ToString().c_str());
-    }
-
-    printf("\n%s", stats_report.c_str());
-
-    return true;
-  }
-
- private:
-  std::shared_ptr<Cache> cache_;
-  const uint64_t max_key_;
-  // Cumulative thresholds in the space of a random uint64_t
-  const uint64_t lookup_insert_threshold_;
-  const uint64_t insert_threshold_;
-  const uint64_t lookup_threshold_;
-  const uint64_t erase_threshold_;
-  const bool skewed_;
-  int max_log_;
-
-  // A benchmark version of gathering stats on an active block cache by
-  // iterating over it. The primary purpose is to measure the impact of
-  // gathering stats with ApplyToAllEntries on throughput- and
-  // latency-sensitive Cache users. Performance of stats gathering is
-  // also reported. The last set of gathered stats is also reported, for
-  // manual sanity checking for logical errors or other unexpected
-  // behavior of cache_bench or the underlying Cache.
-  static void StatsBody(SharedState* shared, HistogramImpl* stats_hist,
-                        std::string* stats_report) {
-    if (!FLAGS_gather_stats) {
-      return;
-    }
-    const auto clock = SystemClock::Default().get();
-    uint64_t total_key_size = 0;
-    uint64_t total_charge = 0;
-    uint64_t total_entry_count = 0;
-    std::set<Cache::DeleterFn> deleters;
-    StopWatchNano timer(clock);
-
-    for (;;) {
-      uint64_t time;
-      time = clock->NowMicros();
-      uint64_t deadline = time + uint64_t{FLAGS_gather_stats_sleep_ms} * 1000;
-
-      {
-        MutexLock l(shared->GetMutex());
-        for (;;) {
-          if (shared->AllDone()) {
-            std::ostringstream ostr;
-            ostr << "Most recent cache entry stats:\n"
-                 << "Number of entries: " << total_entry_count << "\n"
-                 << "Total charge: " << BytesToHumanString(total_charge) << "\n"
-                 << "Average key size: "
-                 << (1.0 * total_key_size / total_entry_count) << "\n"
-                 << "Average charge: "
-                 << BytesToHumanString(1.0 * total_charge / total_entry_count)
-                 << "\n"
-                 << "Unique deleters: " << deleters.size() << "\n";
-            *stats_report = ostr.str();
-            return;
-          }
-          if (clock->NowMicros() >= deadline) {
-            break;
-          }
-          uint64_t diff = deadline - std::min(clock->NowMicros(), deadline);
-          shared->GetCondVar()->TimedWait(diff + 1);
-        }
-      }
-
-      // Now gather stats, outside of mutex
-      total_key_size = 0;
-      total_charge = 0;
-      total_entry_count = 0;
-      deleters.clear();
-      auto fn = [&](const Slice& key, void* /*value*/, size_t charge,
-                    Cache::DeleterFn deleter) {
-        total_key_size += key.size();
-        total_charge += charge;
-        ++total_entry_count;
-        // Something slightly more expensive as in (future) stats by category
-        deleters.insert(deleter);
-      };
-      timer.Start();
-      Cache::ApplyToAllEntriesOptions opts;
-      opts.average_entries_per_lock = FLAGS_gather_stats_entries_per_lock;
-      shared->GetCacheBench()->cache_->ApplyToAllEntries(fn, opts);
-      stats_hist->Add(timer.ElapsedNanos() / 1000);
-    }
-  }
-
-  static void ThreadBody(ThreadState* thread) {
-    SharedState* shared = thread->shared;
-
-    {
-      MutexLock l(shared->GetMutex());
-      shared->IncInitialized();
-      if (shared->AllInitialized()) {
-        shared->GetCondVar()->SignalAll();
-      }
-      while (!shared->Started()) {
-        shared->GetCondVar()->Wait();
-      }
-    }
-    thread->shared->GetCacheBench()->OperateCache(thread);
-
-    {
-      MutexLock l(shared->GetMutex());
-      shared->IncDone();
-      if (shared->AllDone()) {
-        shared->GetCondVar()->SignalAll();
-      }
-    }
-  }
-
-  void OperateCache(ThreadState* thread) {
-    // To use looked-up values
-    uint64_t result = 0;
-    // To hold handles for a non-trivial amount of time
-    Cache::Handle* handle = nullptr;
-    KeyGen gen;
-    const auto clock = SystemClock::Default().get();
-    uint64_t start_time = clock->NowMicros();
-    StopWatchNano timer(clock);
-
-    for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
-      timer.Start();
-      Slice key = gen.GetRand(thread->rnd, max_key_, max_log_);
-      uint64_t random_op = thread->rnd.Next();
-      Cache::CreateCallback create_cb =
-          [](void* buf, size_t size, void** out_obj, size_t* charge) -> Status {
-        *out_obj = reinterpret_cast<void*>(new char[size]);
-        memcpy(*out_obj, buf, size);
-        *charge = size;
-        return Status::OK();
-      };
-
-      if (random_op < lookup_insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do lookup
-        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
-                                true);
-        if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
-        } else {
-          // do insert
-          cache_->Insert(key, createValue(thread->rnd), &helper2,
-                         FLAGS_value_bytes, &handle);
-        }
-      } else if (random_op < insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do insert
-        cache_->Insert(key, createValue(thread->rnd), &helper3,
-                       FLAGS_value_bytes, &handle);
-      } else if (random_op < lookup_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do lookup
-        handle = cache_->Lookup(key, &helper2, create_cb, Cache::Priority::LOW,
-                                true);
-        if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
-        }
-      } else if (random_op < erase_threshold_) {
-        // do erase
-        cache_->Erase(key);
-      } else {
-        // Should be extremely unlikely (noop)
-        assert(random_op >= kHundredthUint64 * 100U);
-      }
-      thread->latency_ns_hist.Add(timer.ElapsedNanos());
-    }
-    if (handle) {
-      cache_->Release(handle);
-      handle = nullptr;
-    }
-    // Ensure computations on `result` are not optimized away.
-    if (result == 1) {
-      printf("You are extremely unlucky(2). Try again.\n");
-      exit(1);
-    }
-    thread->duration_us = clock->NowMicros() - start_time;
-  }
-
-  void PrintEnv() const {
-    printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
-    printf("Number of threads   : %u\n", FLAGS_threads);
-    printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
-    printf("Cache size          : %s\n",
-           BytesToHumanString(FLAGS_cache_size).c_str());
-    printf("Num shard bits      : %u\n", FLAGS_num_shard_bits);
-    printf("Max key             : %" PRIu64 "\n", max_key_);
-    printf("Resident ratio      : %g\n", FLAGS_resident_ratio);
-    printf("Skew degree         : %u\n", FLAGS_skew);
-    printf("Populate cache      : %d\n", int{FLAGS_populate_cache});
-    printf("Lookup+Insert pct   : %u%%\n", FLAGS_lookup_insert_percent);
-    printf("Insert percentage   : %u%%\n", FLAGS_insert_percent);
-    printf("Lookup percentage   : %u%%\n", FLAGS_lookup_percent);
-    printf("Erase percentage    : %u%%\n", FLAGS_erase_percent);
-    std::ostringstream stats;
-    if (FLAGS_gather_stats) {
-      stats << "enabled (" << FLAGS_gather_stats_sleep_ms << "ms, "
-            << FLAGS_gather_stats_entries_per_lock << "/lock)";
-    } else {
-      stats << "disabled";
-    }
-    printf("Gather stats        : %s\n", stats.str().c_str());
-    printf("----------------------------\n");
-  }
-};
-
-int cache_bench_tool(int argc, char** argv) {
-  ParseCommandLineFlags(&argc, &argv, true);
-
-  if (FLAGS_threads <= 0) {
-    fprintf(stderr, "threads number <= 0\n");
-    exit(1);
-  }
-
-  ROCKSDB_NAMESPACE::CacheBench bench;
-  if (FLAGS_populate_cache) {
-    bench.PopulateCache();
-    printf("Population complete\n");
-    printf("----------------------------\n");
-  }
-  if (bench.Run()) {
-    return 0;
-  } else {
-    return 1;
-  }
-}  // namespace ROCKSDB_NAMESPACE
-}  // namespace ROCKSDB_NAMESPACE
-
-#endif  // GFLAGS

cache/cache_entry_roles.cc

@@ -1,66 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "cache/cache_entry_roles.h"
-
-#include <mutex>
-
-#include "port/lang.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToCamelString{{
-    "DataBlock",
-    "FilterBlock",
-    "FilterMetaBlock",
-    "DeprecatedFilterBlock",
-    "IndexBlock",
-    "OtherBlock",
-    "WriteBuffer",
-    "Misc",
-}};
-
-std::array<const char*, kNumCacheEntryRoles> kCacheEntryRoleToHyphenString{{
-    "data-block",
-    "filter-block",
-    "filter-meta-block",
-    "deprecated-filter-block",
-    "index-block",
-    "other-block",
-    "write-buffer",
-    "misc",
-}};
-
-namespace {
-
-struct Registry {
-  std::mutex mutex;
-  std::unordered_map<Cache::DeleterFn, CacheEntryRole> role_map;
-  void Register(Cache::DeleterFn fn, CacheEntryRole role) {
-    std::lock_guard<std::mutex> lock(mutex);
-    role_map[fn] = role;
-  }
-  std::unordered_map<Cache::DeleterFn, CacheEntryRole> Copy() {
-    std::lock_guard<std::mutex> lock(mutex);
-    return role_map;
-  }
-};
-
-Registry& GetRegistry() {
-  STATIC_AVOID_DESTRUCTION(Registry, registry);
-  return registry;
-}
-
-}  // namespace
-
-void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role) {
-  GetRegistry().Register(fn, role);
-}
-
-std::unordered_map<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap() {
-  return GetRegistry().Copy();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_entry_roles.h

@@ -1,122 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <array>
-#include <cstdint>
-#include <unordered_map>
-
-#include "rocksdb/cache.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Classifications of block cache entries, for reporting statistics
-enum class CacheEntryRole {
-  // Block-based table data block
-  kDataBlock,
-  // Block-based table filter block (full or partitioned)
-  kFilterBlock,
-  // Block-based table metadata block for partitioned filter
-  kFilterMetaBlock,
-  // Block-based table deprecated filter block (old "block-based" filter)
-  kDeprecatedFilterBlock,
-  // Block-based table index block
-  kIndexBlock,
-  // Other kinds of block-based table block
-  kOtherBlock,
-  // WriteBufferManager reservations to account for memtable usage
-  kWriteBuffer,
-  // Default bucket, for miscellaneous cache entries. Do not use for
-  // entries that could potentially add up to large usage.
-  kMisc,
-};
-constexpr uint32_t kNumCacheEntryRoles =
-    static_cast<uint32_t>(CacheEntryRole::kMisc) + 1;
-
-extern std::array<const char*, kNumCacheEntryRoles>
-    kCacheEntryRoleToCamelString;
-extern std::array<const char*, kNumCacheEntryRoles>
-    kCacheEntryRoleToHyphenString;
-
-// To associate cache entries with their role, we use a hack on the
-// existing Cache interface. Because the deleter of an entry can authenticate
-// the code origin of an entry, we can elaborate the choice of deleter to
-// also encode role information, without inferring false role information
-// from entries not choosing to encode a role.
-//
-// The rest of this file is for handling mappings between deleters and
-// roles.
-
-// To infer a role from a deleter, the deleter must be registered. This
-// can be done "manually" with this function. This function is thread-safe,
-// and the registration mappings go into private but static storage. (Note
-// that DeleterFn is a function pointer, not std::function. Registrations
-// should not be too many.)
-void RegisterCacheDeleterRole(Cache::DeleterFn fn, CacheEntryRole role);
-
-// Gets a copy of the registered deleter -> role mappings. This is the only
-// function for reading the mappings made with RegisterCacheDeleterRole.
-// Why only this interface for reading?
-// * This function has to be thread safe, which could incur substantial
-// overhead. We should not pay this overhead for every deleter look-up.
-// * This is suitable for preparing for batch operations, like with
-// CacheEntryStatsCollector.
-// * The number of mappings should be sufficiently small (dozens).
-std::unordered_map<Cache::DeleterFn, CacheEntryRole> CopyCacheDeleterRoleMap();
-
-// ************************************************************** //
-// An automatic registration infrastructure. This enables code
-// to simply ask for a deleter associated with a particular type
-// and role, and registration is automatic. In a sense, this is
-// a small dependency injection infrastructure, because linking
-// in new deleter instantiations is essentially sufficient for
-// making stats collection (using CopyCacheDeleterRoleMap) aware
-// of them.
-
-namespace cache_entry_roles_detail {
-
-template <typename T, CacheEntryRole R>
-struct RegisteredDeleter {
-  RegisteredDeleter() { RegisterCacheDeleterRole(Delete, R); }
-
-  // These have global linkage to help ensure compiler optimizations do not
-  // break uniqueness for each <T,R>
-  static void Delete(const Slice& /* key */, void* value) {
-    delete static_cast<T*>(value);
-  }
-};
-
-template <CacheEntryRole R>
-struct RegisteredNoopDeleter {
-  RegisteredNoopDeleter() { RegisterCacheDeleterRole(Delete, R); }
-
-  static void Delete(const Slice& /* key */, void* value) {
-    (void)value;
-    assert(value == nullptr);
-  }
-};
-
-}  // namespace cache_entry_roles_detail
-
-// Get an automatically registered deleter for value type T and role R.
-// Based on C++ semantics, registration is invoked exactly once in a
-// thread-safe way on first call to this function, for each <T, R>.
-template <typename T, CacheEntryRole R>
-Cache::DeleterFn GetCacheEntryDeleterForRole() {
-  static cache_entry_roles_detail::RegisteredDeleter<T, R> reg;
-  return reg.Delete;
-}
-
-// Get an automatically registered no-op deleter (value should be nullptr)
-// and associated with role R. This is used for Cache "reservation" entries
-// such as for WriteBufferManager.
-template <CacheEntryRole R>
-Cache::DeleterFn GetNoopDeleterForRole() {
-  static cache_entry_roles_detail::RegisteredNoopDeleter<R> reg;
-  return reg.Delete;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_entry_stats.h

@@ -1,180 +0,0 @@
-//  Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <array>
-#include <cstdint>
-#include <memory>
-#include <mutex>
-
-#include "cache/cache_helpers.h"
-#include "port/lang.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/status.h"
-#include "rocksdb/system_clock.h"
-#include "util/coding_lean.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// A generic helper object for gathering stats about cache entries by
-// iterating over them with ApplyToAllEntries. This class essentially
-// solves the problem of slowing down a Cache with too many stats
-// collectors that could be sharing stat results, such as from multiple
-// column families or multiple DBs sharing a Cache. We employ a few
-// mitigations:
-// * Only one collector for a particular kind of Stats is alive
-// for each Cache. This is guaranteed using the Cache itself to hold
-// the collector.
-// * A mutex ensures only one thread is gathering stats for this
-// collector.
-// * The most recent gathered stats are saved and simply copied to
-// satisfy requests within a time window (default: 3 minutes) of
-// completion of the most recent stat gathering.
-//
-// Template parameter Stats must be copyable and trivially constructable,
-// as well as...
-// concept Stats {
-//   // Notification before applying callback to all entries
-//   void BeginCollection(Cache*, SystemClock*, uint64_t start_time_micros);
-//   // Get the callback to apply to all entries. `callback`
-//   // type must be compatible with Cache::ApplyToAllEntries
-//   callback GetEntryCallback();
-//   // Notification after applying callback to all entries
-//   void EndCollection(Cache*, SystemClock*, uint64_t end_time_micros);
-//   // Notification that a collection was skipped because of
-//   // sufficiently recent saved results.
-//   void SkippedCollection();
-// }
-template <class Stats>
-class CacheEntryStatsCollector {
- public:
-  // Gather and save stats if saved stats are too old. (Use GetStats() to
-  // read saved stats.)
-  //
-  // Maximum allowed age for a "hit" on saved results is determined by the
-  // two interval parameters. Both set to 0 forces a re-scan. For example
-  // with min_interval_seconds=300 and min_interval_factor=100, if the last
-  // scan took 10s, we would only rescan ("miss") if the age in seconds of
-  // the saved results is > max(300, 100*10).
-  // Justification: scans can vary wildly in duration, e.g. from 0.02 sec
-  // to as much as 20 seconds, so we want to be able to cap the absolute
-  // and relative frequency of scans.
-  void CollectStats(int min_interval_seconds, int min_interval_factor) {
-    // Waits for any pending reader or writer (collector)
-    std::lock_guard<std::mutex> lock(working_mutex_);
-
-    uint64_t max_age_micros =
-        static_cast<uint64_t>(std::max(min_interval_seconds, 0)) * 1000000U;
-
-    if (last_end_time_micros_ > last_start_time_micros_ &&
-        min_interval_factor > 0) {
-      max_age_micros = std::max(
-          max_age_micros, min_interval_factor * (last_end_time_micros_ -
-                                                 last_start_time_micros_));
-    }
-
-    uint64_t start_time_micros = clock_->NowMicros();
-    if ((start_time_micros - last_end_time_micros_) > max_age_micros) {
-      last_start_time_micros_ = start_time_micros;
-      working_stats_.BeginCollection(cache_, clock_, start_time_micros);
-
-      cache_->ApplyToAllEntries(working_stats_.GetEntryCallback(), {});
-      TEST_SYNC_POINT_CALLBACK(
-          "CacheEntryStatsCollector::GetStats:AfterApplyToAllEntries", nullptr);
-
-      uint64_t end_time_micros = clock_->NowMicros();
-      last_end_time_micros_ = end_time_micros;
-      working_stats_.EndCollection(cache_, clock_, end_time_micros);
-    } else {
-      working_stats_.SkippedCollection();
-    }
-
-    // Save so that we don't need to wait for an outstanding collection in
-    // order to make of copy of the last saved stats
-    std::lock_guard<std::mutex> lock2(saved_mutex_);
-    saved_stats_ = working_stats_;
-  }
-
-  // Gets saved stats, regardless of age
-  void GetStats(Stats *stats) {
-    std::lock_guard<std::mutex> lock(saved_mutex_);
-    *stats = saved_stats_;
-  }
-
-  Cache *GetCache() const { return cache_; }
-
-  // Gets or creates a shared instance of CacheEntryStatsCollector in the
-  // cache itself, and saves into `ptr`. This shared_ptr will hold the
-  // entry in cache until all refs are destroyed.
-  static Status GetShared(Cache *cache, SystemClock *clock,
-                          std::shared_ptr<CacheEntryStatsCollector> *ptr) {
-    std::array<uint64_t, 3> cache_key_data{
-        {// First 16 bytes == md5 of class name
-         0x7eba5a8fb5437c90U, 0x8ca68c9b11655855U,
-         // Last 8 bytes based on a function pointer to make unique for each
-         // template instantiation
-         reinterpret_cast<uint64_t>(&CacheEntryStatsCollector::GetShared)}};
-    Slice cache_key = GetSlice(&cache_key_data);
-
-    Cache::Handle *h = cache->Lookup(cache_key);
-    if (h == nullptr) {
-      // Not yet in cache, but Cache doesn't provide a built-in way to
-      // avoid racing insert. So we double-check under a shared mutex,
-      // inspired by TableCache.
-      STATIC_AVOID_DESTRUCTION(std::mutex, static_mutex);
-      std::lock_guard<std::mutex> lock(static_mutex);
-
-      h = cache->Lookup(cache_key);
-      if (h == nullptr) {
-        auto new_ptr = new CacheEntryStatsCollector(cache, clock);
-        // TODO: non-zero charge causes some tests that count block cache
-        // usage to go flaky. Fix the problem somehow so we can use an
-        // accurate charge.
-        size_t charge = 0;
-        Status s = cache->Insert(cache_key, new_ptr, charge, Deleter, &h,
-                                 Cache::Priority::HIGH);
-        if (!s.ok()) {
-          assert(h == nullptr);
-          delete new_ptr;
-          return s;
-        }
-      }
-    }
-    // If we reach here, shared entry is in cache with handle `h`.
-    assert(cache->GetDeleter(h) == Deleter);
-
-    // Build an aliasing shared_ptr that keeps `ptr` in cache while there
-    // are references.
-    *ptr = MakeSharedCacheHandleGuard<CacheEntryStatsCollector>(cache, h);
-    return Status::OK();
-  }
-
- private:
-  explicit CacheEntryStatsCollector(Cache *cache, SystemClock *clock)
-      : saved_stats_(),
-        working_stats_(),
-        last_start_time_micros_(0),
-        last_end_time_micros_(/*pessimistic*/ 10000000),
-        cache_(cache),
-        clock_(clock) {}
-
-  static void Deleter(const Slice &, void *value) {
-    delete static_cast<CacheEntryStatsCollector *>(value);
-  }
-
-  std::mutex saved_mutex_;
-  Stats saved_stats_;
-
-  std::mutex working_mutex_;
-  Stats working_stats_;
-  uint64_t last_start_time_micros_;
-  uint64_t last_end_time_micros_;
-
-  Cache *const cache_;
-  SystemClock *const clock_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_helpers.h

@@ -1,125 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-
-#include "rocksdb/cache.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Returns the cached value given a cache handle.
-template <typename T>
-T* GetFromCacheHandle(Cache* cache, Cache::Handle* handle) {
-  assert(cache);
-  assert(handle);
-
-  return static_cast<T*>(cache->Value(handle));
-}
-
-// Simple generic deleter for Cache (to be used with Cache::Insert).
-template <typename T>
-void DeleteCacheEntry(const Slice& /* key */, void* value) {
-  delete static_cast<T*>(value);
-}
-
-// Turns a T* into a Slice so it can be used as a key with Cache.
-template <typename T>
-Slice GetSlice(const T* t) {
-  return Slice(reinterpret_cast<const char*>(t), sizeof(T));
-}
-
-// Generic resource management object for cache handles that releases the handle
-// when destroyed. Has unique ownership of the handle, so copying it is not
-// allowed, while moving it transfers ownership.
-template <typename T>
-class CacheHandleGuard {
- public:
-  CacheHandleGuard() = default;
-
-  CacheHandleGuard(Cache* cache, Cache::Handle* handle)
-      : cache_(cache),
-        handle_(handle),
-        value_(GetFromCacheHandle<T>(cache, handle)) {
-    assert(cache_ && handle_ && value_);
-  }
-
-  CacheHandleGuard(const CacheHandleGuard&) = delete;
-  CacheHandleGuard& operator=(const CacheHandleGuard&) = delete;
-
-  CacheHandleGuard(CacheHandleGuard&& rhs) noexcept
-      : cache_(rhs.cache_), handle_(rhs.handle_), value_(rhs.value_) {
-    assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_));
-
-    rhs.ResetFields();
-  }
-
-  CacheHandleGuard& operator=(CacheHandleGuard&& rhs) noexcept {
-    if (this == &rhs) {
-      return *this;
-    }
-
-    ReleaseHandle();
-
-    cache_ = rhs.cache_;
-    handle_ = rhs.handle_;
-    value_ = rhs.value_;
-
-    assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_));
-
-    rhs.ResetFields();
-
-    return *this;
-  }
-
-  ~CacheHandleGuard() { ReleaseHandle(); }
-
-  bool IsEmpty() const { return !handle_; }
-
-  Cache* GetCache() const { return cache_; }
-  Cache::Handle* GetCacheHandle() const { return handle_; }
-  T* GetValue() const { return value_; }
-
-  void Reset() {
-    ReleaseHandle();
-    ResetFields();
-  }
-
- private:
-  void ReleaseHandle() {
-    if (IsEmpty()) {
-      return;
-    }
-
-    assert(cache_);
-    cache_->Release(handle_);
-  }
-
-  void ResetFields() {
-    cache_ = nullptr;
-    handle_ = nullptr;
-    value_ = nullptr;
-  }
-
- private:
-  Cache* cache_ = nullptr;
-  Cache::Handle* handle_ = nullptr;
-  T* value_ = nullptr;
-};
-
-// Build an aliasing shared_ptr that keeps `handle` in cache while there
-// are references, but the pointer is to the value for that cache entry,
-// which must be of type T. This is copyable, unlike CacheHandleGuard, but
-// does not provide access to caching details.
-template <typename T>
-std::shared_ptr<T> MakeSharedCacheHandleGuard(Cache* cache,
-                                              Cache::Handle* handle) {
-  auto wrapper = std::make_shared<CacheHandleGuard<T>>(cache, handle);
-  return std::shared_ptr<T>(wrapper, static_cast<T*>(cache->Value(handle)));
-}
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_test.cc

@@ -117,8 +117,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
 
   void Insert(std::shared_ptr<Cache> cache, int key, int value,
               int charge = 1) {
-    EXPECT_OK(cache->Insert(EncodeKey(key), EncodeValue(value), charge,
-                            &CacheTest::Deleter));
+    cache->Insert(EncodeKey(key), EncodeValue(value), charge,
+                  &CacheTest::Deleter);
   }
 
   void Erase(std::shared_ptr<Cache> cache, int key) {
@@ -167,10 +167,9 @@ TEST_P(CacheTest, UsageTest) {
   for (int i = 1; i < 100; ++i) {
     std::string key(i, 'a');
     auto kv_size = key.size() + 5;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter);
+    precise_cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
+                          dumbDeleter);
     usage += kv_size;
     ASSERT_EQ(usage, cache->GetUsage());
     ASSERT_LT(usage, precise_cache->GetUsage());
@@ -184,10 +183,10 @@ TEST_P(CacheTest, UsageTest) {
   // make sure the cache will be overloaded
   for (uint64_t i = 1; i < kCapacity; ++i) {
     auto key = ToString(i);
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                  dumbDeleter);
+    precise_cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                          dumbDeleter);
   }
 
   // the usage should be close to the capacity
@@ -216,12 +215,11 @@ TEST_P(CacheTest, PinnedUsageTest) {
     auto kv_size = key.size() + 5;
     Cache::Handle* handle;
     Cache::Handle* handle_in_precise_cache;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter, &handle));
+    cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter,
+                  &handle);
     assert(handle);
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter,
-                                    &handle_in_precise_cache));
+    precise_cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
+                          dumbDeleter, &handle_in_precise_cache);
     assert(handle_in_precise_cache);
     pinned_usage += kv_size;
     ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
@@ -256,10 +254,10 @@ TEST_P(CacheTest, PinnedUsageTest) {
   // check that overloading the cache does not change the pinned usage
   for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
     auto key = ToString(i);
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                  dumbDeleter);
+    precise_cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                          dumbDeleter);
   }
   ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
   ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
@@ -712,98 +710,25 @@ TEST_P(CacheTest, OverCapacity) {
 }
 
 namespace {
-std::vector<std::pair<int, int>> legacy_callback_state;
-void legacy_callback(void* value, size_t charge) {
-  legacy_callback_state.push_back(
-      {DecodeValue(value), static_cast<int>(charge)});
+std::vector<std::pair<int, int>> callback_state;
+void callback(void* entry, size_t charge) {
+  callback_state.push_back({DecodeValue(entry), static_cast<int>(charge)});
 }
 };
 
-TEST_P(CacheTest, ApplyToAllCacheEntriesTest) {
+TEST_P(CacheTest, ApplyToAllCacheEntiresTest) {
   std::vector<std::pair<int, int>> inserted;
-  legacy_callback_state.clear();
+  callback_state.clear();
 
   for (int i = 0; i < 10; ++i) {
     Insert(i, i * 2, i + 1);
     inserted.push_back({i * 2, i + 1});
   }
-  cache_->ApplyToAllCacheEntries(legacy_callback, true);
-
-  std::sort(inserted.begin(), inserted.end());
-  std::sort(legacy_callback_state.begin(), legacy_callback_state.end());
-  ASSERT_EQ(inserted.size(), legacy_callback_state.size());
-  for (size_t i = 0; i < inserted.size(); ++i) {
-    EXPECT_EQ(inserted[i], legacy_callback_state[i]);
-  }
-}
-
-TEST_P(CacheTest, ApplyToAllEntriesTest) {
-  std::vector<std::string> callback_state;
-  const auto callback = [&](const Slice& key, void* value, size_t charge,
-                            Cache::DeleterFn deleter) {
-    callback_state.push_back(ToString(DecodeKey(key)) + "," +
-                             ToString(DecodeValue(value)) + "," +
-                             ToString(charge));
-    assert(deleter == &CacheTest::Deleter);
-  };
-
-  std::vector<std::string> inserted;
-  callback_state.clear();
-
-  for (int i = 0; i < 10; ++i) {
-    Insert(i, i * 2, i + 1);
-    inserted.push_back(ToString(i) + "," + ToString(i * 2) + "," +
-                       ToString(i + 1));
-  }
-  cache_->ApplyToAllEntries(callback, /*opts*/ {});
+  cache_->ApplyToAllCacheEntries(callback, true);
 
   std::sort(inserted.begin(), inserted.end());
   std::sort(callback_state.begin(), callback_state.end());
-  ASSERT_EQ(inserted.size(), callback_state.size());
-  for (size_t i = 0; i < inserted.size(); ++i) {
-    EXPECT_EQ(inserted[i], callback_state[i]);
-  }
-}
-
-TEST_P(CacheTest, ApplyToAllEntriesDuringResize) {
-  // This is a mini-stress test of ApplyToAllEntries, to ensure
-  // items in the cache that are neither added nor removed
-  // during ApplyToAllEntries are counted exactly once.
-
-  // Insert some entries that we expect to be seen exactly once
-  // during iteration.
-  constexpr int kSpecialCharge = 2;
-  constexpr int kNotSpecialCharge = 1;
-  constexpr int kSpecialCount = 100;
-  for (int i = 0; i < kSpecialCount; ++i) {
-    Insert(i, i * 2, kSpecialCharge);
-  }
-
-  // For callback
-  int special_count = 0;
-  const auto callback = [&](const Slice&, void*, size_t charge,
-                            Cache::DeleterFn) {
-    if (charge == static_cast<size_t>(kSpecialCharge)) {
-      ++special_count;
-    }
-  };
-
-  // Start counting
-  std::thread apply_thread([&]() {
-    // Use small average_entries_per_lock to make the problem difficult
-    Cache::ApplyToAllEntriesOptions opts;
-    opts.average_entries_per_lock = 2;
-    cache_->ApplyToAllEntries(callback, opts);
-  });
-
-  // In parallel, add more entries, enough to cause resize but not enough
-  // to cause ejections
-  for (int i = kSpecialCount * 1; i < kSpecialCount * 6; ++i) {
-    Insert(i, i * 2, kNotSpecialCharge);
-  }
-
-  apply_thread.join();
-  ASSERT_EQ(special_count, kSpecialCount);
+  ASSERT_TRUE(inserted == callback_state);
 }
 
 TEST_P(CacheTest, DefaultShardBits) {
@@ -822,12 +747,11 @@ TEST_P(CacheTest, DefaultShardBits) {
   ASSERT_EQ(6, sc->GetNumShardBits());
 }
 
-TEST_P(CacheTest, GetChargeAndDeleter) {
+TEST_P(CacheTest, GetCharge) {
   Insert(1, 2);
   Cache::Handle* h1 = cache_->Lookup(EncodeKey(1));
   ASSERT_EQ(2, DecodeValue(cache_->Value(h1)));
   ASSERT_EQ(1, cache_->GetCharge(h1));
-  ASSERT_EQ(&CacheTest::Deleter, cache_->GetDeleter(h1));
   cache_->Release(h1);
 }
 

cache/clock_cache.cc

@@ -176,16 +176,13 @@ namespace {
 // Cache entry meta data.
 struct CacheHandle {
   Slice key;
+  uint32_t hash;
   void* value;
   size_t charge;
-  Cache::DeleterFn deleter;
-  uint32_t hash;
-
-  // Addition to "charge" to get "total charge" under metadata policy.
-  uint32_t meta_charge;
+  void (*deleter)(const Slice&, void* value);
 
   // Flags and counters associated with the cache handle:
-  //   lowest bit: in-cache bit
+  //   lowest bit: n-cache bit
   //   second lowest bit: usage bit
   //   the rest bits: reference count
   // The handle is unused when flags equals to 0. The thread decreases the count
@@ -208,8 +205,9 @@ struct CacheHandle {
     return *this;
   }
 
-  inline static uint32_t CalcMetadataCharge(
-      Slice key, CacheMetadataChargePolicy metadata_charge_policy) {
+  inline static size_t CalcTotalCharge(
+      Slice key, size_t charge,
+      CacheMetadataChargePolicy metadata_charge_policy) {
     size_t meta_charge = 0;
     if (metadata_charge_policy == kFullChargeCacheMetadata) {
       meta_charge += sizeof(CacheHandle);
@@ -220,11 +218,13 @@ struct CacheHandle {
       meta_charge += key.size();
 #endif
     }
-    assert(meta_charge <= UINT32_MAX);
-    return static_cast<uint32_t>(meta_charge);
+    return charge + meta_charge;
   }
 
-  inline size_t GetTotalCharge() { return charge + meta_charge; }
+  inline size_t CalcTotalCharge(
+      CacheMetadataChargePolicy metadata_charge_policy) {
+    return CalcTotalCharge(key, charge, metadata_charge_policy);
+  }
 };
 
 // Key of hash map. We store hash value with the key for convenience.
@@ -271,25 +271,7 @@ class ClockCacheShard final : public CacheShard {
   Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
                 void (*deleter)(const Slice& key, void* value),
                 Cache::Handle** handle, Cache::Priority priority) override;
-  Status Insert(const Slice& key, uint32_t hash, void* value,
-                const Cache::CacheItemHelper* helper, size_t charge,
-                Cache::Handle** handle, Cache::Priority priority) override {
-    return Insert(key, hash, value, charge, helper->del_cb, handle, priority);
-  }
   Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
-  Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                        const Cache::CacheItemHelper* /*helper*/,
-                        const Cache::CreateCallback& /*create_cb*/,
-                        Cache::Priority /*priority*/, bool /*wait*/) override {
-    return Lookup(key, hash);
-  }
-  bool Release(Cache::Handle* handle, bool /*useful*/,
-               bool force_erase) override {
-    return Release(handle, force_erase);
-  }
-  bool IsReady(Cache::Handle* /*handle*/) override { return true; }
-  void Wait(Cache::Handle* /*handle*/) override {}
-
   // If the entry in in cache, increase reference count and return true.
   // Return false otherwise.
   //
@@ -302,10 +284,8 @@ class ClockCacheShard final : public CacheShard {
   size_t GetUsage() const override;
   size_t GetPinnedUsage() const override;
   void EraseUnRefEntries() override;
-  void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) override;
+  void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                              bool thread_safe) override;
 
  private:
   static const uint32_t kInCacheBit = 1;
@@ -361,8 +341,7 @@ class ClockCacheShard final : public CacheShard {
   CacheHandle* Insert(const Slice& key, uint32_t hash, void* value,
                       size_t change,
                       void (*deleter)(const Slice& key, void* value),
-                      bool hold_reference, CleanupContext* context,
-                      bool* overwritten);
+                      bool hold_reference, CleanupContext* context);
 
   // Guards list_, head_, and recycle_. In addition, updating table_ also has
   // to hold the mutex, to avoid the cache being in inconsistent state.
@@ -424,46 +403,22 @@ size_t ClockCacheShard::GetPinnedUsage() const {
   return pinned_usage_.load(std::memory_order_relaxed);
 }
 
-void ClockCacheShard::ApplyToSomeEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
-    uint32_t average_entries_per_lock, uint32_t* state) {
-  assert(average_entries_per_lock > 0);
-  MutexLock lock(&mutex_);
-
-  // Figure out the range to iterate, update `state`
-  size_t list_size = list_.size();
-  size_t start_idx = *state;
-  size_t end_idx = start_idx + average_entries_per_lock;
-  if (start_idx > list_size) {
-    // Shouldn't reach here, but recoverable
-    assert(false);
-    // Mark finished with all
-    *state = UINT32_MAX;
-    return;
+void ClockCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                             bool thread_safe) {
+  if (thread_safe) {
+    mutex_.Lock();
   }
-  if (end_idx >= list_size || end_idx >= UINT32_MAX) {
-    // This also includes the hypothetical case of >4 billion
-    // cache handles.
-    end_idx = list_size;
-    // Mark finished with all
-    *state = UINT32_MAX;
-  } else {
-    *state = static_cast<uint32_t>(end_idx);
-  }
-
-  // Do the iteration
-  auto cur = list_.begin() + start_idx;
-  auto end = list_.begin() + end_idx;
-  for (; cur != end; ++cur) {
-    const CacheHandle& handle = *cur;
-    // Use relaxed semantics instead of acquire semantics since we are
-    // holding mutex
+  for (auto& handle : list_) {
+    // Use relaxed semantics instead of acquire semantics since we are either
+    // holding mutex, or don't have thread safe requirement.
     uint32_t flags = handle.flags.load(std::memory_order_relaxed);
     if (InCache(flags)) {
-      callback(handle.key, handle.value, handle.charge, handle.deleter);
+      callback(handle.value, handle.charge);
     }
   }
+  if (thread_safe) {
+    mutex_.Unlock();
+  }
 }
 
 void ClockCacheShard::RecycleHandle(CacheHandle* handle,
@@ -472,8 +427,10 @@ void ClockCacheShard::RecycleHandle(CacheHandle* handle,
   assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
   context->to_delete_key.push_back(handle->key.data());
   context->to_delete_value.emplace_back(*handle);
-  size_t total_charge = handle->GetTotalCharge();
-  // clearing `handle` fields would go here but not strictly required
+  size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
+  handle->key.clear();
+  handle->value = nullptr;
+  handle->deleter = nullptr;
   recycle_.push_back(handle);
   usage_.fetch_sub(total_charge, std::memory_order_relaxed);
 }
@@ -501,7 +458,7 @@ bool ClockCacheShard::Ref(Cache::Handle* h) {
                                             std::memory_order_relaxed)) {
       if (CountRefs(flags) == 0) {
         // No reference count before the operation.
-        size_t total_charge = handle->GetTotalCharge();
+        size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
         pinned_usage_.fetch_add(total_charge, std::memory_order_relaxed);
       }
       return true;
@@ -515,11 +472,6 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
   if (set_usage) {
     handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
   }
-  // If the handle reaches state refs=0 and InCache=true after this
-  // atomic operation then we cannot access `handle` afterward, because
-  // it could be evicted before we access the `handle`.
-  size_t total_charge = handle->GetTotalCharge();
-
   // Use acquire-release semantics as previous operations on the cache entry
   // has to be order before reference count is decreased, and potential cleanup
   // of the entry has to be order after.
@@ -527,6 +479,7 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
   assert(CountRefs(flags) > 0);
   if (CountRefs(flags) == 1) {
     // this is the last reference.
+    size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
     pinned_usage_.fetch_sub(total_charge, std::memory_order_relaxed);
     // Cleanup if it is the last reference.
     if (!InCache(flags)) {
@@ -611,11 +564,9 @@ void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
 CacheHandle* ClockCacheShard::Insert(
     const Slice& key, uint32_t hash, void* value, size_t charge,
     void (*deleter)(const Slice& key, void* value), bool hold_reference,
-    CleanupContext* context, bool* overwritten) {
-  assert(overwritten != nullptr && *overwritten == false);
-  uint32_t meta_charge =
-      CacheHandle::CalcMetadataCharge(key, metadata_charge_policy_);
-  size_t total_charge = charge + meta_charge;
+    CleanupContext* context) {
+  size_t total_charge =
+      CacheHandle::CalcTotalCharge(key, charge, metadata_charge_policy_);
   MutexLock l(&mutex_);
   bool success = EvictFromCache(total_charge, context);
   bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
@@ -641,22 +592,11 @@ CacheHandle* ClockCacheShard::Insert(
   handle->hash = hash;
   handle->value = value;
   handle->charge = charge;
-  handle->meta_charge = meta_charge;
   handle->deleter = deleter;
   uint32_t flags = hold_reference ? kInCacheBit + kOneRef : kInCacheBit;
-
-  // TODO investigate+fix suspected race condition:
-  // [thread 1] Lookup starts, up to Ref()
-  // [thread 2] Erase/evict the entry just looked up
-  // [thread 1] Ref() the handle, even though it's in the recycle bin
-  // [thread 2] Insert with recycling that handle
-  // Here we obliterate the other thread's Ref
-  // Possible fix: never blindly overwrite the flags, but only make
-  // relative updates (fetch_add, etc).
   handle->flags.store(flags, std::memory_order_relaxed);
   HashTable::accessor accessor;
   if (table_.find(accessor, CacheKey(key, hash))) {
-    *overwritten = true;
     CacheHandle* existing_handle = accessor->second;
     table_.erase(accessor);
     UnsetInCache(existing_handle, context);
@@ -679,9 +619,8 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   char* key_data = new char[key.size()];
   memcpy(key_data, key.data(), key.size());
   Slice key_copy(key_data, key.size());
-  bool overwritten = false;
   CacheHandle* handle = Insert(key_copy, hash, value, charge, deleter,
-                               out_handle != nullptr, &context, &overwritten);
+                               out_handle != nullptr, &context);
   Status s;
   if (out_handle != nullptr) {
     if (handle == nullptr) {
@@ -690,10 +629,6 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
       *out_handle = reinterpret_cast<Cache::Handle*>(handle);
     }
   }
-  if (overwritten) {
-    assert(s.ok());
-    s = Status::OkOverwritten();
-  }
   Cleanup(context);
   return s;
 }
@@ -783,11 +718,11 @@ class ClockCache final : public ShardedCache {
 
   const char* Name() const override { return "ClockCache"; }
 
-  CacheShard* GetShard(uint32_t shard) override {
+  CacheShard* GetShard(int shard) override {
     return reinterpret_cast<CacheShard*>(&shards_[shard]);
   }
 
-  const CacheShard* GetShard(uint32_t shard) const override {
+  const CacheShard* GetShard(int shard) const override {
     return reinterpret_cast<CacheShard*>(&shards_[shard]);
   }
 
@@ -803,17 +738,7 @@ class ClockCache final : public ShardedCache {
     return reinterpret_cast<const CacheHandle*>(handle)->hash;
   }
 
-  DeleterFn GetDeleter(Handle* handle) const override {
-    return reinterpret_cast<const CacheHandle*>(handle)->deleter;
-  }
-
-  void DisownData() override {
-#ifndef MUST_FREE_HEAP_ALLOCATIONS
-    shards_ = nullptr;
-#endif
-  }
-
-  void WaitAll(std::vector<Handle*>& /*handles*/) override {}
+  void DisownData() override { shards_ = nullptr; }
 
  private:
   ClockCacheShard* shards_;

cache/lru_cache.cc

@@ -9,28 +9,26 @@
 
 #include "cache/lru_cache.h"
 
-#include <cassert>
-#include <cstdint>
-#include <cstdio>
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string>
 
 #include "util/mutexlock.h"
 
 namespace ROCKSDB_NAMESPACE {
 
-LRUHandleTable::LRUHandleTable(int max_upper_hash_bits)
-    : length_bits_(/* historical starting size*/ 4),
-      list_(new LRUHandle* [size_t{1} << length_bits_] {}),
-      elems_(0),
-      max_length_bits_(max_upper_hash_bits) {}
+LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+  Resize();
+}
 
 LRUHandleTable::~LRUHandleTable() {
-  ApplyToEntriesRange(
-      [](LRUHandle* h) {
-        if (!h->HasRefs()) {
-          h->Free();
-        }
-      },
-      0, uint32_t{1} << length_bits_);
+  ApplyToAllCacheEntries([](LRUHandle* h) {
+    if (!h->HasRefs()) {
+      h->Free();
+    }
+  });
+  delete[] list_;
 }
 
 LRUHandle* LRUHandleTable::Lookup(const Slice& key, uint32_t hash) {
@@ -44,7 +42,7 @@ LRUHandle* LRUHandleTable::Insert(LRUHandle* h) {
   *ptr = h;
   if (old == nullptr) {
     ++elems_;
-    if ((elems_ >> length_bits_) > 0) {  // elems_ >= length
+    if (elems_ > length_) {
       // Since each cache entry is fairly large, we aim for a small
       // average linked list length (<= 1).
       Resize();
@@ -64,7 +62,7 @@ LRUHandle* LRUHandleTable::Remove(const Slice& key, uint32_t hash) {
 }
 
 LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
-  LRUHandle** ptr = &list_[hash >> (32 - length_bits_)];
+  LRUHandle** ptr = &list_[hash & (length_ - 1)];
   while (*ptr != nullptr && ((*ptr)->hash != hash || key != (*ptr)->key())) {
     ptr = &(*ptr)->next_hash;
   }
@@ -72,29 +70,19 @@ LRUHandle** LRUHandleTable::FindPointer(const Slice& key, uint32_t hash) {
 }
 
 void LRUHandleTable::Resize() {
-  if (length_bits_ >= max_length_bits_) {
-    // Due to reaching limit of hash information, if we made the table
-    // bigger, we would allocate more addresses but only the same
-    // number would be used.
-    return;
+  uint32_t new_length = 16;
+  while (new_length < elems_ * 1.5) {
+    new_length *= 2;
   }
-  if (length_bits_ >= 31) {
-    // Avoid undefined behavior shifting uint32_t by 32
-    return;
-  }
-
-  uint32_t old_length = uint32_t{1} << length_bits_;
-  int new_length_bits = length_bits_ + 1;
-  std::unique_ptr<LRUHandle* []> new_list {
-    new LRUHandle* [size_t{1} << new_length_bits] {}
-  };
+  LRUHandle** new_list = new LRUHandle*[new_length];
+  memset(new_list, 0, sizeof(new_list[0]) * new_length);
   uint32_t count = 0;
-  for (uint32_t i = 0; i < old_length; i++) {
+  for (uint32_t i = 0; i < length_; i++) {
     LRUHandle* h = list_[i];
     while (h != nullptr) {
       LRUHandle* next = h->next_hash;
       uint32_t hash = h->hash;
-      LRUHandle** ptr = &new_list[hash >> (32 - new_length_bits)];
+      LRUHandle** ptr = &new_list[hash & (new_length - 1)];
       h->next_hash = *ptr;
       *ptr = h;
       h = next;
@@ -102,25 +90,23 @@ void LRUHandleTable::Resize() {
     }
   }
   assert(elems_ == count);
-  list_ = std::move(new_list);
-  length_bits_ = new_length_bits;
+  delete[] list_;
+  list_ = new_list;
+  length_ = new_length;
 }
 
-LRUCacheShard::LRUCacheShard(
-    size_t capacity, bool strict_capacity_limit, double high_pri_pool_ratio,
-    bool use_adaptive_mutex, CacheMetadataChargePolicy metadata_charge_policy,
-    int max_upper_hash_bits,
-    const std::shared_ptr<SecondaryCache>& secondary_cache)
+LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
+                             double high_pri_pool_ratio,
+                             bool use_adaptive_mutex,
+                             CacheMetadataChargePolicy metadata_charge_policy)
     : capacity_(0),
       high_pri_pool_usage_(0),
       strict_capacity_limit_(strict_capacity_limit),
       high_pri_pool_ratio_(high_pri_pool_ratio),
       high_pri_pool_capacity_(0),
-      table_(max_upper_hash_bits),
       usage_(0),
       lru_usage_(0),
-      mutex_(use_adaptive_mutex),
-      secondary_cache_(secondary_cache) {
+      mutex_(use_adaptive_mutex) {
   set_metadata_charge_policy(metadata_charge_policy);
   // Make empty circular linked list
   lru_.next = &lru_;
@@ -152,40 +138,19 @@ void LRUCacheShard::EraseUnRefEntries() {
   }
 }
 
-void LRUCacheShard::ApplyToSomeEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
-    uint32_t average_entries_per_lock, uint32_t* state) {
-  // The state is essentially going to be the starting hash, which works
-  // nicely even if we resize between calls because we use upper-most
-  // hash bits for table indexes.
-  MutexLock l(&mutex_);
-  uint32_t length_bits = table_.GetLengthBits();
-  uint32_t length = uint32_t{1} << length_bits;
+void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                           bool thread_safe) {
+  const auto applyCallback = [&]() {
+    table_.ApplyToAllCacheEntries(
+        [callback](LRUHandle* h) { callback(h->value, h->charge); });
+  };
 
-  assert(average_entries_per_lock > 0);
-  // Assuming we are called with same average_entries_per_lock repeatedly,
-  // this simplifies some logic (index_end will not overflow)
-  assert(average_entries_per_lock < length || *state == 0);
-
-  uint32_t index_begin = *state >> (32 - length_bits);
-  uint32_t index_end = index_begin + average_entries_per_lock;
-  if (index_end >= length) {
-    // Going to end
-    index_end = length;
-    *state = UINT32_MAX;
+  if (thread_safe) {
+    MutexLock l(&mutex_);
+    applyCallback();
   } else {
-    *state = index_end << (32 - length_bits);
+    applyCallback();
   }
-
-  table_.ApplyToEntriesRange(
-      [callback](LRUHandle* h) {
-        DeleterFn deleter = h->IsSecondaryCacheCompatible()
-                                ? h->info_.helper->del_cb
-                                : h->info_.deleter;
-        callback(h->key(), h->value, h->charge, deleter);
-      },
-      index_begin, index_end);
 }
 
 void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
@@ -292,14 +257,8 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
     EvictFromLRU(0, &last_reference_list);
   }
 
-  // Try to insert the evicted entries into tiered cache
   // Free the entries outside of mutex for performance reasons
   for (auto entry : last_reference_list) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsPromoted()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
-          .PermitUncheckedError();
-    }
     entry->Free();
   }
 }
@@ -309,174 +268,17 @@ void LRUCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
-Status LRUCacheShard::InsertItem(LRUHandle* e, Cache::Handle** handle,
-                                 bool free_handle_on_fail) {
-  Status s = Status::OK();
-  autovector<LRUHandle*> last_reference_list;
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-
-  {
-    MutexLock l(&mutex_);
-
-    // Free the space following strict LRU policy until enough space
-    // is freed or the lru list is empty
-    EvictFromLRU(total_charge, &last_reference_list);
-
-    if ((usage_ + total_charge) > capacity_ &&
-        (strict_capacity_limit_ || handle == nullptr)) {
-      e->SetInCache(false);
-      if (handle == nullptr) {
-        // Don't insert the entry but still return ok, as if the entry inserted
-        // into cache and get evicted immediately.
-        last_reference_list.push_back(e);
-      } else {
-        if (free_handle_on_fail) {
-          delete[] reinterpret_cast<char*>(e);
-          *handle = nullptr;
-        }
-        s = Status::Incomplete("Insert failed due to LRU cache being full.");
-      }
-    } else {
-      // Insert into the cache. Note that the cache might get larger than its
-      // capacity if not enough space was freed up.
-      LRUHandle* old = table_.Insert(e);
-      usage_ += total_charge;
-      if (old != nullptr) {
-        s = Status::OkOverwritten();
-        assert(old->InCache());
-        old->SetInCache(false);
-        if (!old->HasRefs()) {
-          // old is on LRU because it's in cache and its reference count is 0
-          LRU_Remove(old);
-          size_t old_total_charge =
-              old->CalcTotalCharge(metadata_charge_policy_);
-          assert(usage_ >= old_total_charge);
-          usage_ -= old_total_charge;
-          last_reference_list.push_back(old);
-        }
-      }
-      if (handle == nullptr) {
-        LRU_Insert(e);
-      } else {
-        // If caller already holds a ref, no need to take one here
-        if (!e->HasRefs()) {
-          e->Ref();
-        }
-        *handle = reinterpret_cast<Cache::Handle*>(e);
-      }
-    }
-  }
-
-  // Try to insert the evicted entries into the secondary cache
-  // Free the entries here outside of mutex for performance reasons
-  for (auto entry : last_reference_list) {
-    if (secondary_cache_ && entry->IsSecondaryCacheCompatible() &&
-        !entry->IsPromoted()) {
-      secondary_cache_->Insert(entry->key(), entry->value, entry->info_.helper)
-          .PermitUncheckedError();
-    }
-    entry->Free();
-  }
-
-  return s;
-}
-
-void LRUCacheShard::Promote(LRUHandle* e) {
-  SecondaryCacheResultHandle* secondary_handle = e->sec_handle;
-
-  assert(secondary_handle->IsReady());
-  e->SetIncomplete(false);
-  e->SetInCache(true);
-  e->SetPromoted(true);
-  e->value = secondary_handle->Value();
-  e->charge = secondary_handle->Size();
-  delete secondary_handle;
-
-  // This call could fail if the cache is over capacity and
-  // strict_capacity_limit_ is true. In such a case, we don't want
-  // InsertItem() to free the handle, since the item is already in memory
-  // and the caller will most likely just read from disk if we erase it here.
-  if (e->value) {
-    Cache::Handle* handle = reinterpret_cast<Cache::Handle*>(e);
-    Status s = InsertItem(e, &handle, /*free_handle_on_fail=*/false);
-    if (!s.ok()) {
-      // Item is in memory, but not accounted against the cache capacity.
-      // When the handle is released, the item should get deleted
-      assert(!e->InCache());
-    }
-  } else {
-    // Since the secondary cache lookup failed, mark the item as not in cache
-    // Don't charge the cache as its only metadata that'll shortly be released
-    MutexLock l(&mutex_);
-    e->charge = 0;
-    e->SetInCache(false);
-  }
-}
-
-Cache::Handle* LRUCacheShard::Lookup(
-    const Slice& key, uint32_t hash,
-    const ShardedCache::CacheItemHelper* helper,
-    const ShardedCache::CreateCallback& create_cb, Cache::Priority priority,
-    bool wait) {
-  LRUHandle* e = nullptr;
-  {
-    MutexLock l(&mutex_);
-    e = table_.Lookup(key, hash);
-    if (e != nullptr) {
-      assert(e->InCache());
-      if (!e->HasRefs()) {
-        // The entry is in LRU since it's in hash and has no external references
-        LRU_Remove(e);
-      }
-      e->Ref();
-      e->SetHit();
-    }
-  }
-
-  // If handle table lookup failed, then allocate a handle outside the
-  // mutex if we're going to lookup in the secondary cache
-  // Only support synchronous for now
-  // TODO: Support asynchronous lookup in secondary cache
-  if (!e && secondary_cache_ && helper && helper->saveto_cb) {
-    // For objects from the secondary cache, we expect the caller to provide
-    // a way to create/delete the primary cache object. The only case where
-    // a deleter would not be required is for dummy entries inserted for
-    // accounting purposes, which we won't demote to the secondary cache
-    // anyway.
-    assert(create_cb && helper->del_cb);
-    std::unique_ptr<SecondaryCacheResultHandle> secondary_handle =
-        secondary_cache_->Lookup(key, create_cb, wait);
-    if (secondary_handle != nullptr) {
-      e = reinterpret_cast<LRUHandle*>(
-          new char[sizeof(LRUHandle) - 1 + key.size()]);
-
-      e->flags = 0;
-      e->SetSecondaryCacheCompatible(true);
-      e->info_.helper = helper;
-      e->key_length = key.size();
-      e->hash = hash;
-      e->refs = 0;
-      e->next = e->prev = nullptr;
-      e->SetPriority(priority);
-      memcpy(e->key_data, key.data(), key.size());
-      e->value = nullptr;
-      e->sec_handle = secondary_handle.release();
-      e->Ref();
-
-      if (wait) {
-        Promote(e);
-        if (!e->value) {
-          // The secondary cache returned a handle, but the lookup failed
-          e->Unref();
-          e->Free();
-          e = nullptr;
-        }
-      } else {
-        // If wait is false, we always return a handle and let the caller
-        // release the handle after checking for success or failure
-        e->SetIncomplete(true);
-      }
+Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
+  MutexLock l(&mutex_);
+  LRUHandle* e = table_.Lookup(key, hash);
+  if (e != nullptr) {
+    assert(e->InCache());
+    if (!e->HasRefs()) {
+      // The entry is in LRU since it's in hash and has no external references
+      LRU_Remove(e);
     }
+    e->Ref();
+    e->SetHit();
   }
   return reinterpret_cast<Cache::Handle*>(e);
 }
@@ -520,12 +322,7 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
         last_reference = false;
       }
     }
-    // If it was the last reference, and the entry is either not secondary
-    // cache compatible (i.e a dummy entry for accounting), or is secondary
-    // cache compatible and has a non-null value, then decrement the cache
-    // usage. If value is null in the latter case, taht means the lookup
-    // failed and we didn't charge the cache.
-    if (last_reference && (!e->IsSecondaryCacheCompatible() || e->value)) {
+    if (last_reference) {
       size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
       assert(usage_ >= total_charge);
       usage_ -= total_charge;
@@ -542,35 +339,80 @@ bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
 Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                              size_t charge,
                              void (*deleter)(const Slice& key, void* value),
-                             const Cache::CacheItemHelper* helper,
                              Cache::Handle** handle, Cache::Priority priority) {
   // Allocate the memory here outside of the mutex
   // If the cache is full, we'll have to release it
   // It shouldn't happen very often though.
   LRUHandle* e = reinterpret_cast<LRUHandle*>(
       new char[sizeof(LRUHandle) - 1 + key.size()]);
+  Status s = Status::OK();
+  autovector<LRUHandle*> last_reference_list;
 
   e->value = value;
-  e->flags = 0;
-  if (helper) {
-    e->SetSecondaryCacheCompatible(true);
-    e->info_.helper = helper;
-  } else {
-#ifdef __SANITIZE_THREAD__
-    e->is_secondary_cache_compatible_for_tsan = false;
-#endif  // __SANITIZE_THREAD__
-    e->info_.deleter = deleter;
-  }
+  e->deleter = deleter;
   e->charge = charge;
   e->key_length = key.size();
+  e->flags = 0;
   e->hash = hash;
   e->refs = 0;
   e->next = e->prev = nullptr;
   e->SetInCache(true);
   e->SetPriority(priority);
   memcpy(e->key_data, key.data(), key.size());
+  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
 
-  return InsertItem(e, handle, /* free_handle_on_fail */ true);
+  {
+    MutexLock l(&mutex_);
+
+    // Free the space following strict LRU policy until enough space
+    // is freed or the lru list is empty
+    EvictFromLRU(total_charge, &last_reference_list);
+
+    if ((usage_ + total_charge) > capacity_ &&
+        (strict_capacity_limit_ || handle == nullptr)) {
+      if (handle == nullptr) {
+        // Don't insert the entry but still return ok, as if the entry inserted
+        // into cache and get evicted immediately.
+        e->SetInCache(false);
+        last_reference_list.push_back(e);
+      } else {
+        delete[] reinterpret_cast<char*>(e);
+        *handle = nullptr;
+        s = Status::Incomplete("Insert failed due to LRU cache being full.");
+      }
+    } else {
+      // Insert into the cache. Note that the cache might get larger than its
+      // capacity if not enough space was freed up.
+      LRUHandle* old = table_.Insert(e);
+      usage_ += total_charge;
+      if (old != nullptr) {
+        assert(old->InCache());
+        old->SetInCache(false);
+        if (!old->HasRefs()) {
+          // old is on LRU because it's in cache and its reference count is 0
+          LRU_Remove(old);
+          size_t old_total_charge =
+              old->CalcTotalCharge(metadata_charge_policy_);
+          assert(usage_ >= old_total_charge);
+          usage_ -= old_total_charge;
+          last_reference_list.push_back(old);
+        }
+      }
+      if (handle == nullptr) {
+        LRU_Insert(e);
+      } else {
+        e->Ref();
+        *handle = reinterpret_cast<Cache::Handle*>(e);
+      }
+    }
+  }
+
+  // Free the entries here outside of mutex for performance reasons
+  for (auto entry : last_reference_list) {
+    entry->Free();
+  }
+
+  return s;
 }
 
 void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
@@ -600,18 +442,6 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
   }
 }
 
-bool LRUCacheShard::IsReady(Cache::Handle* handle) {
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
-  MutexLock l(&mutex_);
-  bool ready = true;
-  if (e->IsPending()) {
-    assert(secondary_cache_);
-    assert(e->sec_handle);
-    ready = e->sec_handle->IsReady();
-  }
-  return ready;
-}
-
 size_t LRUCacheShard::GetUsage() const {
   MutexLock l(&mutex_);
   return usage_;
@@ -638,8 +468,7 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
                    bool strict_capacity_limit, double high_pri_pool_ratio,
                    std::shared_ptr<MemoryAllocator> allocator,
                    bool use_adaptive_mutex,
-                   CacheMetadataChargePolicy metadata_charge_policy,
-                   const std::shared_ptr<SecondaryCache>& secondary_cache)
+                   CacheMetadataChargePolicy metadata_charge_policy)
     : ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
                    std::move(allocator)) {
   num_shards_ = 1 << num_shard_bits;
@@ -647,12 +476,10 @@ LRUCache::LRUCache(size_t capacity, int num_shard_bits,
       port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
   size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
   for (int i = 0; i < num_shards_; i++) {
-    new (&shards_[i]) LRUCacheShard(
-        per_shard, strict_capacity_limit, high_pri_pool_ratio,
-        use_adaptive_mutex, metadata_charge_policy,
-        /* max_upper_hash_bits */ 32 - num_shard_bits, secondary_cache);
+    new (&shards_[i])
+        LRUCacheShard(per_shard, strict_capacity_limit, high_pri_pool_ratio,
+                      use_adaptive_mutex, metadata_charge_policy);
   }
-  secondary_cache_ = secondary_cache;
 }
 
 LRUCache::~LRUCache() {
@@ -665,11 +492,11 @@ LRUCache::~LRUCache() {
   }
 }
 
-CacheShard* LRUCache::GetShard(uint32_t shard) {
+CacheShard* LRUCache::GetShard(int shard) {
   return reinterpret_cast<CacheShard*>(&shards_[shard]);
 }
 
-const CacheShard* LRUCache::GetShard(uint32_t shard) const {
+const CacheShard* LRUCache::GetShard(int shard) const {
   return reinterpret_cast<CacheShard*>(&shards_[shard]);
 }
 
@@ -681,25 +508,23 @@ size_t LRUCache::GetCharge(Handle* handle) const {
   return reinterpret_cast<const LRUHandle*>(handle)->charge;
 }
 
-Cache::DeleterFn LRUCache::GetDeleter(Handle* handle) const {
-  auto h = reinterpret_cast<const LRUHandle*>(handle);
-  if (h->IsSecondaryCacheCompatible()) {
-    return h->info_.helper->del_cb;
-  } else {
-    return h->info_.deleter;
-  }
-}
-
 uint32_t LRUCache::GetHash(Handle* handle) const {
   return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }
 
 void LRUCache::DisownData() {
 // Do not drop data if compile with ASAN to suppress leak warning.
-#ifndef MUST_FREE_HEAP_ALLOCATIONS
+#if defined(__clang__)
+#if !defined(__has_feature) || !__has_feature(address_sanitizer)
   shards_ = nullptr;
   num_shards_ = 0;
 #endif
+#else  // __clang__
+#ifndef __SANITIZE_ADDRESS__
+  shards_ = nullptr;
+  num_shards_ = 0;
+#endif  // !__SANITIZE_ADDRESS__
+#endif  // __clang__
 }
 
 size_t LRUCache::TEST_GetLRUSize() {
@@ -718,42 +543,19 @@ double LRUCache::GetHighPriPoolRatio() {
   return result;
 }
 
-void LRUCache::WaitAll(std::vector<Handle*>& handles) {
-  if (secondary_cache_) {
-    std::vector<SecondaryCacheResultHandle*> sec_handles;
-    sec_handles.reserve(handles.size());
-    for (Handle* handle : handles) {
-      if (!handle) {
-        continue;
-      }
-      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
-      if (!lru_handle->IsPending()) {
-        continue;
-      }
-      sec_handles.emplace_back(lru_handle->sec_handle);
-    }
-    secondary_cache_->WaitAll(sec_handles);
-    for (Handle* handle : handles) {
-      if (!handle) {
-        continue;
-      }
-      LRUHandle* lru_handle = reinterpret_cast<LRUHandle*>(handle);
-      if (!lru_handle->IsPending()) {
-        continue;
-      }
-      uint32_t hash = GetHash(handle);
-      LRUCacheShard* shard = static_cast<LRUCacheShard*>(GetShard(Shard(hash)));
-      shard->Promote(lru_handle);
-    }
-  }
+std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
+  return NewLRUCache(cache_opts.capacity, cache_opts.num_shard_bits,
+                     cache_opts.strict_capacity_limit,
+                     cache_opts.high_pri_pool_ratio,
+                     cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
+                     cache_opts.metadata_charge_policy);
 }
 
 std::shared_ptr<Cache> NewLRUCache(
     size_t capacity, int num_shard_bits, bool strict_capacity_limit,
     double high_pri_pool_ratio,
     std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
-    CacheMetadataChargePolicy metadata_charge_policy,
-    const std::shared_ptr<SecondaryCache>& secondary_cache) {
+    CacheMetadataChargePolicy metadata_charge_policy) {
   if (num_shard_bits >= 20) {
     return nullptr;  // the cache cannot be sharded into too many fine pieces
   }
@@ -766,25 +568,7 @@ std::shared_ptr<Cache> NewLRUCache(
   }
   return std::make_shared<LRUCache>(
       capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
-      std::move(memory_allocator), use_adaptive_mutex, metadata_charge_policy,
-      secondary_cache);
+      std::move(memory_allocator), use_adaptive_mutex, metadata_charge_policy);
 }
 
-std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
-  return NewLRUCache(
-      cache_opts.capacity, cache_opts.num_shard_bits,
-      cache_opts.strict_capacity_limit, cache_opts.high_pri_pool_ratio,
-      cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
-      cache_opts.metadata_charge_policy, cache_opts.secondary_cache);
-}
-
-std::shared_ptr<Cache> NewLRUCache(
-    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    double high_pri_pool_ratio,
-    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
-    CacheMetadataChargePolicy metadata_charge_policy) {
-  return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit,
-                     high_pri_pool_ratio, memory_allocator, use_adaptive_mutex,
-                     metadata_charge_policy, nullptr);
-}
 }  // namespace ROCKSDB_NAMESPACE

cache/lru_cache.h

@@ -1,4 +1,4 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved
+//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
 //  This source code is licensed under both the GPLv2 (found in the
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
@@ -8,14 +8,12 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 #pragma once
 
-#include <memory>
 #include <string>
 
 #include "cache/sharded_cache.h"
-#include "port/lang.h"
+
 #include "port/malloc.h"
 #include "port/port.h"
-#include "rocksdb/secondary_cache.h"
 #include "util/autovector.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -51,18 +49,8 @@ namespace ROCKSDB_NAMESPACE {
 
 struct LRUHandle {
   void* value;
-  union Info {
-    Info() {}
-    ~Info() {}
-    Cache::DeleterFn deleter;
-    const ShardedCache::CacheItemHelper* helper;
-  } info_;
-  // An entry is not added to the LRUHandleTable until the secondary cache
-  // lookup is complete, so its safe to have this union.
-  union {
-    LRUHandle* next_hash;
-    SecondaryCacheResultHandle* sec_handle;
-  };
+  void (*deleter)(const Slice&, void* value);
+  LRUHandle* next_hash;
   LRUHandle* next;
   LRUHandle* prev;
   size_t charge;  // TODO(opt): Only allow uint32_t?
@@ -79,26 +67,12 @@ struct LRUHandle {
     IS_HIGH_PRI = (1 << 1),
     // Whether this entry is in high-pri pool.
     IN_HIGH_PRI_POOL = (1 << 2),
-    // Whether this entry has had any lookups (hits).
+    // Wwhether this entry has had any lookups (hits).
     HAS_HIT = (1 << 3),
-    // Can this be inserted into the secondary cache
-    IS_SECONDARY_CACHE_COMPATIBLE = (1 << 4),
-    // Is the handle still being read from a lower tier
-    IS_PENDING = (1 << 5),
-    // Has the item been promoted from a lower tier
-    IS_PROMOTED = (1 << 6),
   };
 
   uint8_t flags;
 
-#ifdef __SANITIZE_THREAD__
-  // TSAN can report a false data race on flags, where one thread is writing
-  // to one of the mutable bits and another thread is reading this immutable
-  // bit. So precisely suppress that TSAN warning, we separate out this bit
-  // during TSAN runs.
-  bool is_secondary_cache_compatible_for_tsan;
-#endif  // __SANITIZE_THREAD__
-
   // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
   char key_data[1];
 
@@ -121,15 +95,6 @@ struct LRUHandle {
   bool IsHighPri() const { return flags & IS_HIGH_PRI; }
   bool InHighPriPool() const { return flags & IN_HIGH_PRI_POOL; }
   bool HasHit() const { return flags & HAS_HIT; }
-  bool IsSecondaryCacheCompatible() const {
-#ifdef __SANITIZE_THREAD__
-    return is_secondary_cache_compatible_for_tsan;
-#else
-    return flags & IS_SECONDARY_CACHE_COMPATIBLE;
-#endif  // __SANITIZE_THREAD__
-  }
-  bool IsPending() const { return flags & IS_PENDING; }
-  bool IsPromoted() const { return flags & IS_PROMOTED; }
 
   void SetInCache(bool in_cache) {
     if (in_cache) {
@@ -157,58 +122,15 @@ struct LRUHandle {
 
   void SetHit() { flags |= HAS_HIT; }
 
-  void SetSecondaryCacheCompatible(bool compat) {
-    if (compat) {
-      flags |= IS_SECONDARY_CACHE_COMPATIBLE;
-    } else {
-      flags &= ~IS_SECONDARY_CACHE_COMPATIBLE;
-    }
-#ifdef __SANITIZE_THREAD__
-    is_secondary_cache_compatible_for_tsan = compat;
-#endif  // __SANITIZE_THREAD__
-  }
-
-  void SetIncomplete(bool incomp) {
-    if (incomp) {
-      flags |= IS_PENDING;
-    } else {
-      flags &= ~IS_PENDING;
-    }
-  }
-
-  void SetPromoted(bool promoted) {
-    if (promoted) {
-      flags |= IS_PROMOTED;
-    } else {
-      flags &= ~IS_PROMOTED;
-    }
-  }
-
   void Free() {
     assert(refs == 0);
-#ifdef __SANITIZE_THREAD__
-    // Here we can safely assert they are the same without a data race reported
-    assert(((flags & IS_SECONDARY_CACHE_COMPATIBLE) != 0) ==
-           is_secondary_cache_compatible_for_tsan);
-#endif  // __SANITIZE_THREAD__
-    if (!IsSecondaryCacheCompatible() && info_.deleter) {
-      (*info_.deleter)(key(), value);
-    } else if (IsSecondaryCacheCompatible()) {
-      if (IsPending()) {
-        assert(sec_handle != nullptr);
-        SecondaryCacheResultHandle* tmp_sec_handle = sec_handle;
-        tmp_sec_handle->Wait();
-        value = tmp_sec_handle->Value();
-        delete tmp_sec_handle;
-      }
-      if (value) {
-        (*info_.helper->del_cb)(key(), value);
-      }
+    if (deleter) {
+      (*deleter)(key(), value);
     }
     delete[] reinterpret_cast<char*>(this);
   }
 
-  // Calculate the memory usage by metadata
+  // Caclculate the memory usage by metadata
   inline size_t CalcTotalCharge(
       CacheMetadataChargePolicy metadata_charge_policy) {
     size_t meta_charge = 0;
@@ -231,10 +153,7 @@ struct LRUHandle {
 // 4.4.3's builtin hashtable.
 class LRUHandleTable {
  public:
-  // If the table uses more hash bits than `max_upper_hash_bits`,
-  // it will eat into the bits used for sharding, which are constant
-  // for a given LRUHandleTable.
-  explicit LRUHandleTable(int max_upper_hash_bits);
+  LRUHandleTable();
   ~LRUHandleTable();
 
   LRUHandle* Lookup(const Slice& key, uint32_t hash);
@@ -242,8 +161,8 @@ class LRUHandleTable {
   LRUHandle* Remove(const Slice& key, uint32_t hash);
 
   template <typename T>
-  void ApplyToEntriesRange(T func, uint32_t index_begin, uint32_t index_end) {
-    for (uint32_t i = index_begin; i < index_end; i++) {
+  void ApplyToAllCacheEntries(T func) {
+    for (uint32_t i = 0; i < length_; i++) {
       LRUHandle* h = list_[i];
       while (h != nullptr) {
         auto n = h->next_hash;
@@ -254,8 +173,6 @@ class LRUHandleTable {
     }
   }
 
-  int GetLengthBits() const { return length_bits_; }
-
  private:
   // Return a pointer to slot that points to a cache entry that
   // matches key/hash.  If there is no such cache entry, return a
@@ -264,19 +181,11 @@ class LRUHandleTable {
 
   void Resize();
 
-  // Number of hash bits (upper because lower bits used for sharding)
-  // used for table index. Length == 1 << length_bits_
-  int length_bits_;
-
   // The table consists of an array of buckets where each bucket is
   // a linked list of cache entries that hash into the bucket.
-  std::unique_ptr<LRUHandle*[]> list_;
-
-  // Number of elements currently in the table
+  LRUHandle** list_;
+  uint32_t length_;
   uint32_t elems_;
-
-  // Set from max_upper_hash_bits (see constructor)
-  const int max_length_bits_;
 };
 
 // A single shard of sharded cache.
@@ -284,9 +193,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
  public:
   LRUCacheShard(size_t capacity, bool strict_capacity_limit,
                 double high_pri_pool_ratio, bool use_adaptive_mutex,
-                CacheMetadataChargePolicy metadata_charge_policy,
-                int max_upper_hash_bits,
-                const std::shared_ptr<SecondaryCache>& secondary_cache);
+                CacheMetadataChargePolicy metadata_charge_policy);
   virtual ~LRUCacheShard() override = default;
 
   // Separate from constructor so caller can easily make an array of LRUCache
@@ -302,34 +209,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   // Like Cache methods, but with an extra "hash" parameter.
   virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge, Cache::DeleterFn deleter,
+                        size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
                         Cache::Handle** handle,
-                        Cache::Priority priority) override {
-    return Insert(key, hash, value, charge, deleter, nullptr, handle, priority);
-  }
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        const Cache::CacheItemHelper* helper, size_t charge,
-                        Cache::Handle** handle,
-                        Cache::Priority priority) override {
-    assert(helper);
-    return Insert(key, hash, value, charge, nullptr, helper, handle, priority);
-  }
-  // If helper_cb is null, the values of the following arguments don't
-  // matter
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                                const ShardedCache::CacheItemHelper* helper,
-                                const ShardedCache::CreateCallback& create_cb,
-                                ShardedCache::Priority priority,
-                                bool wait) override;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override {
-    return Lookup(key, hash, nullptr, nullptr, Cache::Priority::LOW, true);
-  }
-  virtual bool Release(Cache::Handle* handle, bool /*useful*/,
-                       bool force_erase) override {
-    return Release(handle, force_erase);
-  }
-  virtual bool IsReady(Cache::Handle* /*handle*/) override;
-  virtual void Wait(Cache::Handle* /*handle*/) override {}
+                        Cache::Priority priority) override;
+  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
   virtual bool Ref(Cache::Handle* handle) override;
   virtual bool Release(Cache::Handle* handle,
                        bool force_erase = false) override;
@@ -342,10 +226,8 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   virtual size_t GetUsage() const override;
   virtual size_t GetPinnedUsage() const override;
 
-  virtual void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) override;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) override;
 
   virtual void EraseUnRefEntries() override;
 
@@ -357,27 +239,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   //  not threadsafe
   size_t TEST_GetLRUSize();
 
-  //  Retrieves high pri pool ratio
+  //  Retrives high pri pool ratio
   double GetHighPriPoolRatio();
 
  private:
-  friend class LRUCache;
-  // Insert an item into the hash table and, if handle is null, insert into
-  // the LRU list. Older items are evicted as necessary. If the cache is full
-  // and free_handle_on_fail is true, the item is deleted and handle is set to.
-  Status InsertItem(LRUHandle* item, Cache::Handle** handle,
-                    bool free_handle_on_fail);
-  Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                DeleterFn deleter, const Cache::CacheItemHelper* helper,
-                Cache::Handle** handle, Cache::Priority priority);
-  // Promote an item looked up from the secondary cache to the LRU cache. The
-  // item is only inserted into the hash table and not the LRU list, and only
-  // if the cache is not at full capacity, as is the case during Insert.  The
-  // caller should hold a reference on the LRUHandle. When the caller releases
-  // the last reference, the item is added to the LRU list.
-  // The item is promoted to the high pri or low pri pool as specified by the
-  // caller in Lookup.
-  void Promote(LRUHandle* e);
   void LRU_Remove(LRUHandle* e);
   void LRU_Insert(LRUHandle* e);
 
@@ -438,8 +303,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // We don't count mutex_ as the cache's internal state so semantically we
   // don't mind mutex_ invoking the non-const actions.
   mutable port::Mutex mutex_;
-
-  std::shared_ptr<SecondaryCache> secondary_cache_;
 };
 
 class LRUCache
@@ -453,28 +316,24 @@ class LRUCache
            std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
            bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
            CacheMetadataChargePolicy metadata_charge_policy =
-               kDontChargeCacheMetadata,
-           const std::shared_ptr<SecondaryCache>& secondary_cache = nullptr);
+               kDontChargeCacheMetadata);
   virtual ~LRUCache();
   virtual const char* Name() const override { return "LRUCache"; }
-  virtual CacheShard* GetShard(uint32_t shard) override;
-  virtual const CacheShard* GetShard(uint32_t shard) const override;
+  virtual CacheShard* GetShard(int shard) override;
+  virtual const CacheShard* GetShard(int shard) const override;
   virtual void* Value(Handle* handle) override;
   virtual size_t GetCharge(Handle* handle) const override;
   virtual uint32_t GetHash(Handle* handle) const override;
-  virtual DeleterFn GetDeleter(Handle* handle) const override;
   virtual void DisownData() override;
-  virtual void WaitAll(std::vector<Handle*>& handles) override;
 
   //  Retrieves number of elements in LRU, for unit test purpose only
   size_t TEST_GetLRUSize();
-  //  Retrieves high pri pool ratio
+  //  Retrives high pri pool ratio
   double GetHighPriPoolRatio();
 
  private:
   LRUCacheShard* shards_ = nullptr;
   int num_shards_ = 0;
-  std::shared_ptr<SecondaryCache> secondary_cache_;
 };
 
 }  // namespace ROCKSDB_NAMESPACE

cache/sharded_cache.cc

@@ -9,96 +9,53 @@
 
 #include "cache/sharded_cache.h"
 
-#include <algorithm>
-#include <cstdint>
-#include <memory>
+#include <string>
 
-#include "util/hash.h"
-#include "util/math.h"
 #include "util/mutexlock.h"
 
 namespace ROCKSDB_NAMESPACE {
 
-namespace {
-
-inline uint32_t HashSlice(const Slice& s) {
-  return Lower32of64(GetSliceNPHash64(s));
-}
-
-}  // namespace
-
 ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
                            bool strict_capacity_limit,
                            std::shared_ptr<MemoryAllocator> allocator)
     : Cache(std::move(allocator)),
-      shard_mask_((uint32_t{1} << num_shard_bits) - 1),
+      num_shard_bits_(num_shard_bits),
       capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit),
       last_id_(1) {}
 
 void ShardedCache::SetCapacity(size_t capacity) {
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   const size_t per_shard = (capacity + (num_shards - 1)) / num_shards;
   MutexLock l(&capacity_mutex_);
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     GetShard(s)->SetCapacity(per_shard);
   }
   capacity_ = capacity;
 }
 
 void ShardedCache::SetStrictCapacityLimit(bool strict_capacity_limit) {
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   MutexLock l(&capacity_mutex_);
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     GetShard(s)->SetStrictCapacityLimit(strict_capacity_limit);
   }
   strict_capacity_limit_ = strict_capacity_limit;
 }
 
 Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
-                            DeleterFn deleter, Handle** handle,
-                            Priority priority) {
+                            void (*deleter)(const Slice& key, void* value),
+                            Handle** handle, Priority priority) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))
       ->Insert(key, hash, value, charge, deleter, handle, priority);
 }
 
-Status ShardedCache::Insert(const Slice& key, void* value,
-                            const CacheItemHelper* helper, size_t charge,
-                            Handle** handle, Priority priority) {
-  uint32_t hash = HashSlice(key);
-  if (!helper) {
-    return Status::InvalidArgument();
-  }
-  return GetShard(Shard(hash))
-      ->Insert(key, hash, value, helper, charge, handle, priority);
-}
-
 Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))->Lookup(key, hash);
 }
 
-Cache::Handle* ShardedCache::Lookup(const Slice& key,
-                                    const CacheItemHelper* helper,
-                                    const CreateCallback& create_cb,
-                                    Priority priority, bool wait,
-                                    Statistics* /*stats*/) {
-  uint32_t hash = HashSlice(key);
-  return GetShard(Shard(hash))
-      ->Lookup(key, hash, helper, create_cb, priority, wait);
-}
-
-bool ShardedCache::IsReady(Handle* handle) {
-  uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->IsReady(handle);
-}
-
-void ShardedCache::Wait(Handle* handle) {
-  uint32_t hash = GetHash(handle);
-  GetShard(Shard(hash))->Wait(handle);
-}
-
 bool ShardedCache::Ref(Handle* handle) {
   uint32_t hash = GetHash(handle);
   return GetShard(Shard(hash))->Ref(handle);
@@ -109,11 +66,6 @@ bool ShardedCache::Release(Handle* handle, bool force_erase) {
   return GetShard(Shard(hash))->Release(handle, force_erase);
 }
 
-bool ShardedCache::Release(Handle* handle, bool useful, bool force_erase) {
-  uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->Release(handle, useful, force_erase);
-}
-
 void ShardedCache::Erase(const Slice& key) {
   uint32_t hash = HashSlice(key);
   GetShard(Shard(hash))->Erase(key, hash);
@@ -135,9 +87,9 @@ bool ShardedCache::HasStrictCapacityLimit() const {
 
 size_t ShardedCache::GetUsage() const {
   // We will not lock the cache when getting the usage from shards.
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   size_t usage = 0;
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     usage += GetShard(s)->GetUsage();
   }
   return usage;
@@ -149,42 +101,25 @@ size_t ShardedCache::GetUsage(Handle* handle) const {
 
 size_t ShardedCache::GetPinnedUsage() const {
   // We will not lock the cache when getting the usage from shards.
-  uint32_t num_shards = GetNumShards();
+  int num_shards = 1 << num_shard_bits_;
   size_t usage = 0;
-  for (uint32_t s = 0; s < num_shards; s++) {
+  for (int s = 0; s < num_shards; s++) {
     usage += GetShard(s)->GetPinnedUsage();
   }
   return usage;
 }
 
-void ShardedCache::ApplyToAllEntries(
-    const std::function<void(const Slice& key, void* value, size_t charge,
-                             DeleterFn deleter)>& callback,
-    const ApplyToAllEntriesOptions& opts) {
-  uint32_t num_shards = GetNumShards();
-  // Iterate over part of each shard, rotating between shards, to
-  // minimize impact on latency of concurrent operations.
-  std::unique_ptr<uint32_t[]> states(new uint32_t[num_shards]{});
-
-  uint32_t aepl_in_32 = static_cast<uint32_t>(
-      std::min(size_t{UINT32_MAX}, opts.average_entries_per_lock));
-  aepl_in_32 = std::min(aepl_in_32, uint32_t{1});
-
-  bool remaining_work;
-  do {
-    remaining_work = false;
-    for (uint32_t s = 0; s < num_shards; s++) {
-      if (states[s] != UINT32_MAX) {
-        GetShard(s)->ApplyToSomeEntries(callback, aepl_in_32, &states[s]);
-        remaining_work |= states[s] != UINT32_MAX;
-      }
-    }
-  } while (remaining_work);
+void ShardedCache::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                          bool thread_safe) {
+  int num_shards = 1 << num_shard_bits_;
+  for (int s = 0; s < num_shards; s++) {
+    GetShard(s)->ApplyToAllCacheEntries(callback, thread_safe);
+  }
 }
 
 void ShardedCache::EraseUnRefEntries() {
-  uint32_t num_shards = GetNumShards();
-  for (uint32_t s = 0; s < num_shards; s++) {
+  int num_shards = 1 << num_shard_bits_;
+  for (int s = 0; s < num_shards; s++) {
     GetShard(s)->EraseUnRefEntries();
   }
 }
@@ -199,8 +134,7 @@ std::string ShardedCache::GetPrintableOptions() const {
     snprintf(buffer, kBufferSize, "    capacity : %" ROCKSDB_PRIszt "\n",
              capacity_);
     ret.append(buffer);
-    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n",
-             GetNumShardBits());
+    snprintf(buffer, kBufferSize, "    num_shard_bits : %d\n", num_shard_bits_);
     ret.append(buffer);
     snprintf(buffer, kBufferSize, "    strict_capacity_limit : %d\n",
              strict_capacity_limit_);
@@ -225,8 +159,4 @@ int GetDefaultCacheShardBits(size_t capacity) {
   return num_shard_bits;
 }
 
-int ShardedCache::GetNumShardBits() const { return BitsSetToOne(shard_mask_); }
-
-uint32_t ShardedCache::GetNumShards() const { return shard_mask_ + 1; }
-
 }  // namespace ROCKSDB_NAMESPACE

cache/sharded_cache.h

@@ -14,6 +14,7 @@
 
 #include "port/port.h"
 #include "rocksdb/cache.h"
+#include "util/hash.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -23,37 +24,20 @@ class CacheShard {
   CacheShard() = default;
   virtual ~CacheShard() = default;
 
-  using DeleterFn = Cache::DeleterFn;
   virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        size_t charge, DeleterFn deleter,
-                        Cache::Handle** handle, Cache::Priority priority) = 0;
-  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
-                        const Cache::CacheItemHelper* helper, size_t charge,
+                        size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
                         Cache::Handle** handle, Cache::Priority priority) = 0;
   virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
-  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash,
-                                const Cache::CacheItemHelper* helper,
-                                const Cache::CreateCallback& create_cb,
-                                Cache::Priority priority, bool wait) = 0;
-  virtual bool Release(Cache::Handle* handle, bool useful,
-                       bool force_erase) = 0;
-  virtual bool IsReady(Cache::Handle* handle) = 0;
-  virtual void Wait(Cache::Handle* handle) = 0;
   virtual bool Ref(Cache::Handle* handle) = 0;
-  virtual bool Release(Cache::Handle* handle, bool force_erase) = 0;
+  virtual bool Release(Cache::Handle* handle, bool force_erase = false) = 0;
   virtual void Erase(const Slice& key, uint32_t hash) = 0;
   virtual void SetCapacity(size_t capacity) = 0;
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
   virtual size_t GetUsage() const = 0;
   virtual size_t GetPinnedUsage() const = 0;
-  // Handles iterating over roughly `average_entries_per_lock` entries, using
-  // `state` to somehow record where it last ended up. Caller initially uses
-  // *state == 0 and implementation sets *state = UINT32_MAX to indicate
-  // completion.
-  virtual void ApplyToSomeEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      uint32_t average_entries_per_lock, uint32_t* state) = 0;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) = 0;
   virtual void EraseUnRefEntries() = 0;
   virtual std::string GetPrintableOptions() const { return ""; }
   void set_metadata_charge_policy(
@@ -73,29 +57,22 @@ class ShardedCache : public Cache {
   ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
                std::shared_ptr<MemoryAllocator> memory_allocator = nullptr);
   virtual ~ShardedCache() = default;
-  virtual CacheShard* GetShard(uint32_t shard) = 0;
-  virtual const CacheShard* GetShard(uint32_t shard) const = 0;
+  virtual const char* Name() const override = 0;
+  virtual CacheShard* GetShard(int shard) = 0;
+  virtual const CacheShard* GetShard(int shard) const = 0;
+  virtual void* Value(Handle* handle) override = 0;
+  virtual size_t GetCharge(Handle* handle) const override = 0;
 
   virtual uint32_t GetHash(Handle* handle) const = 0;
+  virtual void DisownData() override = 0;
 
   virtual void SetCapacity(size_t capacity) override;
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
 
   virtual Status Insert(const Slice& key, void* value, size_t charge,
-                        DeleterFn deleter, Handle** handle,
-                        Priority priority) override;
-  virtual Status Insert(const Slice& key, void* value,
-                        const CacheItemHelper* helper, size_t chargge,
-                        Handle** handle = nullptr,
-                        Priority priority = Priority::LOW) override;
+                        void (*deleter)(const Slice& key, void* value),
+                        Handle** handle, Priority priority) override;
   virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
-  virtual Handle* Lookup(const Slice& key, const CacheItemHelper* helper,
-                         const CreateCallback& create_cb, Priority priority,
-                         bool wait, Statistics* stats = nullptr) override;
-  virtual bool Release(Handle* handle, bool useful,
-                       bool force_erase = false) override;
-  virtual bool IsReady(Handle* handle) override;
-  virtual void Wait(Handle* handle) override;
   virtual bool Ref(Handle* handle) override;
   virtual bool Release(Handle* handle, bool force_erase = false) override;
   virtual void Erase(const Slice& key) override;
@@ -105,21 +82,24 @@ class ShardedCache : public Cache {
   virtual size_t GetUsage() const override;
   virtual size_t GetUsage(Handle* handle) const override;
   virtual size_t GetPinnedUsage() const override;
-  virtual void ApplyToAllEntries(
-      const std::function<void(const Slice& key, void* value, size_t charge,
-                               DeleterFn deleter)>& callback,
-      const ApplyToAllEntriesOptions& opts) override;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) override;
   virtual void EraseUnRefEntries() override;
   virtual std::string GetPrintableOptions() const override;
 
-  int GetNumShardBits() const;
-  uint32_t GetNumShards() const;
-
- protected:
-  inline uint32_t Shard(uint32_t hash) { return hash & shard_mask_; }
+  int GetNumShardBits() const { return num_shard_bits_; }
 
  private:
-  const uint32_t shard_mask_;
+  static inline uint32_t HashSlice(const Slice& s) {
+    return static_cast<uint32_t>(GetSliceNPHash64(s));
+  }
+
+  uint32_t Shard(uint32_t hash) {
+    // Note, hash >> 32 yields hash in gcc, not the zero we expect!
+    return (num_shard_bits_ > 0) ? (hash >> (32 - num_shard_bits_)) : 0;
+  }
+
+  int num_shard_bits_;
   mutable port::Mutex capacity_mutex_;
   size_t capacity_;
   bool strict_capacity_limit_;

cmake/RocksDBConfig.cmake.in

@@ -1,54 +1,3 @@
 @PACKAGE_INIT@
-
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
-
-include(CMakeFindDependencyMacro)
-
-set(GFLAGS_USE_TARGET_NAMESPACE @GFLAGS_USE_TARGET_NAMESPACE@)
-
-if(@WITH_JEMALLOC@)
-  find_dependency(JeMalloc)
-endif()
-
-if(@WITH_GFLAGS@)
-  find_dependency(gflags CONFIG)
-  if(NOT gflags_FOUND)
-    find_dependency(gflags)
-  endif()
-endif()
-
-if(@WITH_SNAPPY@)
-  find_dependency(Snappy CONFIG)
-  if(NOT Snappy_FOUND)
-    find_dependency(Snappy)
-  endif()
-endif()
-
-if(@WITH_ZLIB@)
-  find_dependency(ZLIB)
-endif()
-
-if(@WITH_BZ2@)
-  find_dependency(BZip2)
-endif()
-
-if(@WITH_LZ4@)
-  find_dependency(lz4)
-endif()
-
-if(@WITH_ZSTD@)
-  find_dependency(zstd)
-endif()
-
-if(@WITH_NUMA@)
-  find_dependency(NUMA)
-endif()
-
-if(@WITH_TBB@)
-  find_dependency(TBB)
-endif()
-
-find_dependency(Threads)
-
 include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
 check_required_components(RocksDB)

cmake/modules/CxxFlags.cmake

@@ -1,7 +0,0 @@
-macro(get_cxx_std_flags FLAGS_VARIABLE)
-  if( CMAKE_CXX_STANDARD_REQUIRED )
-    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION})
-  else()
-    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_EXTENSION_COMPILE_OPTION})
-  endif()
-endmacro()

cmake/modules/FindSnappy.cmake

@@ -1,29 +0,0 @@
-# - Find Snappy
-# Find the snappy compression library and includes
-#
-# Snappy_INCLUDE_DIRS - where to find snappy.h, etc.
-# Snappy_LIBRARIES - List of libraries when using snappy.
-# Snappy_FOUND - True if snappy found.
-
-find_path(Snappy_INCLUDE_DIRS
-  NAMES snappy.h
-  HINTS ${snappy_ROOT_DIR}/include)
-
-find_library(Snappy_LIBRARIES
-  NAMES snappy
-  HINTS ${snappy_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_LIBRARIES Snappy_INCLUDE_DIRS)
-
-mark_as_advanced(
-  Snappy_LIBRARIES
-  Snappy_INCLUDE_DIRS)
-
-if(Snappy_FOUND AND NOT (TARGET Snappy::snappy))
-  add_library (Snappy::snappy UNKNOWN IMPORTED)
-  set_target_properties(Snappy::snappy
-    PROPERTIES
-      IMPORTED_LOCATION ${Snappy_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${Snappy_INCLUDE_DIRS})
-endif()

cmake/modules/Findgflags.cmake

@@ -1,8 +1,8 @@
 # - Find gflags library
 # Find the gflags includes and library
 #
-# GFLAGS_INCLUDE_DIR - where to find gflags.h.
-# GFLAGS_LIBRARIES - List of libraries when using gflags.
+# gflags_INCLUDE_DIR - where to find gflags.h.
+# gflags_LIBRARIES - List of libraries when using gflags.
 # gflags_FOUND - True if gflags found.
 
 find_path(GFLAGS_INCLUDE_DIR

cmake/modules/Findsnappy.cmake

@@ -0,0 +1,29 @@
+# - Find Snappy
+# Find the snappy compression library and includes
+#
+# snappy_INCLUDE_DIRS - where to find snappy.h, etc.
+# snappy_LIBRARIES - List of libraries when using snappy.
+# snappy_FOUND - True if snappy found.
+
+find_path(snappy_INCLUDE_DIRS
+  NAMES snappy.h
+  HINTS ${snappy_ROOT_DIR}/include)
+
+find_library(snappy_LIBRARIES
+  NAMES snappy
+  HINTS ${snappy_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(snappy DEFAULT_MSG snappy_LIBRARIES snappy_INCLUDE_DIRS)
+
+mark_as_advanced(
+  snappy_LIBRARIES
+  snappy_INCLUDE_DIRS)
+
+if(snappy_FOUND AND NOT (TARGET snappy::snappy))
+  add_library (snappy::snappy UNKNOWN IMPORTED)
+  set_target_properties(snappy::snappy
+    PROPERTIES
+      IMPORTED_LOCATION ${snappy_LIBRARIES}
+      INTERFACE_INCLUDE_DIRECTORIES ${snappy_INCLUDE_DIRS})
+endif()

coverage/coverage_test.sh

@@ -12,24 +12,21 @@ fi
 ROOT=".."
 # Fetch right version of gcov
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-  source $ROOT/build_tools/fbcode_config_platform007.sh
+  source $ROOT/build_tools/fbcode_config.sh
   GCOV=$GCC_BASE/bin/gcov
 else
   GCOV=$(which gcov)
 fi
-echo -e "Using $GCOV"
 
 COVERAGE_DIR="$PWD/COVERAGE_REPORT"
 mkdir -p $COVERAGE_DIR
 
 # Find all gcno files to generate the coverage report
 
-PYTHON=${1:-`which python`}
-echo -e "Using $PYTHON"
 GCNO_FILES=`find $ROOT -name "*.gcno"`
 $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
   # Parse the raw gcov report to more human readable form.
-  $PYTHON $ROOT/coverage/parse_gcov_output.py |
+  python $ROOT/coverage/parse_gcov_output.py |
   # Write the output to both stdout and report file.
   tee $COVERAGE_DIR/coverage_report_all.txt &&
 echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
@@ -44,7 +41,7 @@ RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
 
 echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
 $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
-  $PYTHON $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
+  python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
   tee -a $RECENT_REPORT &&
 echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
 

coverage/parse_gcov_output.py

@@ -1,12 +1,10 @@
-#!/usr/bin/env python
+#!/usr/bin/env python2
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-
-from __future__ import print_function
-
-import optparse
 import re
 import sys
 
+from optparse import OptionParser
+
 # the gcov report follows certain pattern. Each file will have two lines
 # of report, from which we can extract the file name, total lines and coverage
 # percentage.
@@ -50,7 +48,7 @@ def parse_gcov_report(gcov_input):
 def get_option_parser():
     usage = "Parse the gcov output and generate more human-readable code " +\
             "coverage report."
-    parser = optparse.OptionParser(usage)
+    parser = OptionParser(usage)
 
     parser.add_option(
         "--interested-files", "-i",
@@ -75,8 +73,8 @@ def display_file_coverage(per_file_coverage, total_coverage):
     header_template = \
         "%" + str(max_file_name_length) + "s\t%s\t%s"
     separator = "-" * (max_file_name_length + 10 + 20)
-    print(header_template % ("Filename", "Coverage", "Lines"))  # noqa: E999 T25377293 Grandfathered in
-    print(separator)
+    print header_template % ("Filename", "Coverage", "Lines")  # noqa: E999 T25377293 Grandfathered in
+    print separator
 
     # -- Print body
     # template for printing coverage report for each file.
@@ -84,12 +82,12 @@ def display_file_coverage(per_file_coverage, total_coverage):
 
     for fname, coverage_info in per_file_coverage.items():
         coverage, lines = coverage_info
-        print(record_template % (fname, coverage, lines))
+        print record_template % (fname, coverage, lines)
 
     # -- Print footer
     if total_coverage:
-        print(separator)
-        print(record_template % ("Total", total_coverage[0], total_coverage[1]))
+        print separator
+        print record_template % ("Total", total_coverage[0], total_coverage[1])
 
 def report_coverage():
     parser = get_option_parser()
@@ -113,7 +111,7 @@ def report_coverage():
         total_coverage = None
 
     if not len(per_file_coverage):
-        print("Cannot find coverage info for the given files.", file=sys.stderr)
+        print >> sys.stderr, "Cannot find coverage info for the given files."
         return
     display_file_coverage(per_file_coverage, total_coverage)
 

db/arena_wrapped_db_iter.cc

@@ -30,20 +30,21 @@ Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
   return db_iter_->GetProperty(prop_name, prop);
 }
 
-void ArenaWrappedDBIter::Init(
-    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
+void ArenaWrappedDBIter::Init(Env* env, const ReadOptions& read_options,
+                              const ImmutableCFOptions& cf_options,
+                              const MutableCFOptions& mutable_cf_options,
+                              const SequenceNumber& sequence,
+                              uint64_t max_sequential_skip_in_iteration,
+                              uint64_t version_number,
+                              ReadCallback* read_callback, DBImpl* db_impl,
+                              ColumnFamilyData* cfd, bool allow_blob,
+                              bool allow_refresh) {
   auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ =
-      new (mem) DBIter(env, read_options, ioptions, mutable_cf_options,
-                       ioptions.user_comparator, /* iter */ nullptr, version,
-                       sequence, true, max_sequential_skip_in_iteration,
-                       read_callback, db_impl, cfd, expose_blob_index);
+  db_iter_ = new (mem) DBIter(env, read_options, cf_options, mutable_cf_options,
+                              cf_options.user_comparator, nullptr, sequence,
+                              true, max_sequential_skip_in_iteration,
+                              read_callback, db_impl, cfd, allow_blob);
   sv_number_ = version_number;
-  read_options_ = read_options;
   allow_refresh_ = allow_refresh;
 }
 
@@ -55,9 +56,8 @@ Status ArenaWrappedDBIter::Refresh() {
   // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the
   // correct behavior. Will be corrected automatically when we take a snapshot
   // here for the case of WritePreparedTxnDB.
+  SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
   uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
-  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
   if (sv_number_ != cur_sv_number) {
     Env* env = db_iter_->env();
     db_iter_->~DBIter();
@@ -65,39 +65,39 @@ Status ArenaWrappedDBIter::Refresh() {
     new (&arena_) Arena();
 
     SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
-    SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
     if (read_callback_) {
       read_callback_->Refresh(latest_seq);
     }
     Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options,
-         sv->current, latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
+         latest_seq, sv->mutable_cf_options.max_sequential_skip_in_iterations,
+         cur_sv_number, read_callback_, db_impl_, cfd_, allow_blob_,
          allow_refresh_);
 
     InternalIterator* internal_iter = db_impl_->NewInternalIterator(
         read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator(),
-        latest_seq, /* allow_unprepared_value */ true);
+        latest_seq);
     SetIterUnderDBIter(internal_iter);
   } else {
-    db_iter_->set_sequence(db_impl_->GetLatestSequenceNumber());
+    db_iter_->set_sequence(latest_seq);
     db_iter_->set_valid(false);
   }
   return Status::OK();
 }
 
 ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
+    Env* env, const ReadOptions& read_options,
+    const ImmutableCFOptions& cf_options,
+    const MutableCFOptions& mutable_cf_options, const SequenceNumber& sequence,
+    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd,
+    bool allow_blob, bool allow_refresh) {
   ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, ioptions, mutable_cf_options, version, sequence,
+  iter->Init(env, read_options, cf_options, mutable_cf_options, sequence,
              max_sequential_skip_in_iterations, version_number, read_callback,
-             db_impl, cfd, expose_blob_index, allow_refresh);
+             db_impl, cfd, allow_blob, allow_refresh);
   if (db_impl != nullptr && cfd != nullptr && allow_refresh) {
-    iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index);
+    iter->StoreRefreshInfo(read_options, db_impl, cfd, read_callback,
+                           allow_blob);
   }
 
   return iter;

db/arena_wrapped_db_iter.h

@@ -23,7 +23,6 @@
 namespace ROCKSDB_NAMESPACE {
 
 class Arena;
-class Version;
 
 // A wrapper iterator which wraps DB Iterator and the arena, with which the DB
 // iterator is supposed to be allocated. This class is used as an entry point of
@@ -42,7 +41,6 @@ class ArenaWrappedDBIter : public Iterator {
   virtual ReadRangeDelAggregator* GetRangeDelAggregator() {
     return db_iter_->GetRangeDelAggregator();
   }
-  const ReadOptions& GetReadOptions() { return read_options_; }
 
   // Set the internal iterator wrapped inside the DB Iterator. Usually it is
   // a merging iterator.
@@ -72,21 +70,23 @@ class ArenaWrappedDBIter : public Iterator {
   Status Refresh() override;
 
   void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableOptions& ioptions,
-            const MutableCFOptions& mutable_cf_options, const Version* version,
+            const ImmutableCFOptions& cf_options,
+            const MutableCFOptions& mutable_cf_options,
             const SequenceNumber& sequence,
             uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
             ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd,
-            bool expose_blob_index, bool allow_refresh);
+            bool allow_blob, bool allow_refresh);
 
   // Store some parameters so we can refresh the iterator at a later point
   // with these same params
-  void StoreRefreshInfo(DBImpl* db_impl, ColumnFamilyData* cfd,
-                        ReadCallback* read_callback, bool expose_blob_index) {
+  void StoreRefreshInfo(const ReadOptions& read_options, DBImpl* db_impl,
+                        ColumnFamilyData* cfd, ReadCallback* read_callback,
+                        bool allow_blob) {
+    read_options_ = read_options;
     db_impl_ = db_impl;
     cfd_ = cfd;
     read_callback_ = read_callback;
-    expose_blob_index_ = expose_blob_index;
+    allow_blob_ = allow_blob;
   }
 
  private:
@@ -97,7 +97,7 @@ class ArenaWrappedDBIter : public Iterator {
   DBImpl* db_impl_ = nullptr;
   ReadOptions read_options_;
   ReadCallback* read_callback_;
-  bool expose_blob_index_ = false;
+  bool allow_blob_ = false;
   bool allow_refresh_ = true;
 };
 
@@ -105,10 +105,11 @@ class ArenaWrappedDBIter : public Iterator {
 // `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
 // be supported.
 extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options, const ImmutableOptions& ioptions,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
-    bool expose_blob_index = false, bool allow_refresh = true);
+    Env* env, const ReadOptions& read_options,
+    const ImmutableCFOptions& cf_options,
+    const MutableCFOptions& mutable_cf_options, const SequenceNumber& sequence,
+    uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
+    ReadCallback* read_callback, DBImpl* db_impl = nullptr,
+    ColumnFamilyData* cfd = nullptr, bool allow_blob = false,
+    bool allow_refresh = true);
 }  // namespace ROCKSDB_NAMESPACE

db/blob/blob_counting_iterator.h

@@ -1,146 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-
-#include "db/blob/blob_garbage_meter.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "rocksdb/status.h"
-#include "table/internal_iterator.h"
-#include "test_util/sync_point.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// An internal iterator that passes each key-value encountered to
-// BlobGarbageMeter as inflow in order to measure the total number and size of
-// blobs in the compaction input on a per-blob file basis.
-class BlobCountingIterator : public InternalIterator {
- public:
-  BlobCountingIterator(InternalIterator* iter,
-                       BlobGarbageMeter* blob_garbage_meter)
-      : iter_(iter), blob_garbage_meter_(blob_garbage_meter) {
-    assert(iter_);
-    assert(blob_garbage_meter_);
-
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  bool Valid() const override { return iter_->Valid() && status_.ok(); }
-
-  void SeekToFirst() override {
-    iter_->SeekToFirst();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void SeekToLast() override {
-    iter_->SeekToLast();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void Seek(const Slice& target) override {
-    iter_->Seek(target);
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void SeekForPrev(const Slice& target) override {
-    iter_->SeekForPrev(target);
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  void Next() override {
-    assert(Valid());
-
-    iter_->Next();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  bool NextAndGetResult(IterateResult* result) override {
-    assert(Valid());
-
-    const bool res = iter_->NextAndGetResult(result);
-    UpdateAndCountBlobIfNeeded();
-    return res;
-  }
-
-  void Prev() override {
-    assert(Valid());
-
-    iter_->Prev();
-    UpdateAndCountBlobIfNeeded();
-  }
-
-  Slice key() const override {
-    assert(Valid());
-    return iter_->key();
-  }
-
-  Slice user_key() const override {
-    assert(Valid());
-    return iter_->user_key();
-  }
-
-  Slice value() const override {
-    assert(Valid());
-    return iter_->value();
-  }
-
-  Status status() const override { return status_; }
-
-  bool PrepareValue() override {
-    assert(Valid());
-    return iter_->PrepareValue();
-  }
-
-  bool MayBeOutOfLowerBound() override {
-    assert(Valid());
-    return iter_->MayBeOutOfLowerBound();
-  }
-
-  IterBoundCheck UpperBoundCheckResult() override {
-    assert(Valid());
-    return iter_->UpperBoundCheckResult();
-  }
-
-  void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override {
-    iter_->SetPinnedItersMgr(pinned_iters_mgr);
-  }
-
-  bool IsKeyPinned() const override {
-    assert(Valid());
-    return iter_->IsKeyPinned();
-  }
-
-  bool IsValuePinned() const override {
-    assert(Valid());
-    return iter_->IsValuePinned();
-  }
-
-  Status GetProperty(std::string prop_name, std::string* prop) override {
-    return iter_->GetProperty(prop_name, prop);
-  }
-
- private:
-  void UpdateAndCountBlobIfNeeded() {
-    assert(!iter_->Valid() || iter_->status().ok());
-
-    if (!iter_->Valid()) {
-      status_ = iter_->status();
-      return;
-    }
-
-    TEST_SYNC_POINT(
-        "BlobCountingIterator::UpdateAndCountBlobIfNeeded:ProcessInFlow");
-
-    status_ = blob_garbage_meter_->ProcessInFlow(key(), value());
-  }
-
-  InternalIterator* iter_;
-  BlobGarbageMeter* blob_garbage_meter_;
-  Status status_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_counting_iterator_test.cc

@@ -1,325 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_counting_iterator.h"
-
-#include <string>
-#include <vector>
-
-#include "db/blob/blob_garbage_meter.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/dbformat.h"
-#include "test_util/testharness.h"
-#include "test_util/testutil.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-void CheckInFlow(const BlobGarbageMeter& blob_garbage_meter,
-                 uint64_t blob_file_number, uint64_t count, uint64_t bytes) {
-  const auto& flows = blob_garbage_meter.flows();
-
-  const auto it = flows.find(blob_file_number);
-  if (it == flows.end()) {
-    ASSERT_EQ(count, 0);
-    ASSERT_EQ(bytes, 0);
-    return;
-  }
-
-  const auto& in = it->second.GetInFlow();
-
-  ASSERT_EQ(in.GetCount(), count);
-  ASSERT_EQ(in.GetBytes(), bytes);
-}
-
-TEST(BlobCountingIteratorTest, CountBlobs) {
-  // Note: the input consists of three key-values: two are blob references to
-  // different blob files, while the third one is a plain value.
-  constexpr char user_key0[] = "key0";
-  constexpr char user_key1[] = "key1";
-  constexpr char user_key2[] = "key2";
-
-  const std::vector<std::string> keys{
-      test::KeyStr(user_key0, 1, kTypeBlobIndex),
-      test::KeyStr(user_key1, 2, kTypeBlobIndex),
-      test::KeyStr(user_key2, 3, kTypeValue)};
-
-  constexpr uint64_t first_blob_file_number = 4;
-  constexpr uint64_t first_offset = 1000;
-  constexpr uint64_t first_size = 2000;
-
-  std::string first_blob_index;
-  BlobIndex::EncodeBlob(&first_blob_index, first_blob_file_number, first_offset,
-                        first_size, kNoCompression);
-
-  constexpr uint64_t second_blob_file_number = 6;
-  constexpr uint64_t second_offset = 2000;
-  constexpr uint64_t second_size = 4000;
-
-  std::string second_blob_index;
-  BlobIndex::EncodeBlob(&second_blob_index, second_blob_file_number,
-                        second_offset, second_size, kNoCompression);
-
-  const std::vector<std::string> values{first_blob_index, second_blob_index,
-                                        "raw_value"};
-
-  assert(keys.size() == values.size());
-
-  test::VectorIterator input(keys, values);
-  BlobGarbageMeter blob_garbage_meter;
-
-  BlobCountingIterator blob_counter(&input, &blob_garbage_meter);
-
-  constexpr uint64_t first_expected_bytes =
-      first_size +
-      BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(user_key0) - 1);
-  constexpr uint64_t second_expected_bytes =
-      second_size +
-      BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(user_key1) - 1);
-
-  // Call SeekToFirst and iterate forward
-  blob_counter.SeekToFirst();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 0, 0);
-
-  blob_counter.Next();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  blob_counter.Next();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  blob_counter.Next();
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 1,
-              first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  // Do it again using NextAndGetResult
-  blob_counter.SeekToFirst();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-              2 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 1,
-              second_expected_bytes);
-
-  {
-    IterateResult result;
-    ASSERT_TRUE(blob_counter.NextAndGetResult(&result));
-    ASSERT_EQ(result.key, keys[1]);
-    ASSERT_EQ(blob_counter.user_key(), user_key1);
-    ASSERT_TRUE(blob_counter.Valid());
-    ASSERT_OK(blob_counter.status());
-    ASSERT_EQ(blob_counter.key(), keys[1]);
-    ASSERT_EQ(blob_counter.value(), values[1]);
-    CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-                2 * first_expected_bytes);
-    CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-                2 * second_expected_bytes);
-  }
-
-  {
-    IterateResult result;
-    ASSERT_TRUE(blob_counter.NextAndGetResult(&result));
-    ASSERT_EQ(result.key, keys[2]);
-    ASSERT_EQ(blob_counter.user_key(), user_key2);
-    ASSERT_TRUE(blob_counter.Valid());
-    ASSERT_OK(blob_counter.status());
-    ASSERT_EQ(blob_counter.key(), keys[2]);
-    ASSERT_EQ(blob_counter.value(), values[2]);
-    CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-                2 * first_expected_bytes);
-    CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-                2 * second_expected_bytes);
-  }
-
-  {
-    IterateResult result;
-    ASSERT_FALSE(blob_counter.NextAndGetResult(&result));
-    ASSERT_FALSE(blob_counter.Valid());
-    ASSERT_OK(blob_counter.status());
-    CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-                2 * first_expected_bytes);
-    CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-                2 * second_expected_bytes);
-  }
-
-  // Call SeekToLast and iterate backward
-  blob_counter.SeekToLast();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-              2 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 2,
-              2 * second_expected_bytes);
-
-  blob_counter.Prev();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 2,
-              2 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  blob_counter.Prev();
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 3,
-              3 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  blob_counter.Prev();
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 3,
-              3 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  // Call Seek for all keys (plus one that's greater than all of them)
-  blob_counter.Seek(keys[0]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 3,
-              3 * second_expected_bytes);
-
-  blob_counter.Seek(keys[1]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.Seek(keys[2]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.Seek("zzz");
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  // Call SeekForPrev for all keys (plus one that's less than all of them)
-  blob_counter.SeekForPrev("aaa");
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 4,
-              4 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.SeekForPrev(keys[0]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[0]);
-  ASSERT_EQ(blob_counter.user_key(), user_key0);
-  ASSERT_EQ(blob_counter.value(), values[0]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 5,
-              5 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 4,
-              4 * second_expected_bytes);
-
-  blob_counter.SeekForPrev(keys[1]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[1]);
-  ASSERT_EQ(blob_counter.user_key(), user_key1);
-  ASSERT_EQ(blob_counter.value(), values[1]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 5,
-              5 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 5,
-              5 * second_expected_bytes);
-
-  blob_counter.SeekForPrev(keys[2]);
-  ASSERT_TRUE(blob_counter.Valid());
-  ASSERT_OK(blob_counter.status());
-  ASSERT_EQ(blob_counter.key(), keys[2]);
-  ASSERT_EQ(blob_counter.user_key(), user_key2);
-  ASSERT_EQ(blob_counter.value(), values[2]);
-  CheckInFlow(blob_garbage_meter, first_blob_file_number, 5,
-              5 * first_expected_bytes);
-  CheckInFlow(blob_garbage_meter, second_blob_file_number, 5,
-              5 * second_expected_bytes);
-}
-
-TEST(BlobCountingIteratorTest, CorruptBlobIndex) {
-  const std::vector<std::string> keys{
-      test::KeyStr("user_key", 1, kTypeBlobIndex)};
-  const std::vector<std::string> values{"i_am_not_a_blob_index"};
-
-  assert(keys.size() == values.size());
-
-  test::VectorIterator input(keys, values);
-  BlobGarbageMeter blob_garbage_meter;
-
-  BlobCountingIterator blob_counter(&input, &blob_garbage_meter);
-
-  blob_counter.SeekToFirst();
-  ASSERT_FALSE(blob_counter.Valid());
-  ASSERT_NOK(blob_counter.status());
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_fetcher.cc

@@ -1,22 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_fetcher.h"
-
-#include "db/version_set.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status BlobFetcher::FetchBlob(const Slice& user_key, const Slice& blob_index,
-                              PinnableSlice* blob_value) {
-  Status s;
-  assert(version_);
-  constexpr uint64_t* bytes_read = nullptr;
-  s = version_->GetBlob(read_options_, user_key, blob_index, blob_value,
-                        bytes_read);
-  return s;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_fetcher.h

@@ -1,26 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include "rocksdb/options.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-class Version;
-
-class BlobFetcher {
- public:
-  BlobFetcher(Version* version, const ReadOptions& read_options)
-      : version_(version), read_options_(read_options) {}
-
-  Status FetchBlob(const Slice& user_key, const Slice& blob_index,
-                   PinnableSlice* blob_value);
-
- private:
-  Version* version_;
-  ReadOptions read_options_;
-};
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition.cc

@@ -135,8 +135,7 @@ std::ostream& operator<<(std::ostream& os,
      << " total_blob_count: " << blob_file_addition.GetTotalBlobCount()
      << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes()
      << " checksum_method: " << blob_file_addition.GetChecksumMethod()
-     << " checksum_value: "
-     << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true);
+     << " checksum_value: " << blob_file_addition.GetChecksumValue();
 
   return os;
 }
@@ -147,8 +146,7 @@ JSONWriter& operator<<(JSONWriter& jw,
      << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount()
      << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes()
      << "ChecksumMethod" << blob_file_addition.GetChecksumMethod()
-     << "ChecksumValue"
-     << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true);
+     << "ChecksumValue" << blob_file_addition.GetChecksumValue();
 
   return jw;
 }

db/blob/blob_file_addition_test.cc

@@ -46,9 +46,7 @@ TEST_F(BlobFileAdditionTest, NonEmpty) {
   constexpr uint64_t total_blob_count = 2;
   constexpr uint64_t total_blob_bytes = 123456;
   const std::string checksum_method("SHA1");
-  const std::string checksum_value(
-      "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
-      "\x5c\xbd");
+  const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd");
 
   BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
                                       total_blob_bytes, checksum_method,
@@ -115,9 +113,7 @@ TEST_F(BlobFileAdditionTest, DecodeErrors) {
     ASSERT_TRUE(std::strstr(s.getState(), "checksum value"));
   }
 
-  constexpr char checksum_value[] =
-      "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52"
-      "\x5c\xbd";
+  constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
   PutLengthPrefixedSlice(&str, checksum_value);
   slice = str;
 
@@ -154,7 +150,7 @@ TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) {
   constexpr uint64_t total_blob_count = 9999;
   constexpr uint64_t total_blob_bytes = 100000000;
   const std::string checksum_method("CRC32");
-  const std::string checksum_value("\x3d\x87\xff\x57");
+  const std::string checksum_value("3d87ff57");
 
   BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
                                       total_blob_bytes, checksum_method,
@@ -182,7 +178,7 @@ TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) {
   constexpr uint64_t total_blob_count = 100;
   constexpr uint64_t total_blob_bytes = 2000000;
   const std::string checksum_method("CRC32B");
-  const std::string checksum_value("\x6d\xbd\xf2\x3a");
+  const std::string checksum_value("6dbdf23a");
 
   BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
                                       total_blob_bytes, checksum_method,

db/blob/blob_file_builder.cc

@@ -1,349 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_builder.h"
-
-#include <cassert>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_file_completion_callback.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "db/version_set.h"
-#include "file/filename.h"
-#include "file/read_write_util.h"
-#include "file/writable_file_writer.h"
-#include "logging/logging.h"
-#include "options/cf_options.h"
-#include "options/options_helper.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "trace_replay/io_tracer.h"
-#include "util/compression.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobFileBuilder::BlobFileBuilder(
-    VersionSet* versions, FileSystem* fs,
-    const ImmutableOptions* immutable_options,
-    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCompletionCallback* blob_callback,
-    std::vector<std::string>* blob_file_paths,
-    std::vector<BlobFileAddition>* blob_file_additions)
-    : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
-                      immutable_options, mutable_cf_options, file_options,
-                      job_id, column_family_id, column_family_name, io_priority,
-                      write_hint, io_tracer, blob_callback, blob_file_paths,
-                      blob_file_additions) {}
-
-BlobFileBuilder::BlobFileBuilder(
-    std::function<uint64_t()> file_number_generator, FileSystem* fs,
-    const ImmutableOptions* immutable_options,
-    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCompletionCallback* blob_callback,
-    std::vector<std::string>* blob_file_paths,
-    std::vector<BlobFileAddition>* blob_file_additions)
-    : file_number_generator_(std::move(file_number_generator)),
-      fs_(fs),
-      immutable_options_(immutable_options),
-      min_blob_size_(mutable_cf_options->min_blob_size),
-      blob_file_size_(mutable_cf_options->blob_file_size),
-      blob_compression_type_(mutable_cf_options->blob_compression_type),
-      file_options_(file_options),
-      job_id_(job_id),
-      column_family_id_(column_family_id),
-      column_family_name_(column_family_name),
-      io_priority_(io_priority),
-      write_hint_(write_hint),
-      io_tracer_(io_tracer),
-      blob_callback_(blob_callback),
-      blob_file_paths_(blob_file_paths),
-      blob_file_additions_(blob_file_additions),
-      blob_count_(0),
-      blob_bytes_(0) {
-  assert(file_number_generator_);
-  assert(fs_);
-  assert(immutable_options_);
-  assert(file_options_);
-  assert(blob_file_paths_);
-  assert(blob_file_paths_->empty());
-  assert(blob_file_additions_);
-  assert(blob_file_additions_->empty());
-}
-
-BlobFileBuilder::~BlobFileBuilder() = default;
-
-Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
-                            std::string* blob_index) {
-  assert(blob_index);
-  assert(blob_index->empty());
-
-  if (value.size() < min_blob_size_) {
-    return Status::OK();
-  }
-
-  {
-    const Status s = OpenBlobFileIfNeeded();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  Slice blob = value;
-  std::string compressed_blob;
-
-  {
-    const Status s = CompressBlobIfNeeded(&blob, &compressed_blob);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  uint64_t blob_file_number = 0;
-  uint64_t blob_offset = 0;
-
-  {
-    const Status s =
-        WriteBlobToFile(key, blob, &blob_file_number, &blob_offset);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  {
-    const Status s = CloseBlobFileIfNeeded();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(),
-                        blob_compression_type_);
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::Finish() {
-  if (!IsBlobFileOpen()) {
-    return Status::OK();
-  }
-
-  return CloseBlobFile();
-}
-
-bool BlobFileBuilder::IsBlobFileOpen() const { return !!writer_; }
-
-Status BlobFileBuilder::OpenBlobFileIfNeeded() {
-  if (IsBlobFileOpen()) {
-    return Status::OK();
-  }
-
-  assert(!blob_count_);
-  assert(!blob_bytes_);
-
-  assert(file_number_generator_);
-  const uint64_t blob_file_number = file_number_generator_();
-
-  assert(immutable_options_);
-  assert(!immutable_options_->cf_paths.empty());
-  std::string blob_file_path =
-      BlobFileName(immutable_options_->cf_paths.front().path, blob_file_number);
-
-  std::unique_ptr<FSWritableFile> file;
-
-  {
-    assert(file_options_);
-    Status s = NewWritableFile(fs_, blob_file_path, &file, *file_options_);
-
-    TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile", &s);
-
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  // Note: files get added to blob_file_paths_ right after the open, so they
-  // can be cleaned up upon failure. Contrast this with blob_file_additions_,
-  // which only contains successfully written files.
-  assert(blob_file_paths_);
-  blob_file_paths_->emplace_back(std::move(blob_file_path));
-
-  assert(file);
-  file->SetIOPriority(io_priority_);
-  file->SetWriteLifeTimeHint(write_hint_);
-  FileTypeSet tmp_set = immutable_options_->checksum_handoff_file_types;
-  Statistics* const statistics = immutable_options_->stats;
-  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-      std::move(file), blob_file_paths_->back(), *file_options_,
-      immutable_options_->clock, io_tracer_, statistics,
-      immutable_options_->listeners,
-      immutable_options_->file_checksum_gen_factory.get(),
-      tmp_set.Contains(FileType::kBlobFile), false));
-
-  constexpr bool do_flush = false;
-
-  std::unique_ptr<BlobLogWriter> blob_log_writer(new BlobLogWriter(
-      std::move(file_writer), immutable_options_->clock, statistics,
-      blob_file_number, immutable_options_->use_fsync, do_flush));
-
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-
-  BlobLogHeader header(column_family_id_, blob_compression_type_, has_ttl,
-                       expiration_range);
-
-  {
-    Status s = blob_log_writer->WriteHeader(header);
-
-    TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
-
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  writer_ = std::move(blob_log_writer);
-
-  assert(IsBlobFileOpen());
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::CompressBlobIfNeeded(
-    Slice* blob, std::string* compressed_blob) const {
-  assert(blob);
-  assert(compressed_blob);
-  assert(compressed_blob->empty());
-
-  if (blob_compression_type_ == kNoCompression) {
-    return Status::OK();
-  }
-
-  CompressionOptions opts;
-  CompressionContext context(blob_compression_type_);
-  constexpr uint64_t sample_for_compression = 0;
-
-  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                       blob_compression_type_, sample_for_compression);
-
-  constexpr uint32_t compression_format_version = 2;
-
-  if (!CompressData(*blob, info, compression_format_version, compressed_blob)) {
-    return Status::Corruption("Error compressing blob");
-  }
-
-  *blob = Slice(*compressed_blob);
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob,
-                                        uint64_t* blob_file_number,
-                                        uint64_t* blob_offset) {
-  assert(IsBlobFileOpen());
-  assert(blob_file_number);
-  assert(blob_offset);
-
-  uint64_t key_offset = 0;
-
-  Status s = writer_->AddRecord(key, blob, &key_offset, blob_offset);
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  *blob_file_number = writer_->get_log_number();
-
-  ++blob_count_;
-  blob_bytes_ += BlobLogRecord::kHeaderSize + key.size() + blob.size();
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::CloseBlobFile() {
-  assert(IsBlobFileOpen());
-
-  BlobLogFooter footer;
-  footer.blob_count = blob_count_;
-
-  std::string checksum_method;
-  std::string checksum_value;
-
-  Status s = writer_->AppendFooter(footer, &checksum_method, &checksum_value);
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  const uint64_t blob_file_number = writer_->get_log_number();
-
-  assert(blob_file_additions_);
-  blob_file_additions_->emplace_back(blob_file_number, blob_count_, blob_bytes_,
-                                     std::move(checksum_method),
-                                     std::move(checksum_value));
-
-  assert(immutable_options_);
-  ROCKS_LOG_INFO(immutable_options_->logger,
-                 "[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64
-                 " total blobs, %" PRIu64 " total bytes",
-                 column_family_name_.c_str(), job_id_, blob_file_number,
-                 blob_count_, blob_bytes_);
-  if (blob_callback_) {
-    s = blob_callback_->OnBlobFileCompleted(blob_file_paths_->back());
-  }
-
-  writer_.reset();
-  blob_count_ = 0;
-  blob_bytes_ = 0;
-
-  return s;
-}
-
-Status BlobFileBuilder::CloseBlobFileIfNeeded() {
-  assert(IsBlobFileOpen());
-
-  const WritableFileWriter* const file_writer = writer_->file();
-  assert(file_writer);
-
-  if (file_writer->GetFileSize() < blob_file_size_) {
-    return Status::OK();
-  }
-
-  return CloseBlobFile();
-}
-
-void BlobFileBuilder::Abandon() {
-  if (!IsBlobFileOpen()) {
-    return;
-  }
-
-  if (blob_callback_) {
-    // BlobFileBuilder::Abandon() is called because of error while writing to
-    // Blob files. So we can ignore the below error.
-    blob_callback_->OnBlobFileCompleted(blob_file_paths_->back())
-        .PermitUncheckedError();
-  }
-
-  writer_.reset();
-  blob_count_ = 0;
-  blob_bytes_ = 0;
-}
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder.h

@@ -1,99 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <cinttypes>
-#include <functional>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "rocksdb/compression_type.h"
-#include "rocksdb/env.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class VersionSet;
-class FileSystem;
-class SystemClock;
-struct ImmutableOptions;
-struct MutableCFOptions;
-struct FileOptions;
-class BlobFileAddition;
-class Status;
-class Slice;
-class BlobLogWriter;
-class IOTracer;
-class BlobFileCompletionCallback;
-
-class BlobFileBuilder {
- public:
-  BlobFileBuilder(VersionSet* versions, FileSystem* fs,
-                  const ImmutableOptions* immutable_options,
-                  const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
-                  uint32_t column_family_id,
-                  const std::string& column_family_name,
-                  Env::IOPriority io_priority,
-                  Env::WriteLifeTimeHint write_hint,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  BlobFileCompletionCallback* blob_callback,
-                  std::vector<std::string>* blob_file_paths,
-                  std::vector<BlobFileAddition>* blob_file_additions);
-
-  BlobFileBuilder(std::function<uint64_t()> file_number_generator,
-                  FileSystem* fs, const ImmutableOptions* immutable_options,
-                  const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
-                  uint32_t column_family_id,
-                  const std::string& column_family_name,
-                  Env::IOPriority io_priority,
-                  Env::WriteLifeTimeHint write_hint,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  BlobFileCompletionCallback* blob_callback,
-                  std::vector<std::string>* blob_file_paths,
-                  std::vector<BlobFileAddition>* blob_file_additions);
-
-  BlobFileBuilder(const BlobFileBuilder&) = delete;
-  BlobFileBuilder& operator=(const BlobFileBuilder&) = delete;
-
-  ~BlobFileBuilder();
-
-  Status Add(const Slice& key, const Slice& value, std::string* blob_index);
-  Status Finish();
-  void Abandon();
-
- private:
-  bool IsBlobFileOpen() const;
-  Status OpenBlobFileIfNeeded();
-  Status CompressBlobIfNeeded(Slice* blob, std::string* compressed_blob) const;
-  Status WriteBlobToFile(const Slice& key, const Slice& blob,
-                         uint64_t* blob_file_number, uint64_t* blob_offset);
-  Status CloseBlobFile();
-  Status CloseBlobFileIfNeeded();
-
-  std::function<uint64_t()> file_number_generator_;
-  FileSystem* fs_;
-  const ImmutableOptions* immutable_options_;
-  uint64_t min_blob_size_;
-  uint64_t blob_file_size_;
-  CompressionType blob_compression_type_;
-  const FileOptions* file_options_;
-  int job_id_;
-  uint32_t column_family_id_;
-  std::string column_family_name_;
-  Env::IOPriority io_priority_;
-  Env::WriteLifeTimeHint write_hint_;
-  std::shared_ptr<IOTracer> io_tracer_;
-  BlobFileCompletionCallback* blob_callback_;
-  std::vector<std::string>* blob_file_paths_;
-  std::vector<BlobFileAddition>* blob_file_additions_;
-  std::unique_ptr<BlobLogWriter> writer_;
-  uint64_t blob_count_;
-  uint64_t blob_bytes_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder_test.cc

@@ -1,666 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_builder.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_sequential_reader.h"
-#include "env/mock_env.h"
-#include "file/filename.h"
-#include "file/random_access_file_reader.h"
-#include "options/cf_options.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_checksum.h"
-#include "rocksdb/options.h"
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/compression.h"
-#include "utilities/fault_injection_env.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class TestFileNumberGenerator {
- public:
-  uint64_t operator()() { return ++next_file_number_; }
-
- private:
-  uint64_t next_file_number_ = 1;
-};
-
-class BlobFileBuilderTest : public testing::Test {
- protected:
-  BlobFileBuilderTest() : mock_env_(Env::Default()) {
-    fs_ = mock_env_.GetFileSystem().get();
-    clock_ = mock_env_.GetSystemClock().get();
-  }
-
-  void VerifyBlobFile(uint64_t blob_file_number,
-                      const std::string& blob_file_path,
-                      uint32_t column_family_id,
-                      CompressionType blob_compression_type,
-                      const std::vector<std::pair<std::string, std::string>>&
-                          expected_key_value_pairs,
-                      const std::vector<std::string>& blob_indexes) {
-    assert(expected_key_value_pairs.size() == blob_indexes.size());
-
-    std::unique_ptr<FSRandomAccessFile> file;
-    constexpr IODebugContext* dbg = nullptr;
-    ASSERT_OK(
-        fs_->NewRandomAccessFile(blob_file_path, file_options_, &file, dbg));
-
-    std::unique_ptr<RandomAccessFileReader> file_reader(
-        new RandomAccessFileReader(std::move(file), blob_file_path, clock_));
-
-    constexpr Statistics* statistics = nullptr;
-    BlobLogSequentialReader blob_log_reader(std::move(file_reader), clock_,
-                                            statistics);
-
-    BlobLogHeader header;
-    ASSERT_OK(blob_log_reader.ReadHeader(&header));
-    ASSERT_EQ(header.version, kVersion1);
-    ASSERT_EQ(header.column_family_id, column_family_id);
-    ASSERT_EQ(header.compression, blob_compression_type);
-    ASSERT_FALSE(header.has_ttl);
-    ASSERT_EQ(header.expiration_range, ExpirationRange());
-
-    for (size_t i = 0; i < expected_key_value_pairs.size(); ++i) {
-      BlobLogRecord record;
-      uint64_t blob_offset = 0;
-
-      ASSERT_OK(blob_log_reader.ReadRecord(
-          &record, BlobLogSequentialReader::kReadHeaderKeyBlob, &blob_offset));
-
-      // Check the contents of the blob file
-      const auto& expected_key_value = expected_key_value_pairs[i];
-      const auto& key = expected_key_value.first;
-      const auto& value = expected_key_value.second;
-
-      ASSERT_EQ(record.key_size, key.size());
-      ASSERT_EQ(record.value_size, value.size());
-      ASSERT_EQ(record.expiration, 0);
-      ASSERT_EQ(record.key, key);
-      ASSERT_EQ(record.value, value);
-
-      // Make sure the blob reference returned by the builder points to the
-      // right place
-      BlobIndex blob_index;
-      ASSERT_OK(blob_index.DecodeFrom(blob_indexes[i]));
-      ASSERT_FALSE(blob_index.IsInlined());
-      ASSERT_FALSE(blob_index.HasTTL());
-      ASSERT_EQ(blob_index.file_number(), blob_file_number);
-      ASSERT_EQ(blob_index.offset(), blob_offset);
-      ASSERT_EQ(blob_index.size(), value.size());
-    }
-
-    BlobLogFooter footer;
-    ASSERT_OK(blob_log_reader.ReadFooter(&footer));
-    ASSERT_EQ(footer.blob_count, expected_key_value_pairs.size());
-    ASSERT_EQ(footer.expiration_range, ExpirationRange());
-  }
-
-  MockEnv mock_env_;
-  FileSystem* fs_;
-  SystemClock* clock_;
-  FileOptions file_options_;
-};
-
-TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
-  // Build a single blob file
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 4;
-  constexpr size_t value_offset = 1234;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileBuilderTest_BuildAndCheckOneFile"),
-      0);
-  options.enable_blob_files = true;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
-      number_of_blobs);
-  std::vector<std::string> blob_indexes(number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    auto& expected_key_value = expected_key_value_pairs[i];
-
-    auto& key = expected_key_value.first;
-    key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    auto& value = expected_key_value.second;
-    value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    auto& blob_index = blob_indexes[i];
-
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_FALSE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(
-      blob_file_path,
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), number_of_blobs);
-  ASSERT_EQ(
-      blob_file_addition.GetTotalBlobBytes(),
-      number_of_blobs * (BlobLogRecord::kHeaderSize + key_size + value_size));
-
-  // Verify the contents of the new blob file as well as the blob references
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kNoCompression, expected_key_value_pairs, blob_indexes);
-}
-
-TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
-  // Build multiple blob files: file size limit is set to the size of a single
-  // value, so each blob ends up in a file of its own
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 10;
-  constexpr size_t value_offset = 1234567890;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileBuilderTest_BuildAndCheckMultipleFiles"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_file_size = value_size;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
-      number_of_blobs);
-  std::vector<std::string> blob_indexes(number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    auto& expected_key_value = expected_key_value_pairs[i];
-
-    auto& key = expected_key_value.first;
-    key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    auto& value = expected_key_value.second;
-    value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    auto& blob_index = blob_indexes[i];
-
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_FALSE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  ASSERT_EQ(blob_file_paths.size(), number_of_blobs);
-  ASSERT_EQ(blob_file_additions.size(), number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    const uint64_t blob_file_number = i + 2;
-
-    ASSERT_EQ(blob_file_paths[i],
-              BlobFileName(immutable_options.cf_paths.front().path,
-                           blob_file_number));
-
-    const auto& blob_file_addition = blob_file_additions[i];
-
-    ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-    ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-    ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-              BlobLogRecord::kHeaderSize + key_size + value_size);
-  }
-
-  // Verify the contents of the new blob files as well as the blob references
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    std::vector<std::pair<std::string, std::string>> expected_key_value_pair{
-        expected_key_value_pairs[i]};
-    std::vector<std::string> blob_index{blob_indexes[i]};
-
-    VerifyBlobFile(i + 2, blob_file_paths[i], column_family_id, kNoCompression,
-                   expected_key_value_pair, blob_index);
-  }
-}
-
-TEST_F(BlobFileBuilderTest, InlinedValues) {
-  // All values are below the min_blob_size threshold; no blob files get written
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 10;
-  constexpr size_t value_offset = 1234567890;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_InlinedValues"),
-      0);
-  options.enable_blob_files = true;
-  options.min_blob_size = 1024;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    const std::string key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    const std::string value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    std::string blob_index;
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_TRUE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  ASSERT_TRUE(blob_file_paths.empty());
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-TEST_F(BlobFileBuilderTest, Compression) {
-  // Build a blob file with a compressed blob
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 100;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_Compression"), 0);
-  options.enable_blob_files = true;
-  options.blob_compression_type = kSnappyCompression;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  const std::string key("1");
-  const std::string uncompressed_value(value_size, 'x');
-
-  std::string blob_index;
-
-  ASSERT_OK(builder.Add(key, uncompressed_value, &blob_index));
-  ASSERT_FALSE(blob_index.empty());
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(
-      blob_file_path,
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-
-  CompressionOptions opts;
-  CompressionContext context(kSnappyCompression);
-  constexpr uint64_t sample_for_compression = 0;
-
-  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                       kSnappyCompression, sample_for_compression);
-
-  std::string compressed_value;
-  ASSERT_TRUE(Snappy_Compress(info, uncompressed_value.data(),
-                              uncompressed_value.size(), &compressed_value));
-
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-            BlobLogRecord::kHeaderSize + key_size + compressed_value.size());
-
-  // Verify the contents of the new blob file as well as the blob reference
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
-      {key, compressed_value}};
-  std::vector<std::string> blob_indexes{blob_index};
-
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kSnappyCompression, expected_key_value_pairs, blob_indexes);
-}
-
-TEST_F(BlobFileBuilderTest, CompressionError) {
-  // Simulate an error during compression
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_CompressionError"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_compression_type = kSnappyCompression;
-  options.env = &mock_env_;
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
-                                        [](void* arg) {
-                                          bool* ret = static_cast<bool*>(arg);
-                                          *ret = false;
-                                        });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr char key[] = "1";
-  constexpr char value[] = "deadbeef";
-
-  std::string blob_index;
-
-  ASSERT_TRUE(builder.Add(key, value, &blob_index).IsCorruption());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-  ASSERT_EQ(
-      blob_file_paths[0],
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-TEST_F(BlobFileBuilderTest, Checksum) {
-  // Build a blob file with checksum
-
-  class DummyFileChecksumGenerator : public FileChecksumGenerator {
-   public:
-    void Update(const char* /* data */, size_t /* n */) override {}
-
-    void Finalize() override {}
-
-    std::string GetChecksum() const override { return std::string("dummy"); }
-
-    const char* Name() const override { return "DummyFileChecksum"; }
-  };
-
-  class DummyFileChecksumGenFactory : public FileChecksumGenFactory {
-   public:
-    std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
-        const FileChecksumGenContext& /* context */) override {
-      return std::unique_ptr<FileChecksumGenerator>(
-          new DummyFileChecksumGenerator);
-    }
-
-    const char* Name() const override { return "DummyFileChecksumGenFactory"; }
-  };
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_Checksum"), 0);
-  options.enable_blob_files = true;
-  options.file_checksum_gen_factory =
-      std::make_shared<DummyFileChecksumGenFactory>();
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  const std::string key("1");
-  const std::string value("deadbeef");
-
-  std::string blob_index;
-
-  ASSERT_OK(builder.Add(key, value, &blob_index));
-  ASSERT_FALSE(blob_index.empty());
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(
-      blob_file_path,
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-            BlobLogRecord::kHeaderSize + key.size() + value.size());
-  ASSERT_EQ(blob_file_addition.GetChecksumMethod(), "DummyFileChecksum");
-  ASSERT_EQ(blob_file_addition.GetChecksumValue(), "dummy");
-
-  // Verify the contents of the new blob file as well as the blob reference
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
-      {key, value}};
-  std::vector<std::string> blob_indexes{blob_index};
-
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kNoCompression, expected_key_value_pairs, blob_indexes);
-}
-
-class BlobFileBuilderIOErrorTest
-    : public testing::Test,
-      public testing::WithParamInterface<std::string> {
- protected:
-  BlobFileBuilderIOErrorTest()
-      : mock_env_(Env::Default()),
-        fs_(mock_env_.GetFileSystem().get()),
-        sync_point_(GetParam()) {}
-
-  MockEnv mock_env_;
-  FileSystem* fs_;
-  FileOptions file_options_;
-  std::string sync_point_;
-};
-
-INSTANTIATE_TEST_CASE_P(
-    BlobFileBuilderTest, BlobFileBuilderIOErrorTest,
-    ::testing::ValuesIn(std::vector<std::string>{
-        "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile",
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader",
-        "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
-
-TEST_P(BlobFileBuilderIOErrorTest, IOError) {
-  // Simulate an I/O error during the specified step of Add()
-  // Note: blob_file_size will be set to value_size in order for the first blob
-  // to trigger close
-  constexpr size_t value_size = 8;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderIOErrorTest_IOError"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_file_size = value_size;
-  options.env = &mock_env_;
-
-  ImmutableOptions immutable_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(
-      TestFileNumberGenerator(), fs_, &immutable_options, &mutable_cf_options,
-      &file_options_, job_id, column_family_id, column_family_name, io_priority,
-      write_hint, nullptr /*IOTracer*/, nullptr /*BlobFileCompletionCallback*/,
-      &blob_file_paths, &blob_file_additions);
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
-    Status* const s = static_cast<Status*>(arg);
-    assert(s);
-
-    (*s) = Status::IOError(sync_point_);
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr char key[] = "1";
-  constexpr char value[] = "deadbeef";
-
-  std::string blob_index;
-
-  ASSERT_TRUE(builder.Add(key, value, &blob_index).IsIOError());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  if (sync_point_ == "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile") {
-    ASSERT_TRUE(blob_file_paths.empty());
-  } else {
-    constexpr uint64_t blob_file_number = 2;
-
-    ASSERT_EQ(blob_file_paths.size(), 1);
-    ASSERT_EQ(blob_file_paths[0],
-              BlobFileName(immutable_options.cf_paths.front().path,
-                           blob_file_number));
-  }
-
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_cache.cc

@@ -1,102 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_cache.h"
-
-#include <cassert>
-#include <memory>
-
-#include "db/blob/blob_file_reader.h"
-#include "options/cf_options.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/slice.h"
-#include "test_util/sync_point.h"
-#include "trace_replay/io_tracer.h"
-#include "util/hash.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobFileCache::BlobFileCache(Cache* cache,
-                             const ImmutableOptions* immutable_options,
-                             const FileOptions* file_options,
-                             uint32_t column_family_id,
-                             HistogramImpl* blob_file_read_hist,
-                             const std::shared_ptr<IOTracer>& io_tracer)
-    : cache_(cache),
-      mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
-      immutable_options_(immutable_options),
-      file_options_(file_options),
-      column_family_id_(column_family_id),
-      blob_file_read_hist_(blob_file_read_hist),
-      io_tracer_(io_tracer) {
-  assert(cache_);
-  assert(immutable_options_);
-  assert(file_options_);
-}
-
-Status BlobFileCache::GetBlobFileReader(
-    uint64_t blob_file_number,
-    CacheHandleGuard<BlobFileReader>* blob_file_reader) {
-  assert(blob_file_reader);
-  assert(blob_file_reader->IsEmpty());
-
-  const Slice key = GetSlice(&blob_file_number);
-
-  assert(cache_);
-
-  Cache::Handle* handle = cache_->Lookup(key);
-  if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-    return Status::OK();
-  }
-
-  TEST_SYNC_POINT("BlobFileCache::GetBlobFileReader:DoubleCheck");
-
-  // Check again while holding mutex
-  MutexLock lock(mutex_.get(key));
-
-  handle = cache_->Lookup(key);
-  if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-    return Status::OK();
-  }
-
-  assert(immutable_options_);
-  Statistics* const statistics = immutable_options_->stats;
-
-  RecordTick(statistics, NO_FILE_OPENS);
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  {
-    assert(file_options_);
-    const Status s = BlobFileReader::Create(
-        *immutable_options_, *file_options_, column_family_id_,
-        blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
-    if (!s.ok()) {
-      RecordTick(statistics, NO_FILE_ERRORS);
-      return s;
-    }
-  }
-
-  {
-    constexpr size_t charge = 1;
-
-    const Status s = cache_->Insert(key, reader.get(), charge,
-                                    &DeleteCacheEntry<BlobFileReader>, &handle);
-    if (!s.ok()) {
-      RecordTick(statistics, NO_FILE_ERRORS);
-      return s;
-    }
-  }
-
-  reader.release();
-
-  *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_cache.h

@@ -1,52 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cinttypes>
-
-#include "cache/cache_helpers.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "util/mutexlock.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Cache;
-struct ImmutableOptions;
-struct FileOptions;
-class HistogramImpl;
-class Status;
-class BlobFileReader;
-class Slice;
-class IOTracer;
-
-class BlobFileCache {
- public:
-  BlobFileCache(Cache* cache, const ImmutableOptions* immutable_options,
-                const FileOptions* file_options, uint32_t column_family_id,
-                HistogramImpl* blob_file_read_hist,
-                const std::shared_ptr<IOTracer>& io_tracer);
-
-  BlobFileCache(const BlobFileCache&) = delete;
-  BlobFileCache& operator=(const BlobFileCache&) = delete;
-
-  Status GetBlobFileReader(uint64_t blob_file_number,
-                           CacheHandleGuard<BlobFileReader>* blob_file_reader);
-
- private:
-  Cache* cache_;
-  // Note: mutex_ below is used to guard against multiple threads racing to open
-  // the same file.
-  Striped<port::Mutex, Slice> mutex_;
-  const ImmutableOptions* immutable_options_;
-  const FileOptions* file_options_;
-  uint32_t column_family_id_;
-  HistogramImpl* blob_file_read_hist_;
-  std::shared_ptr<IOTracer> io_tracer_;
-
-  static constexpr size_t kNumberOfMutexStripes = 1 << 7;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_cache_test.cc

@@ -1,267 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_cache.h"
-
-#include <cassert>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "env/mock_env.h"
-#include "file/filename.h"
-#include "file/read_write_util.h"
-#include "file/writable_file_writer.h"
-#include "options/cf_options.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_system.h"
-#include "rocksdb/options.h"
-#include "rocksdb/statistics.h"
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-namespace {
-
-// Creates a test blob file with a single blob in it.
-void WriteBlobFile(uint32_t column_family_id,
-                   const ImmutableOptions& immutable_options,
-                   uint64_t blob_file_number) {
-  assert(!immutable_options.cf_paths.empty());
-
-  const std::string blob_file_path =
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number);
-
-  std::unique_ptr<FSWritableFile> file;
-  ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file,
-                            FileOptions()));
-
-  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-      std::move(file), blob_file_path, FileOptions(), immutable_options.clock));
-
-  constexpr Statistics* statistics = nullptr;
-  constexpr bool use_fsync = false;
-  constexpr bool do_flush = false;
-
-  BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, blob_file_number, use_fsync,
-                                do_flush);
-
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-
-  BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
-                       expiration_range);
-
-  ASSERT_OK(blob_log_writer.WriteHeader(header));
-
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  std::string compressed_blob;
-
-  uint64_t key_offset = 0;
-  uint64_t blob_offset = 0;
-
-  ASSERT_OK(blob_log_writer.AddRecord(key, blob, &key_offset, &blob_offset));
-
-  BlobLogFooter footer;
-  footer.blob_count = 1;
-  footer.expiration_range = expiration_range;
-
-  std::string checksum_method;
-  std::string checksum_value;
-
-  ASSERT_OK(
-      blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
-}
-
-}  // anonymous namespace
-
-class BlobFileCacheTest : public testing::Test {
- protected:
-  BlobFileCacheTest() : mock_env_(Env::Default()) {}
-
-  MockEnv mock_env_;
-};
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileCacheTest_GetBlobFileReader"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr uint32_t column_family_id = 1;
-  ImmutableOptions immutable_options(options);
-  constexpr uint64_t blob_file_number = 123;
-
-  WriteBlobFile(column_family_id, immutable_options, blob_file_number);
-
-  constexpr size_t capacity = 10;
-  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
-
-  FileOptions file_options;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  // First try: reader should be opened and put in cache
-  CacheHandleGuard<BlobFileReader> first;
-
-  ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &first));
-  ASSERT_NE(first.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-
-  // Second try: reader should be served from cache
-  CacheHandleGuard<BlobFileReader> second;
-
-  ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &second));
-  ASSERT_NE(second.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-
-  ASSERT_EQ(first.GetValue(), second.GetValue());
-}
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileCacheTest_GetBlobFileReader_Race"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr uint32_t column_family_id = 1;
-  ImmutableOptions immutable_options(options);
-  constexpr uint64_t blob_file_number = 123;
-
-  WriteBlobFile(column_family_id, immutable_options, blob_file_number);
-
-  constexpr size_t capacity = 10;
-  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
-
-  FileOptions file_options;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  CacheHandleGuard<BlobFileReader> first;
-  CacheHandleGuard<BlobFileReader> second;
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileCache::GetBlobFileReader:DoubleCheck", [&](void* /* arg */) {
-        // Disabling sync points to prevent infinite recursion
-        SyncPoint::GetInstance()->DisableProcessing();
-
-        ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &second));
-        ASSERT_NE(second.GetValue(), nullptr);
-        ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-        ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &first));
-  ASSERT_NE(first.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-
-  ASSERT_EQ(first.GetValue(), second.GetValue());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileCacheTest_GetBlobFileReader_IOError"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr size_t capacity = 10;
-  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
-
-  ImmutableOptions immutable_options(options);
-  FileOptions file_options;
-  constexpr uint32_t column_family_id = 1;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  // Note: there is no blob file with the below number
-  constexpr uint64_t blob_file_number = 123;
-
-  CacheHandleGuard<BlobFileReader> reader;
-
-  ASSERT_TRUE(
-      blob_file_cache.GetBlobFileReader(blob_file_number, &reader).IsIOError());
-  ASSERT_EQ(reader.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1);
-}
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileCacheTest_GetBlobFileReader_CacheFull"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr uint32_t column_family_id = 1;
-  ImmutableOptions immutable_options(options);
-  constexpr uint64_t blob_file_number = 123;
-
-  WriteBlobFile(column_family_id, immutable_options, blob_file_number);
-
-  constexpr size_t capacity = 0;
-  constexpr int num_shard_bits = -1;  // determined automatically
-  constexpr bool strict_capacity_limit = true;
-  std::shared_ptr<Cache> backing_cache =
-      NewLRUCache(capacity, num_shard_bits, strict_capacity_limit);
-
-  FileOptions file_options;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  // Insert into cache should fail since it has zero capacity and
-  // strict_capacity_limit is set
-  CacheHandleGuard<BlobFileReader> reader;
-
-  ASSERT_TRUE(blob_file_cache.GetBlobFileReader(blob_file_number, &reader)
-                  .IsIncomplete());
-  ASSERT_EQ(reader.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1);
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_completion_callback.h

@@ -1,57 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-
-#include "db/error_handler.h"
-#include "file/sst_file_manager_impl.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlobFileCompletionCallback {
- public:
-#ifdef ROCKSDB_LITE
-  BlobFileCompletionCallback(SstFileManager* /*sst_file_manager*/,
-                             InstrumentedMutex* /*mutex*/,
-                             ErrorHandler* /*error_handler*/) {}
-  Status OnBlobFileCompleted(const std::string& /*file_name*/) {
-    return Status::OK();
-  }
-#else
-  BlobFileCompletionCallback(SstFileManager* sst_file_manager,
-                             InstrumentedMutex* mutex,
-                             ErrorHandler* error_handler)
-      : sst_file_manager_(sst_file_manager),
-        mutex_(mutex),
-        error_handler_(error_handler) {}
-
-  Status OnBlobFileCompleted(const std::string& file_name) {
-    Status s;
-    auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager_);
-    if (sfm) {
-      // Report new blob files to SstFileManagerImpl
-      s = sfm->OnAddFile(file_name);
-      if (sfm->IsMaxAllowedSpaceReached()) {
-        s = Status::SpaceLimit("Max allowed space was reached");
-        TEST_SYNC_POINT(
-            "BlobFileCompletionCallback::CallBack::MaxAllowedSpaceReached");
-        InstrumentedMutexLock l(mutex_);
-        error_handler_->SetBGError(s, BackgroundErrorReason::kFlush);
-      }
-    }
-    return s;
-  }
-
- private:
-  SstFileManager* sst_file_manager_;
-  InstrumentedMutex* mutex_;
-  ErrorHandler* error_handler_;
-#endif  // ROCKSDB_LITE
-};
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_meta.cc

@@ -1,62 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_meta.h"
-
-#include <ostream>
-#include <sstream>
-
-#include "db/blob/blob_log_format.h"
-#include "rocksdb/slice.h"
-
-namespace ROCKSDB_NAMESPACE {
-uint64_t SharedBlobFileMetaData::GetBlobFileSize() const {
-  return BlobLogHeader::kSize + total_blob_bytes_ + BlobLogFooter::kSize;
-}
-
-std::string SharedBlobFileMetaData::DebugString() const {
-  std::ostringstream oss;
-  oss << (*this);
-
-  return oss.str();
-}
-
-std::ostream& operator<<(std::ostream& os,
-                         const SharedBlobFileMetaData& shared_meta) {
-  os << "blob_file_number: " << shared_meta.GetBlobFileNumber()
-     << " total_blob_count: " << shared_meta.GetTotalBlobCount()
-     << " total_blob_bytes: " << shared_meta.GetTotalBlobBytes()
-     << " checksum_method: " << shared_meta.GetChecksumMethod()
-     << " checksum_value: "
-     << Slice(shared_meta.GetChecksumValue()).ToString(/* hex */ true);
-
-  return os;
-}
-
-std::string BlobFileMetaData::DebugString() const {
-  std::ostringstream oss;
-  oss << (*this);
-
-  return oss.str();
-}
-
-std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta) {
-  const auto& shared_meta = meta.GetSharedMeta();
-  assert(shared_meta);
-  os << (*shared_meta);
-
-  os << " linked_ssts: {";
-  for (uint64_t file_number : meta.GetLinkedSsts()) {
-    os << ' ' << file_number;
-  }
-  os << " }";
-
-  os << " garbage_blob_count: " << meta.GetGarbageBlobCount()
-     << " garbage_blob_bytes: " << meta.GetGarbageBlobBytes();
-
-  return os;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_meta.h

@@ -1,170 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-#include <iosfwd>
-#include <memory>
-#include <string>
-#include <unordered_set>
-
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// SharedBlobFileMetaData represents the immutable part of blob files' metadata,
-// like the blob file number, total number and size of blobs, or checksum
-// method and value. There is supposed to be one object of this class per blob
-// file (shared across all versions that include the blob file in question);
-// hence, the type is neither copyable nor movable. A blob file can be marked
-// obsolete when the corresponding SharedBlobFileMetaData object is destroyed.
-
-class SharedBlobFileMetaData {
- public:
-  static std::shared_ptr<SharedBlobFileMetaData> Create(
-      uint64_t blob_file_number, uint64_t total_blob_count,
-      uint64_t total_blob_bytes, std::string checksum_method,
-      std::string checksum_value) {
-    return std::shared_ptr<SharedBlobFileMetaData>(new SharedBlobFileMetaData(
-        blob_file_number, total_blob_count, total_blob_bytes,
-        std::move(checksum_method), std::move(checksum_value)));
-  }
-
-  template <typename Deleter>
-  static std::shared_ptr<SharedBlobFileMetaData> Create(
-      uint64_t blob_file_number, uint64_t total_blob_count,
-      uint64_t total_blob_bytes, std::string checksum_method,
-      std::string checksum_value, Deleter deleter) {
-    return std::shared_ptr<SharedBlobFileMetaData>(
-        new SharedBlobFileMetaData(blob_file_number, total_blob_count,
-                                   total_blob_bytes, std::move(checksum_method),
-                                   std::move(checksum_value)),
-        deleter);
-  }
-
-  SharedBlobFileMetaData(const SharedBlobFileMetaData&) = delete;
-  SharedBlobFileMetaData& operator=(const SharedBlobFileMetaData&) = delete;
-
-  SharedBlobFileMetaData(SharedBlobFileMetaData&&) = delete;
-  SharedBlobFileMetaData& operator=(SharedBlobFileMetaData&&) = delete;
-
-  uint64_t GetBlobFileSize() const;
-  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
-  uint64_t GetTotalBlobCount() const { return total_blob_count_; }
-  uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; }
-  const std::string& GetChecksumMethod() const { return checksum_method_; }
-  const std::string& GetChecksumValue() const { return checksum_value_; }
-
-  std::string DebugString() const;
-
- private:
-  SharedBlobFileMetaData(uint64_t blob_file_number, uint64_t total_blob_count,
-                         uint64_t total_blob_bytes, std::string checksum_method,
-                         std::string checksum_value)
-      : blob_file_number_(blob_file_number),
-        total_blob_count_(total_blob_count),
-        total_blob_bytes_(total_blob_bytes),
-        checksum_method_(std::move(checksum_method)),
-        checksum_value_(std::move(checksum_value)) {
-    assert(checksum_method_.empty() == checksum_value_.empty());
-  }
-
-  uint64_t blob_file_number_;
-  uint64_t total_blob_count_;
-  uint64_t total_blob_bytes_;
-  std::string checksum_method_;
-  std::string checksum_value_;
-};
-
-std::ostream& operator<<(std::ostream& os,
-                         const SharedBlobFileMetaData& shared_meta);
-
-// BlobFileMetaData contains the part of the metadata for blob files that can
-// vary across versions, like the amount of garbage in the blob file. In
-// addition, BlobFileMetaData objects point to and share the ownership of the
-// SharedBlobFileMetaData object for the corresponding blob file. Similarly to
-// SharedBlobFileMetaData, BlobFileMetaData are not copyable or movable. They
-// are meant to be jointly owned by the versions in which the blob file has the
-// same (immutable *and* mutable) state.
-
-class BlobFileMetaData {
- public:
-  using LinkedSsts = std::unordered_set<uint64_t>;
-
-  static std::shared_ptr<BlobFileMetaData> Create(
-      std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-      LinkedSsts linked_ssts, uint64_t garbage_blob_count,
-      uint64_t garbage_blob_bytes) {
-    return std::shared_ptr<BlobFileMetaData>(
-        new BlobFileMetaData(std::move(shared_meta), std::move(linked_ssts),
-                             garbage_blob_count, garbage_blob_bytes));
-  }
-
-  BlobFileMetaData(const BlobFileMetaData&) = delete;
-  BlobFileMetaData& operator=(const BlobFileMetaData&) = delete;
-
-  BlobFileMetaData(BlobFileMetaData&&) = delete;
-  BlobFileMetaData& operator=(BlobFileMetaData&&) = delete;
-
-  const std::shared_ptr<SharedBlobFileMetaData>& GetSharedMeta() const {
-    return shared_meta_;
-  }
-
-  uint64_t GetBlobFileSize() const {
-    assert(shared_meta_);
-    return shared_meta_->GetBlobFileSize();
-  }
-
-  uint64_t GetBlobFileNumber() const {
-    assert(shared_meta_);
-    return shared_meta_->GetBlobFileNumber();
-  }
-  uint64_t GetTotalBlobCount() const {
-    assert(shared_meta_);
-    return shared_meta_->GetTotalBlobCount();
-  }
-  uint64_t GetTotalBlobBytes() const {
-    assert(shared_meta_);
-    return shared_meta_->GetTotalBlobBytes();
-  }
-  const std::string& GetChecksumMethod() const {
-    assert(shared_meta_);
-    return shared_meta_->GetChecksumMethod();
-  }
-  const std::string& GetChecksumValue() const {
-    assert(shared_meta_);
-    return shared_meta_->GetChecksumValue();
-  }
-
-  const LinkedSsts& GetLinkedSsts() const { return linked_ssts_; }
-
-  uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; }
-  uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; }
-
-  std::string DebugString() const;
-
- private:
-  BlobFileMetaData(std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-                   LinkedSsts linked_ssts, uint64_t garbage_blob_count,
-                   uint64_t garbage_blob_bytes)
-      : shared_meta_(std::move(shared_meta)),
-        linked_ssts_(std::move(linked_ssts)),
-        garbage_blob_count_(garbage_blob_count),
-        garbage_blob_bytes_(garbage_blob_bytes) {
-    assert(shared_meta_);
-    assert(garbage_blob_count_ <= shared_meta_->GetTotalBlobCount());
-    assert(garbage_blob_bytes_ <= shared_meta_->GetTotalBlobBytes());
-  }
-
-  std::shared_ptr<SharedBlobFileMetaData> shared_meta_;
-  LinkedSsts linked_ssts_;
-  uint64_t garbage_blob_count_;
-  uint64_t garbage_blob_bytes_;
-};
-
-std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta);
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_reader.cc

@@ -1,428 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_reader.h"
-
-#include <cassert>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "file/filename.h"
-#include "options/cf_options.h"
-#include "rocksdb/file_system.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "util/compression.h"
-#include "util/crc32c.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status BlobFileReader::Create(
-    const ImmutableOptions& immutable_options, const FileOptions& file_options,
-    uint32_t column_family_id, HistogramImpl* blob_file_read_hist,
-    uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
-    std::unique_ptr<BlobFileReader>* blob_file_reader) {
-  assert(blob_file_reader);
-  assert(!*blob_file_reader);
-
-  uint64_t file_size = 0;
-  std::unique_ptr<RandomAccessFileReader> file_reader;
-
-  {
-    const Status s =
-        OpenFile(immutable_options, file_options, blob_file_read_hist,
-                 blob_file_number, io_tracer, &file_size, &file_reader);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  assert(file_reader);
-
-  CompressionType compression_type = kNoCompression;
-
-  {
-    const Status s =
-        ReadHeader(file_reader.get(), column_family_id, &compression_type);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  {
-    const Status s = ReadFooter(file_size, file_reader.get());
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  blob_file_reader->reset(
-      new BlobFileReader(std::move(file_reader), file_size, compression_type));
-
-  return Status::OK();
-}
-
-Status BlobFileReader::OpenFile(
-    const ImmutableOptions& immutable_options, const FileOptions& file_opts,
-    HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
-    const std::shared_ptr<IOTracer>& io_tracer, uint64_t* file_size,
-    std::unique_ptr<RandomAccessFileReader>* file_reader) {
-  assert(file_size);
-  assert(file_reader);
-
-  const auto& cf_paths = immutable_options.cf_paths;
-  assert(!cf_paths.empty());
-
-  const std::string blob_file_path =
-      BlobFileName(cf_paths.front().path, blob_file_number);
-
-  FileSystem* const fs = immutable_options.fs.get();
-  assert(fs);
-
-  constexpr IODebugContext* dbg = nullptr;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::OpenFile:GetFileSize");
-
-    const Status s =
-        fs->GetFileSize(blob_file_path, IOOptions(), file_size, dbg);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (*file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) {
-    return Status::Corruption("Malformed blob file");
-  }
-
-  std::unique_ptr<FSRandomAccessFile> file;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::OpenFile:NewRandomAccessFile");
-
-    const Status s =
-        fs->NewRandomAccessFile(blob_file_path, file_opts, &file, dbg);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  assert(file);
-
-  if (immutable_options.advise_random_on_open) {
-    file->Hint(FSRandomAccessFile::kRandom);
-  }
-
-  file_reader->reset(new RandomAccessFileReader(
-      std::move(file), blob_file_path, immutable_options.clock, io_tracer,
-      immutable_options.stats, BLOB_DB_BLOB_FILE_READ_MICROS,
-      blob_file_read_hist, immutable_options.rate_limiter.get(),
-      immutable_options.listeners));
-
-  return Status::OK();
-}
-
-Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader,
-                                  uint32_t column_family_id,
-                                  CompressionType* compression_type) {
-  assert(file_reader);
-  assert(compression_type);
-
-  Slice header_slice;
-  Buffer buf;
-  AlignedBuf aligned_buf;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::ReadHeader:ReadFromFile");
-
-    constexpr uint64_t read_offset = 0;
-    constexpr size_t read_size = BlobLogHeader::kSize;
-
-    const Status s = ReadFromFile(file_reader, read_offset, read_size,
-                                  &header_slice, &buf, &aligned_buf);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::ReadHeader:TamperWithResult",
-                             &header_slice);
-  }
-
-  BlobLogHeader header;
-
-  {
-    const Status s = header.DecodeFrom(header_slice);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  constexpr ExpirationRange no_expiration_range;
-
-  if (header.has_ttl || header.expiration_range != no_expiration_range) {
-    return Status::Corruption("Unexpected TTL blob file");
-  }
-
-  if (header.column_family_id != column_family_id) {
-    return Status::Corruption("Column family ID mismatch");
-  }
-
-  *compression_type = header.compression;
-
-  return Status::OK();
-}
-
-Status BlobFileReader::ReadFooter(uint64_t file_size,
-                                  const RandomAccessFileReader* file_reader) {
-  assert(file_size >= BlobLogHeader::kSize + BlobLogFooter::kSize);
-  assert(file_reader);
-
-  Slice footer_slice;
-  Buffer buf;
-  AlignedBuf aligned_buf;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::ReadFooter:ReadFromFile");
-
-    const uint64_t read_offset = file_size - BlobLogFooter::kSize;
-    constexpr size_t read_size = BlobLogFooter::kSize;
-
-    const Status s = ReadFromFile(file_reader, read_offset, read_size,
-                                  &footer_slice, &buf, &aligned_buf);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::ReadFooter:TamperWithResult",
-                             &footer_slice);
-  }
-
-  BlobLogFooter footer;
-
-  {
-    const Status s = footer.DecodeFrom(footer_slice);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  constexpr ExpirationRange no_expiration_range;
-
-  if (footer.expiration_range != no_expiration_range) {
-    return Status::Corruption("Unexpected TTL blob file");
-  }
-
-  return Status::OK();
-}
-
-Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
-                                    uint64_t read_offset, size_t read_size,
-                                    Slice* slice, Buffer* buf,
-                                    AlignedBuf* aligned_buf) {
-  assert(slice);
-  assert(buf);
-  assert(aligned_buf);
-
-  assert(file_reader);
-
-  Status s;
-
-  if (file_reader->use_direct_io()) {
-    constexpr char* scratch = nullptr;
-
-    s = file_reader->Read(IOOptions(), read_offset, read_size, slice, scratch,
-                          aligned_buf);
-  } else {
-    buf->reset(new char[read_size]);
-    constexpr AlignedBuf* aligned_scratch = nullptr;
-
-    s = file_reader->Read(IOOptions(), read_offset, read_size, slice,
-                          buf->get(), aligned_scratch);
-  }
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (slice->size() != read_size) {
-    return Status::Corruption("Failed to read data from blob file");
-  }
-
-  return Status::OK();
-}
-
-BlobFileReader::BlobFileReader(
-    std::unique_ptr<RandomAccessFileReader>&& file_reader, uint64_t file_size,
-    CompressionType compression_type)
-    : file_reader_(std::move(file_reader)),
-      file_size_(file_size),
-      compression_type_(compression_type) {
-  assert(file_reader_);
-}
-
-BlobFileReader::~BlobFileReader() = default;
-
-Status BlobFileReader::GetBlob(const ReadOptions& read_options,
-                               const Slice& user_key, uint64_t offset,
-                               uint64_t value_size,
-                               CompressionType compression_type,
-                               PinnableSlice* value,
-                               uint64_t* bytes_read) const {
-  assert(value);
-
-  const uint64_t key_size = user_key.size();
-
-  if (!IsValidBlobOffset(offset, key_size, value_size, file_size_)) {
-    return Status::Corruption("Invalid blob offset");
-  }
-
-  if (compression_type != compression_type_) {
-    return Status::Corruption("Compression type mismatch when reading blob");
-  }
-
-  // Note: if verify_checksum is set, we read the entire blob record to be able
-  // to perform the verification; otherwise, we just read the blob itself. Since
-  // the offset in BlobIndex actually points to the blob value, we need to make
-  // an adjustment in the former case.
-  const uint64_t adjustment =
-      read_options.verify_checksums
-          ? BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size)
-          : 0;
-  assert(offset >= adjustment);
-
-  const uint64_t record_offset = offset - adjustment;
-  const uint64_t record_size = value_size + adjustment;
-
-  Slice record_slice;
-  Buffer buf;
-  AlignedBuf aligned_buf;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::GetBlob:ReadFromFile");
-
-    const Status s = ReadFromFile(file_reader_.get(), record_offset,
-                                  static_cast<size_t>(record_size),
-                                  &record_slice, &buf, &aligned_buf);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::GetBlob:TamperWithResult",
-                             &record_slice);
-  }
-
-  if (read_options.verify_checksums) {
-    const Status s = VerifyBlob(record_slice, user_key, value_size);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  const Slice value_slice(record_slice.data() + adjustment, value_size);
-
-  {
-    const Status s =
-        UncompressBlobIfNeeded(value_slice, compression_type, value);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (bytes_read) {
-    *bytes_read = record_size;
-  }
-
-  return Status::OK();
-}
-
-Status BlobFileReader::VerifyBlob(const Slice& record_slice,
-                                  const Slice& user_key, uint64_t value_size) {
-  BlobLogRecord record;
-
-  const Slice header_slice(record_slice.data(), BlobLogRecord::kHeaderSize);
-
-  {
-    const Status s = record.DecodeHeaderFrom(header_slice);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (record.key_size != user_key.size()) {
-    return Status::Corruption("Key size mismatch when reading blob");
-  }
-
-  if (record.value_size != value_size) {
-    return Status::Corruption("Value size mismatch when reading blob");
-  }
-
-  record.key =
-      Slice(record_slice.data() + BlobLogRecord::kHeaderSize, record.key_size);
-  if (record.key != user_key) {
-    return Status::Corruption("Key mismatch when reading blob");
-  }
-
-  record.value = Slice(record.key.data() + record.key_size, value_size);
-
-  {
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::VerifyBlob:CheckBlobCRC",
-                             &record);
-
-    const Status s = record.CheckBlobCRC();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  return Status::OK();
-}
-
-Status BlobFileReader::UncompressBlobIfNeeded(const Slice& value_slice,
-                                              CompressionType compression_type,
-                                              PinnableSlice* value) {
-  assert(value);
-
-  if (compression_type == kNoCompression) {
-    SaveValue(value_slice, value);
-
-    return Status::OK();
-  }
-
-  UncompressionContext context(compression_type);
-  UncompressionInfo info(context, UncompressionDict::GetEmptyDict(),
-                         compression_type);
-
-  size_t uncompressed_size = 0;
-  constexpr uint32_t compression_format_version = 2;
-  constexpr MemoryAllocator* allocator = nullptr;
-
-  CacheAllocationPtr output =
-      UncompressData(info, value_slice.data(), value_slice.size(),
-                     &uncompressed_size, compression_format_version, allocator);
-
-  TEST_SYNC_POINT_CALLBACK(
-      "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", &output);
-
-  if (!output) {
-    return Status::Corruption("Unable to uncompress blob");
-  }
-
-  SaveValue(Slice(output.get(), uncompressed_size), value);
-
-  return Status::OK();
-}
-
-void BlobFileReader::SaveValue(const Slice& src, PinnableSlice* dst) {
-  assert(dst);
-
-  if (dst->IsPinned()) {
-    dst->Reset();
-  }
-
-  dst->PinSelf(src);
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_reader.h

@@ -1,85 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cinttypes>
-#include <memory>
-
-#include "file/random_access_file_reader.h"
-#include "rocksdb/compression_type.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Status;
-struct ImmutableOptions;
-struct FileOptions;
-class HistogramImpl;
-struct ReadOptions;
-class Slice;
-class PinnableSlice;
-
-class BlobFileReader {
- public:
-  static Status Create(const ImmutableOptions& immutable_options,
-                       const FileOptions& file_options,
-                       uint32_t column_family_id,
-                       HistogramImpl* blob_file_read_hist,
-                       uint64_t blob_file_number,
-                       const std::shared_ptr<IOTracer>& io_tracer,
-                       std::unique_ptr<BlobFileReader>* reader);
-
-  BlobFileReader(const BlobFileReader&) = delete;
-  BlobFileReader& operator=(const BlobFileReader&) = delete;
-
-  ~BlobFileReader();
-
-  Status GetBlob(const ReadOptions& read_options, const Slice& user_key,
-                 uint64_t offset, uint64_t value_size,
-                 CompressionType compression_type, PinnableSlice* value,
-                 uint64_t* bytes_read) const;
-
- private:
-  BlobFileReader(std::unique_ptr<RandomAccessFileReader>&& file_reader,
-                 uint64_t file_size, CompressionType compression_type);
-
-  static Status OpenFile(const ImmutableOptions& immutable_options,
-                         const FileOptions& file_opts,
-                         HistogramImpl* blob_file_read_hist,
-                         uint64_t blob_file_number,
-                         const std::shared_ptr<IOTracer>& io_tracer,
-                         uint64_t* file_size,
-                         std::unique_ptr<RandomAccessFileReader>* file_reader);
-
-  static Status ReadHeader(const RandomAccessFileReader* file_reader,
-                           uint32_t column_family_id,
-                           CompressionType* compression_type);
-
-  static Status ReadFooter(uint64_t file_size,
-                           const RandomAccessFileReader* file_reader);
-
-  using Buffer = std::unique_ptr<char[]>;
-
-  static Status ReadFromFile(const RandomAccessFileReader* file_reader,
-                             uint64_t read_offset, size_t read_size,
-                             Slice* slice, Buffer* buf,
-                             AlignedBuf* aligned_buf);
-
-  static Status VerifyBlob(const Slice& record_slice, const Slice& user_key,
-                           uint64_t value_size);
-
-  static Status UncompressBlobIfNeeded(const Slice& value_slice,
-                                       CompressionType compression_type,
-                                       PinnableSlice* value);
-
-  static void SaveValue(const Slice& src, PinnableSlice* dst);
-
-  std::unique_ptr<RandomAccessFileReader> file_reader_;
-  uint64_t file_size_;
-  CompressionType compression_type_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_reader_test.cc

@@ -1,812 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_reader.h"
-
-#include <cassert>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "env/mock_env.h"
-#include "file/filename.h"
-#include "file/read_write_util.h"
-#include "file/writable_file_writer.h"
-#include "options/cf_options.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_system.h"
-#include "rocksdb/options.h"
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/compression.h"
-#include "utilities/fault_injection_env.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-namespace {
-
-// Creates a test blob file with a single blob in it. Note: this method
-// makes it possible to test various corner cases by allowing the caller
-// to specify the contents of various blob file header/footer fields.
-void WriteBlobFile(const ImmutableOptions& immutable_options,
-                   uint32_t column_family_id, bool has_ttl,
-                   const ExpirationRange& expiration_range_header,
-                   const ExpirationRange& expiration_range_footer,
-                   uint64_t blob_file_number, const Slice& key,
-                   const Slice& blob, CompressionType compression_type,
-                   uint64_t* blob_offset, uint64_t* blob_size) {
-  assert(!immutable_options.cf_paths.empty());
-  assert(blob_offset);
-  assert(blob_size);
-
-  const std::string blob_file_path =
-      BlobFileName(immutable_options.cf_paths.front().path, blob_file_number);
-
-  std::unique_ptr<FSWritableFile> file;
-  ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file,
-                            FileOptions()));
-
-  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-      std::move(file), blob_file_path, FileOptions(), immutable_options.clock));
-
-  constexpr Statistics* statistics = nullptr;
-  constexpr bool use_fsync = false;
-  constexpr bool do_flush = false;
-
-  BlobLogWriter blob_log_writer(std::move(file_writer), immutable_options.clock,
-                                statistics, blob_file_number, use_fsync,
-                                do_flush);
-
-  BlobLogHeader header(column_family_id, compression_type, has_ttl,
-                       expiration_range_header);
-
-  ASSERT_OK(blob_log_writer.WriteHeader(header));
-
-  std::string compressed_blob;
-  Slice blob_to_write;
-
-  if (compression_type == kNoCompression) {
-    blob_to_write = blob;
-    *blob_size = blob.size();
-  } else {
-    CompressionOptions opts;
-    CompressionContext context(compression_type);
-    constexpr uint64_t sample_for_compression = 0;
-
-    CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                         compression_type, sample_for_compression);
-
-    constexpr uint32_t compression_format_version = 2;
-
-    ASSERT_TRUE(
-        CompressData(blob, info, compression_format_version, &compressed_blob));
-
-    blob_to_write = compressed_blob;
-    *blob_size = compressed_blob.size();
-  }
-
-  uint64_t key_offset = 0;
-
-  ASSERT_OK(
-      blob_log_writer.AddRecord(key, blob_to_write, &key_offset, blob_offset));
-
-  BlobLogFooter footer;
-  footer.blob_count = 1;
-  footer.expiration_range = expiration_range_footer;
-
-  std::string checksum_method;
-  std::string checksum_value;
-
-  ASSERT_OK(
-      blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
-}
-
-}  // anonymous namespace
-
-class BlobFileReaderTest : public testing::Test {
- protected:
-  BlobFileReaderTest() : mock_env_(Env::Default()) {}
-
-  MockEnv mock_env_;
-};
-
-TEST_F(BlobFileReaderTest, CreateReaderAndGetBlob) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileReaderTest_CreateReaderAndGetBlob"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob, kNoCompression,
-                &blob_offset, &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_OK(BlobFileReader::Create(
-      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
-      blob_file_number, nullptr /*IOTracer*/, &reader));
-
-  // Make sure the blob can be retrieved with and without checksum verification
-  ReadOptions read_options;
-  read_options.verify_checksums = false;
-
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size,
-                              kNoCompression, &value, &bytes_read));
-    ASSERT_EQ(value, blob);
-    ASSERT_EQ(bytes_read, blob_size);
-  }
-
-  read_options.verify_checksums = true;
-
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size,
-                              kNoCompression, &value, &bytes_read));
-    ASSERT_EQ(value, blob);
-
-    constexpr uint64_t key_size = sizeof(key) - 1;
-    ASSERT_EQ(bytes_read,
-              BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) +
-                  blob_size);
-  }
-
-  // Invalid offset (too close to start of file)
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(read_options, key, blob_offset - 1, blob_size,
-                              kNoCompression, &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  // Invalid offset (too close to end of file)
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(read_options, key, blob_offset + 1, blob_size,
-                              kNoCompression, &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  // Incorrect compression type
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(read_options, key, blob_offset, blob_size, kZSTD,
-                              &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  // Incorrect key size
-  {
-    constexpr char shorter_key[] = "k";
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(read_options, shorter_key,
-                              blob_offset - (sizeof(key) - sizeof(shorter_key)),
-                              blob_size, kNoCompression, &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  // Incorrect key
-  {
-    constexpr char incorrect_key[] = "foo";
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(read_options, incorrect_key, blob_offset,
-                              blob_size, kNoCompression, &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  // Incorrect value size
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(read_options, key, blob_offset, blob_size + 1,
-                              kNoCompression, &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-}
-
-TEST_F(BlobFileReaderTest, Malformed) {
-  // Write a blob file consisting of nothing but a header, and make sure we
-  // detect the error when we open it for reading
-
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_Malformed"), 0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr uint64_t blob_file_number = 1;
-
-  {
-    constexpr bool has_ttl = false;
-    constexpr ExpirationRange expiration_range;
-
-    const std::string blob_file_path =
-        BlobFileName(immutable_options.cf_paths.front().path, blob_file_number);
-
-    std::unique_ptr<FSWritableFile> file;
-    ASSERT_OK(NewWritableFile(immutable_options.fs.get(), blob_file_path, &file,
-                              FileOptions()));
-
-    std::unique_ptr<WritableFileWriter> file_writer(
-        new WritableFileWriter(std::move(file), blob_file_path, FileOptions(),
-                               immutable_options.clock));
-
-    constexpr Statistics* statistics = nullptr;
-    constexpr bool use_fsync = false;
-    constexpr bool do_flush = false;
-
-    BlobLogWriter blob_log_writer(std::move(file_writer),
-                                  immutable_options.clock, statistics,
-                                  blob_file_number, use_fsync, do_flush);
-
-    BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
-                         expiration_range);
-
-    ASSERT_OK(blob_log_writer.WriteHeader(header));
-  }
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
-                                     column_family_id, blob_file_read_hist,
-                                     blob_file_number, nullptr /*IOTracer*/,
-                                     &reader)
-                  .IsCorruption());
-}
-
-TEST_F(BlobFileReaderTest, TTL) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_TTL"), 0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = true;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob, kNoCompression,
-                &blob_offset, &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
-                                     column_family_id, blob_file_read_hist,
-                                     blob_file_number, nullptr /*IOTracer*/,
-                                     &reader)
-                  .IsCorruption());
-}
-
-TEST_F(BlobFileReaderTest, ExpirationRangeInHeader) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileReaderTest_ExpirationRangeInHeader"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  const ExpirationRange expiration_range_header(
-      1, 2);  // can be made constexpr when we adopt C++14
-  constexpr ExpirationRange expiration_range_footer;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl,
-                expiration_range_header, expiration_range_footer,
-                blob_file_number, key, blob, kNoCompression, &blob_offset,
-                &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
-                                     column_family_id, blob_file_read_hist,
-                                     blob_file_number, nullptr /*IOTracer*/,
-                                     &reader)
-                  .IsCorruption());
-}
-
-TEST_F(BlobFileReaderTest, ExpirationRangeInFooter) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileReaderTest_ExpirationRangeInFooter"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range_header;
-  const ExpirationRange expiration_range_footer(
-      1, 2);  // can be made constexpr when we adopt C++14
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl,
-                expiration_range_header, expiration_range_footer,
-                blob_file_number, key, blob, kNoCompression, &blob_offset,
-                &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
-                                     column_family_id, blob_file_read_hist,
-                                     blob_file_number, nullptr /*IOTracer*/,
-                                     &reader)
-                  .IsCorruption());
-}
-
-TEST_F(BlobFileReaderTest, IncorrectColumnFamily) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileReaderTest_IncorrectColumnFamily"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob, kNoCompression,
-                &blob_offset, &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  constexpr uint32_t incorrect_column_family_id = 2;
-
-  ASSERT_TRUE(BlobFileReader::Create(immutable_options, FileOptions(),
-                                     incorrect_column_family_id,
-                                     blob_file_read_hist, blob_file_number,
-                                     nullptr /*IOTracer*/, &reader)
-                  .IsCorruption());
-}
-
-TEST_F(BlobFileReaderTest, BlobCRCError) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_BlobCRCError"), 0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob, kNoCompression,
-                &blob_offset, &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_OK(BlobFileReader::Create(
-      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
-      blob_file_number, nullptr /*IOTracer*/, &reader));
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileReader::VerifyBlob:CheckBlobCRC", [](void* arg) {
-        BlobLogRecord* const record = static_cast<BlobLogRecord*>(arg);
-        assert(record);
-
-        record->blob_crc = 0xfaceb00c;
-      });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  PinnableSlice value;
-  uint64_t bytes_read = 0;
-
-  ASSERT_TRUE(reader
-                  ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                            kNoCompression, &value, &bytes_read)
-                  .IsCorruption());
-  ASSERT_EQ(bytes_read, 0);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(BlobFileReaderTest, Compression) {
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileReaderTest_Compression"), 0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob,
-                kSnappyCompression, &blob_offset, &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_OK(BlobFileReader::Create(
-      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
-      blob_file_number, nullptr /*IOTracer*/, &reader));
-
-  // Make sure the blob can be retrieved with and without checksum verification
-  ReadOptions read_options;
-  read_options.verify_checksums = false;
-
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size,
-                              kSnappyCompression, &value, &bytes_read));
-    ASSERT_EQ(value, blob);
-    ASSERT_EQ(bytes_read, blob_size);
-  }
-
-  read_options.verify_checksums = true;
-
-  {
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_OK(reader->GetBlob(read_options, key, blob_offset, blob_size,
-                              kSnappyCompression, &value, &bytes_read));
-    ASSERT_EQ(value, blob);
-
-    constexpr uint64_t key_size = sizeof(key) - 1;
-    ASSERT_EQ(bytes_read,
-              BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size) +
-                  blob_size);
-  }
-}
-
-TEST_F(BlobFileReaderTest, UncompressionError) {
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileReaderTest_UncompressionError"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob,
-                kSnappyCompression, &blob_offset, &blob_size);
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  ASSERT_OK(BlobFileReader::Create(
-      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
-      blob_file_number, nullptr /*IOTracer*/, &reader));
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", [](void* arg) {
-        CacheAllocationPtr* const output =
-            static_cast<CacheAllocationPtr*>(arg);
-        assert(output);
-
-        output->reset();
-      });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  PinnableSlice value;
-  uint64_t bytes_read = 0;
-
-  ASSERT_TRUE(reader
-                  ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                            kSnappyCompression, &value, &bytes_read)
-                  .IsCorruption());
-  ASSERT_EQ(bytes_read, 0);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-class BlobFileReaderIOErrorTest
-    : public testing::Test,
-      public testing::WithParamInterface<std::string> {
- protected:
-  BlobFileReaderIOErrorTest()
-      : mock_env_(Env::Default()),
-        fault_injection_env_(&mock_env_),
-        sync_point_(GetParam()) {}
-
-  MockEnv mock_env_;
-  FaultInjectionTestEnv fault_injection_env_;
-  std::string sync_point_;
-};
-
-INSTANTIATE_TEST_CASE_P(BlobFileReaderTest, BlobFileReaderIOErrorTest,
-                        ::testing::ValuesIn(std::vector<std::string>{
-                            "BlobFileReader::OpenFile:GetFileSize",
-                            "BlobFileReader::OpenFile:NewRandomAccessFile",
-                            "BlobFileReader::ReadHeader:ReadFromFile",
-                            "BlobFileReader::ReadFooter:ReadFromFile",
-                            "BlobFileReader::GetBlob:ReadFromFile"}));
-
-TEST_P(BlobFileReaderIOErrorTest, IOError) {
-  // Simulates an I/O error during the specified step
-
-  Options options;
-  options.env = &fault_injection_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&fault_injection_env_,
-                            "BlobFileReaderIOErrorTest_IOError"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob, kNoCompression,
-                &blob_offset, &blob_size);
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
-    fault_injection_env_.SetFilesystemActive(false,
-                                             Status::IOError(sync_point_));
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  const Status s = BlobFileReader::Create(
-      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
-      blob_file_number, nullptr /*IOTracer*/, &reader);
-
-  const bool fail_during_create =
-      (sync_point_ != "BlobFileReader::GetBlob:ReadFromFile");
-
-  if (fail_during_create) {
-    ASSERT_TRUE(s.IsIOError());
-  } else {
-    ASSERT_OK(s);
-
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                              kNoCompression, &value, &bytes_read)
-                    .IsIOError());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-class BlobFileReaderDecodingErrorTest
-    : public testing::Test,
-      public testing::WithParamInterface<std::string> {
- protected:
-  BlobFileReaderDecodingErrorTest()
-      : mock_env_(Env::Default()), sync_point_(GetParam()) {}
-
-  MockEnv mock_env_;
-  std::string sync_point_;
-};
-
-INSTANTIATE_TEST_CASE_P(BlobFileReaderTest, BlobFileReaderDecodingErrorTest,
-                        ::testing::ValuesIn(std::vector<std::string>{
-                            "BlobFileReader::ReadHeader:TamperWithResult",
-                            "BlobFileReader::ReadFooter:TamperWithResult",
-                            "BlobFileReader::GetBlob:TamperWithResult"}));
-
-TEST_P(BlobFileReaderDecodingErrorTest, DecodingError) {
-  Options options;
-  options.env = &mock_env_;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileReaderDecodingErrorTest_DecodingError"),
-      0);
-  options.enable_blob_files = true;
-
-  ImmutableOptions immutable_options(options);
-
-  constexpr uint32_t column_family_id = 1;
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-  constexpr uint64_t blob_file_number = 1;
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  uint64_t blob_offset = 0;
-  uint64_t blob_size = 0;
-
-  WriteBlobFile(immutable_options, column_family_id, has_ttl, expiration_range,
-                expiration_range, blob_file_number, key, blob, kNoCompression,
-                &blob_offset, &blob_size);
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [](void* arg) {
-    Slice* const slice = static_cast<Slice*>(arg);
-    assert(slice);
-    assert(!slice->empty());
-
-    slice->remove_prefix(1);
-  });
-
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  const Status s = BlobFileReader::Create(
-      immutable_options, FileOptions(), column_family_id, blob_file_read_hist,
-      blob_file_number, nullptr /*IOTracer*/, &reader);
-
-  const bool fail_during_create =
-      sync_point_ != "BlobFileReader::GetBlob:TamperWithResult";
-
-  if (fail_during_create) {
-    ASSERT_TRUE(s.IsCorruption());
-  } else {
-    ASSERT_OK(s);
-
-    PinnableSlice value;
-    uint64_t bytes_read = 0;
-
-    ASSERT_TRUE(reader
-                    ->GetBlob(ReadOptions(), key, blob_offset, blob_size,
-                              kNoCompression, &value, &bytes_read)
-                    .IsCorruption());
-    ASSERT_EQ(bytes_read, 0);
-  }
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_garbage_meter.cc

@@ -1,100 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_garbage_meter.h"
-
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/dbformat.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status BlobGarbageMeter::ProcessInFlow(const Slice& key, const Slice& value) {
-  uint64_t blob_file_number = kInvalidBlobFileNumber;
-  uint64_t bytes = 0;
-
-  const Status s = Parse(key, value, &blob_file_number, &bytes);
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (blob_file_number == kInvalidBlobFileNumber) {
-    return Status::OK();
-  }
-
-  flows_[blob_file_number].AddInFlow(bytes);
-
-  return Status::OK();
-}
-
-Status BlobGarbageMeter::ProcessOutFlow(const Slice& key, const Slice& value) {
-  uint64_t blob_file_number = kInvalidBlobFileNumber;
-  uint64_t bytes = 0;
-
-  const Status s = Parse(key, value, &blob_file_number, &bytes);
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (blob_file_number == kInvalidBlobFileNumber) {
-    return Status::OK();
-  }
-
-  // Note: in order to measure the amount of additional garbage, we only need to
-  // track the outflow for preexisting files, i.e. those that also had inflow.
-  // (Newly written files would only have outflow.)
-  auto it = flows_.find(blob_file_number);
-  if (it == flows_.end()) {
-    return Status::OK();
-  }
-
-  it->second.AddOutFlow(bytes);
-
-  return Status::OK();
-}
-
-Status BlobGarbageMeter::Parse(const Slice& key, const Slice& value,
-                               uint64_t* blob_file_number, uint64_t* bytes) {
-  assert(blob_file_number);
-  assert(*blob_file_number == kInvalidBlobFileNumber);
-  assert(bytes);
-  assert(*bytes == 0);
-
-  ParsedInternalKey ikey;
-
-  {
-    constexpr bool log_err_key = false;
-    const Status s = ParseInternalKey(key, &ikey, log_err_key);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (ikey.type != kTypeBlobIndex) {
-    return Status::OK();
-  }
-
-  BlobIndex blob_index;
-
-  {
-    const Status s = blob_index.DecodeFrom(value);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (blob_index.IsInlined() || blob_index.HasTTL()) {
-    return Status::Corruption("Unexpected TTL/inlined blob index");
-  }
-
-  *blob_file_number = blob_index.file_number();
-  *bytes =
-      blob_index.size() +
-      BlobLogRecord::CalculateAdjustmentForRecordHeader(ikey.user_key.size());
-
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_garbage_meter.h

@@ -1,102 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-#include <cstdint>
-#include <unordered_map>
-
-#include "db/blob/blob_constants.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Slice;
-
-// A class that can be used to compute the amount of additional garbage
-// generated by a compaction. It parses the keys and blob references in the
-// input and output of a compaction, and aggregates the "inflow" and "outflow"
-// on a per-blob file basis. The amount of additional garbage for any given blob
-// file can then be computed by subtracting the outflow from the inflow.
-class BlobGarbageMeter {
- public:
-  // A class to store the number and total size of blobs on a per-blob file
-  // basis.
-  class BlobStats {
-   public:
-    void Add(uint64_t bytes) {
-      ++count_;
-      bytes_ += bytes;
-    }
-    void Add(uint64_t count, uint64_t bytes) {
-      count_ += count;
-      bytes_ += bytes;
-    }
-
-    uint64_t GetCount() const { return count_; }
-    uint64_t GetBytes() const { return bytes_; }
-
-   private:
-    uint64_t count_ = 0;
-    uint64_t bytes_ = 0;
-  };
-
-  // A class to keep track of the "inflow" and the "outflow" and to compute the
-  // amount of additional garbage for a given blob file.
-  class BlobInOutFlow {
-   public:
-    void AddInFlow(uint64_t bytes) {
-      in_flow_.Add(bytes);
-      assert(IsValid());
-    }
-    void AddOutFlow(uint64_t bytes) {
-      out_flow_.Add(bytes);
-      assert(IsValid());
-    }
-
-    const BlobStats& GetInFlow() const { return in_flow_; }
-    const BlobStats& GetOutFlow() const { return out_flow_; }
-
-    bool IsValid() const {
-      return in_flow_.GetCount() >= out_flow_.GetCount() &&
-             in_flow_.GetBytes() >= out_flow_.GetBytes();
-    }
-    bool HasGarbage() const {
-      assert(IsValid());
-      return in_flow_.GetCount() > out_flow_.GetCount();
-    }
-    uint64_t GetGarbageCount() const {
-      assert(IsValid());
-      assert(HasGarbage());
-      return in_flow_.GetCount() - out_flow_.GetCount();
-    }
-    uint64_t GetGarbageBytes() const {
-      assert(IsValid());
-      assert(HasGarbage());
-      return in_flow_.GetBytes() - out_flow_.GetBytes();
-    }
-
-   private:
-    BlobStats in_flow_;
-    BlobStats out_flow_;
-  };
-
-  Status ProcessInFlow(const Slice& key, const Slice& value);
-  Status ProcessOutFlow(const Slice& key, const Slice& value);
-
-  const std::unordered_map<uint64_t, BlobInOutFlow>& flows() const {
-    return flows_;
-  }
-
- private:
-  static Status Parse(const Slice& key, const Slice& value,
-                      uint64_t* blob_file_number, uint64_t* bytes);
-
-  std::unordered_map<uint64_t, BlobInOutFlow> flows_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_garbage_meter_test.cc

@@ -1,196 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_garbage_meter.h"
-
-#include <string>
-#include <vector>
-
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/dbformat.h"
-#include "test_util/testharness.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-TEST(BlobGarbageMeterTest, MeasureGarbage) {
-  BlobGarbageMeter blob_garbage_meter;
-
-  struct BlobDescriptor {
-    std::string user_key;
-    uint64_t blob_file_number;
-    uint64_t offset;
-    uint64_t size;
-    CompressionType compression_type;
-    bool has_in_flow;
-    bool has_out_flow;
-
-    uint64_t GetExpectedBytes() const {
-      return size +
-             BlobLogRecord::CalculateAdjustmentForRecordHeader(user_key.size());
-    }
-  };
-
-  // Note: blob file 4 has the same inflow and outflow and hence no additional
-  // garbage. Blob file 5 has less outflow than inflow and thus it does have
-  // additional garbage. Blob file 6 is a newly written file (i.e. no inflow,
-  // only outflow) and is thus not tracked by the meter.
-  std::vector<BlobDescriptor> blobs{
-      {"key", 4, 1234, 555, kLZ4Compression, true, true},
-      {"other_key", 4, 6789, 101010, kLZ4Compression, true, true},
-      {"yet_another_key", 5, 22222, 3456, kLZ4Compression, true, true},
-      {"foo_key", 5, 77777, 8888, kLZ4Compression, true, true},
-      {"bar_key", 5, 999999, 1212, kLZ4Compression, true, false},
-      {"baz_key", 5, 1234567, 890, kLZ4Compression, true, false},
-      {"new_key", 6, 7777, 9999, kNoCompression, false, true}};
-
-  for (const auto& blob : blobs) {
-    constexpr SequenceNumber seq = 123;
-    const InternalKey key(blob.user_key, seq, kTypeBlobIndex);
-    const Slice key_slice = key.Encode();
-
-    std::string value;
-    BlobIndex::EncodeBlob(&value, blob.blob_file_number, blob.offset, blob.size,
-                          blob.compression_type);
-    const Slice value_slice(value);
-
-    if (blob.has_in_flow) {
-      ASSERT_OK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice));
-    }
-    if (blob.has_out_flow) {
-      ASSERT_OK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice));
-    }
-  }
-
-  const auto& flows = blob_garbage_meter.flows();
-  ASSERT_EQ(flows.size(), 2);
-
-  {
-    const auto it = flows.find(4);
-    ASSERT_NE(it, flows.end());
-
-    const auto& flow = it->second;
-
-    constexpr uint64_t expected_count = 2;
-    const uint64_t expected_bytes =
-        blobs[0].GetExpectedBytes() + blobs[1].GetExpectedBytes();
-
-    const auto& in = flow.GetInFlow();
-    ASSERT_EQ(in.GetCount(), expected_count);
-    ASSERT_EQ(in.GetBytes(), expected_bytes);
-
-    const auto& out = flow.GetOutFlow();
-    ASSERT_EQ(out.GetCount(), expected_count);
-    ASSERT_EQ(out.GetBytes(), expected_bytes);
-
-    ASSERT_TRUE(flow.IsValid());
-    ASSERT_FALSE(flow.HasGarbage());
-  }
-
-  {
-    const auto it = flows.find(5);
-    ASSERT_NE(it, flows.end());
-
-    const auto& flow = it->second;
-
-    const auto& in = flow.GetInFlow();
-
-    constexpr uint64_t expected_in_count = 4;
-    const uint64_t expected_in_bytes =
-        blobs[2].GetExpectedBytes() + blobs[3].GetExpectedBytes() +
-        blobs[4].GetExpectedBytes() + blobs[5].GetExpectedBytes();
-
-    ASSERT_EQ(in.GetCount(), expected_in_count);
-    ASSERT_EQ(in.GetBytes(), expected_in_bytes);
-
-    const auto& out = flow.GetOutFlow();
-
-    constexpr uint64_t expected_out_count = 2;
-    const uint64_t expected_out_bytes =
-        blobs[2].GetExpectedBytes() + blobs[3].GetExpectedBytes();
-
-    ASSERT_EQ(out.GetCount(), expected_out_count);
-    ASSERT_EQ(out.GetBytes(), expected_out_bytes);
-
-    ASSERT_TRUE(flow.IsValid());
-    ASSERT_TRUE(flow.HasGarbage());
-    ASSERT_EQ(flow.GetGarbageCount(), expected_in_count - expected_out_count);
-    ASSERT_EQ(flow.GetGarbageBytes(), expected_in_bytes - expected_out_bytes);
-  }
-}
-
-TEST(BlobGarbageMeterTest, PlainValue) {
-  constexpr char user_key[] = "user_key";
-  constexpr SequenceNumber seq = 123;
-
-  const InternalKey key(user_key, seq, kTypeValue);
-  const Slice key_slice = key.Encode();
-
-  constexpr char value[] = "value";
-  const Slice value_slice(value);
-
-  BlobGarbageMeter blob_garbage_meter;
-
-  ASSERT_OK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice));
-  ASSERT_OK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice));
-  ASSERT_TRUE(blob_garbage_meter.flows().empty());
-}
-
-TEST(BlobGarbageMeterTest, CorruptInternalKey) {
-  constexpr char corrupt_key[] = "i_am_corrupt";
-  const Slice key_slice(corrupt_key);
-
-  constexpr char value[] = "value";
-  const Slice value_slice(value);
-
-  BlobGarbageMeter blob_garbage_meter;
-
-  ASSERT_NOK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice));
-  ASSERT_NOK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice));
-}
-
-TEST(BlobGarbageMeterTest, CorruptBlobIndex) {
-  constexpr char user_key[] = "user_key";
-  constexpr SequenceNumber seq = 123;
-
-  const InternalKey key(user_key, seq, kTypeBlobIndex);
-  const Slice key_slice = key.Encode();
-
-  constexpr char value[] = "i_am_not_a_blob_index";
-  const Slice value_slice(value);
-
-  BlobGarbageMeter blob_garbage_meter;
-
-  ASSERT_NOK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice));
-  ASSERT_NOK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice));
-}
-
-TEST(BlobGarbageMeterTest, InlinedTTLBlobIndex) {
-  constexpr char user_key[] = "user_key";
-  constexpr SequenceNumber seq = 123;
-
-  const InternalKey key(user_key, seq, kTypeBlobIndex);
-  const Slice key_slice = key.Encode();
-
-  constexpr uint64_t expiration = 1234567890;
-  constexpr char inlined_value[] = "inlined";
-
-  std::string value;
-  BlobIndex::EncodeInlinedTTL(&value, expiration, inlined_value);
-
-  const Slice value_slice(value);
-
-  BlobGarbageMeter blob_garbage_meter;
-
-  ASSERT_NOK(blob_garbage_meter.ProcessInFlow(key_slice, value_slice));
-  ASSERT_NOK(blob_garbage_meter.ProcessOutFlow(key_slice, value_slice));
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_index.h

@@ -3,13 +3,13 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 #pragma once
+#ifndef ROCKSDB_LITE
 
 #include <sstream>
 #include <string>
 
-#include "rocksdb/compression_type.h"
+#include "rocksdb/options.h"
 #include "util/coding.h"
-#include "util/compression.h"
 #include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -83,11 +83,6 @@ class BlobIndex {
     return size_;
   }
 
-  CompressionType compression() const {
-    assert(!IsInlined());
-    return compression_;
-  }
-
   Status DecodeFrom(Slice slice) {
     static const std::string kErrorMessage = "Error while decoding blob index";
     assert(slice.size() > 0);
@@ -123,8 +118,7 @@ class BlobIndex {
       oss << "[inlined blob] value:" << value_.ToString(output_hex);
     } else {
       oss << "[blob ref] file:" << file_number_ << " offset:" << offset_
-          << " size:" << size_
-          << " compression: " << CompressionTypeToString(compression_);
+          << " size:" << size_;
     }
 
     if (HasTTL()) {
@@ -182,3 +176,4 @@ class BlobIndex {
 };
 
 }  // namespace ROCKSDB_NAMESPACE
+#endif  // ROCKSDB_LITE

db/blob/blob_log_sequential_reader.h

@@ -1,78 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-#pragma once
-
-#include <memory>
-
-#include "db/blob/blob_log_format.h"
-#include "rocksdb/slice.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class RandomAccessFileReader;
-class Env;
-class Statistics;
-class Status;
-class SystemClock;
-
-/**
- * BlobLogSequentialReader is a general purpose log stream reader
- * implementation. The actual job of reading from the device is implemented by
- * the RandomAccessFileReader interface.
- *
- * Please see BlobLogWriter for details on the file and record layout.
- */
-
-class BlobLogSequentialReader {
- public:
-  enum ReadLevel {
-    kReadHeader,
-    kReadHeaderKey,
-    kReadHeaderKeyBlob,
-  };
-
-  // Create a reader that will return log records from "*file_reader".
-  BlobLogSequentialReader(std::unique_ptr<RandomAccessFileReader>&& file_reader,
-                          SystemClock* clock, Statistics* statistics);
-
-  // No copying allowed
-  BlobLogSequentialReader(const BlobLogSequentialReader&) = delete;
-  BlobLogSequentialReader& operator=(const BlobLogSequentialReader&) = delete;
-
-  ~BlobLogSequentialReader();
-
-  Status ReadHeader(BlobLogHeader* header);
-
-  // Read the next record into *record.  Returns true if read
-  // successfully, false if we hit end of the input. The contents filled in
-  // *record will only be valid until the next mutating operation on this
-  // reader.
-  // If blob_offset is non-null, return offset of the blob through it.
-  Status ReadRecord(BlobLogRecord* record, ReadLevel level = kReadHeader,
-                    uint64_t* blob_offset = nullptr);
-
-  Status ReadFooter(BlobLogFooter* footer);
-
-  void ResetNextByte() { next_byte_ = 0; }
-
-  uint64_t GetNextByte() const { return next_byte_; }
-
- private:
-  Status ReadSlice(uint64_t size, Slice* slice, char* buf);
-
-  const std::unique_ptr<RandomAccessFileReader> file_;
-  SystemClock* clock_;
-
-  Statistics* statistics_;
-
-  Slice buffer_;
-  char header_buf_[BlobLogRecord::kHeaderSize];
-
-  // which byte to read next
-  uint64_t next_byte_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/db_blob_basic_test.cc

@@ -1,517 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include <array>
-
-#include "db/blob/blob_index.h"
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "test_util/sync_point.h"
-#include "utilities/fault_injection_env.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class DBBlobBasicTest : public DBTestBase {
- protected:
-  DBBlobBasicTest()
-      : DBTestBase("/db_blob_basic_test", /* env_do_fsync */ false) {}
-};
-
-TEST_F(DBBlobBasicTest, GetBlob) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  constexpr char key[] = "key";
-  constexpr char blob_value[] = "blob_value";
-
-  ASSERT_OK(Put(key, blob_value));
-
-  ASSERT_OK(Flush());
-
-  ASSERT_EQ(Get(key), blob_value);
-
-  // Try again with no I/O allowed. The table and the necessary blocks should
-  // already be in their respective caches; however, the blob itself can only be
-  // read from the blob file, so the read should return Incomplete.
-  ReadOptions read_options;
-  read_options.read_tier = kBlockCacheTier;
-
-  PinnableSlice result;
-  ASSERT_TRUE(db_->Get(read_options, db_->DefaultColumnFamily(), key, &result)
-                  .IsIncomplete());
-}
-
-TEST_F(DBBlobBasicTest, MultiGetBlobs) {
-  constexpr size_t min_blob_size = 6;
-
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = min_blob_size;
-
-  Reopen(options);
-
-  // Put then retrieve three key-values. The first value is below the size limit
-  // and is thus stored inline; the other two are stored separately as blobs.
-  constexpr size_t num_keys = 3;
-
-  constexpr char first_key[] = "first_key";
-  constexpr char first_value[] = "short";
-  static_assert(sizeof(first_value) - 1 < min_blob_size,
-                "first_value too long to be inlined");
-
-  ASSERT_OK(Put(first_key, first_value));
-
-  constexpr char second_key[] = "second_key";
-  constexpr char second_value[] = "long_value";
-  static_assert(sizeof(second_value) - 1 >= min_blob_size,
-                "second_value too short to be stored as blob");
-
-  ASSERT_OK(Put(second_key, second_value));
-
-  constexpr char third_key[] = "third_key";
-  constexpr char third_value[] = "other_long_value";
-  static_assert(sizeof(third_value) - 1 >= min_blob_size,
-                "third_value too short to be stored as blob");
-
-  ASSERT_OK(Put(third_key, third_value));
-
-  ASSERT_OK(Flush());
-
-  ReadOptions read_options;
-
-  std::array<Slice, num_keys> keys{{first_key, second_key, third_key}};
-
-  {
-    std::array<PinnableSlice, num_keys> values;
-    std::array<Status, num_keys> statuses;
-
-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
-
-    ASSERT_OK(statuses[0]);
-    ASSERT_EQ(values[0], first_value);
-
-    ASSERT_OK(statuses[1]);
-    ASSERT_EQ(values[1], second_value);
-
-    ASSERT_OK(statuses[2]);
-    ASSERT_EQ(values[2], third_value);
-  }
-
-  // Try again with no I/O allowed. The table and the necessary blocks should
-  // already be in their respective caches. The first (inlined) value should be
-  // successfully read; however, the two blob values could only be read from the
-  // blob file, so for those the read should return Incomplete.
-  read_options.read_tier = kBlockCacheTier;
-
-  {
-    std::array<PinnableSlice, num_keys> values;
-    std::array<Status, num_keys> statuses;
-
-    db_->MultiGet(read_options, db_->DefaultColumnFamily(), num_keys, &keys[0],
-                  &values[0], &statuses[0]);
-
-    ASSERT_OK(statuses[0]);
-    ASSERT_EQ(values[0], first_value);
-
-    ASSERT_TRUE(statuses[1].IsIncomplete());
-
-    ASSERT_TRUE(statuses[2].IsIncomplete());
-  }
-}
-
-TEST_F(DBBlobBasicTest, GetBlob_CorruptIndex) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  constexpr char key[] = "key";
-
-  // Fake a corrupt blob index.
-  const std::string blob_index("foobar");
-
-  WriteBatch batch;
-  ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index));
-  ASSERT_OK(db_->Write(WriteOptions(), &batch));
-
-  ASSERT_OK(Flush());
-
-  PinnableSlice result;
-  ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result)
-                  .IsCorruption());
-}
-
-TEST_F(DBBlobBasicTest, GetBlob_InlinedTTLIndex) {
-  constexpr uint64_t min_blob_size = 10;
-
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = min_blob_size;
-
-  Reopen(options);
-
-  constexpr char key[] = "key";
-  constexpr char blob[] = "short";
-  static_assert(sizeof(short) - 1 < min_blob_size,
-                "Blob too long to be inlined");
-
-  // Fake an inlined TTL blob index.
-  std::string blob_index;
-
-  constexpr uint64_t expiration = 1234567890;
-
-  BlobIndex::EncodeInlinedTTL(&blob_index, expiration, blob);
-
-  WriteBatch batch;
-  ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index));
-  ASSERT_OK(db_->Write(WriteOptions(), &batch));
-
-  ASSERT_OK(Flush());
-
-  PinnableSlice result;
-  ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result)
-                  .IsCorruption());
-}
-
-TEST_F(DBBlobBasicTest, GetBlob_IndexWithInvalidFileNumber) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  constexpr char key[] = "key";
-
-  // Fake a blob index referencing a non-existent blob file.
-  std::string blob_index;
-
-  constexpr uint64_t blob_file_number = 1000;
-  constexpr uint64_t offset = 1234;
-  constexpr uint64_t size = 5678;
-
-  BlobIndex::EncodeBlob(&blob_index, blob_file_number, offset, size,
-                        kNoCompression);
-
-  WriteBatch batch;
-  ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index));
-  ASSERT_OK(db_->Write(WriteOptions(), &batch));
-
-  ASSERT_OK(Flush());
-
-  PinnableSlice result;
-  ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result)
-                  .IsCorruption());
-}
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBlobBasicTest, GenerateIOTracing) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  std::string trace_file = dbname_ + "/io_trace_file";
-
-  Reopen(options);
-  {
-    // Create IO trace file
-    std::unique_ptr<TraceWriter> trace_writer;
-    ASSERT_OK(
-        NewFileTraceWriter(env_, EnvOptions(), trace_file, &trace_writer));
-    ASSERT_OK(db_->StartIOTrace(TraceOptions(), std::move(trace_writer)));
-
-    constexpr char key[] = "key";
-    constexpr char blob_value[] = "blob_value";
-
-    ASSERT_OK(Put(key, blob_value));
-    ASSERT_OK(Flush());
-    ASSERT_EQ(Get(key), blob_value);
-
-    ASSERT_OK(db_->EndIOTrace());
-    ASSERT_OK(env_->FileExists(trace_file));
-  }
-  {
-    // Parse trace file to check file operations related to blob files are
-    // recorded.
-    std::unique_ptr<TraceReader> trace_reader;
-    ASSERT_OK(
-        NewFileTraceReader(env_, EnvOptions(), trace_file, &trace_reader));
-    IOTraceReader reader(std::move(trace_reader));
-
-    IOTraceHeader header;
-    ASSERT_OK(reader.ReadHeader(&header));
-    ASSERT_EQ(kMajorVersion, static_cast<int>(header.rocksdb_major_version));
-    ASSERT_EQ(kMinorVersion, static_cast<int>(header.rocksdb_minor_version));
-
-    // Read records.
-    int blob_files_op_count = 0;
-    Status status;
-    while (true) {
-      IOTraceRecord record;
-      status = reader.ReadIOOp(&record);
-      if (!status.ok()) {
-        break;
-      }
-      if (record.file_name.find("blob") != std::string::npos) {
-        blob_files_op_count++;
-      }
-    }
-    // Assuming blob files will have Append, Close and then Read operations.
-    ASSERT_GT(blob_files_op_count, 2);
-  }
-}
-#endif  // !ROCKSDB_LITE
-
-TEST_F(DBBlobBasicTest, BestEffortsRecovery_MissingNewestBlobFile) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.create_if_missing = true;
-  Reopen(options);
-
-  ASSERT_OK(dbfull()->DisableFileDeletions());
-  constexpr int kNumTableFiles = 2;
-  for (int i = 0; i < kNumTableFiles; ++i) {
-    for (char ch = 'a'; ch != 'c'; ++ch) {
-      std::string key(1, ch);
-      ASSERT_OK(Put(key, "value" + std::to_string(i)));
-    }
-    ASSERT_OK(Flush());
-  }
-
-  Close();
-
-  std::vector<std::string> files;
-  ASSERT_OK(env_->GetChildren(dbname_, &files));
-  std::string blob_file_path;
-  uint64_t max_blob_file_num = kInvalidBlobFileNumber;
-  for (const auto& fname : files) {
-    uint64_t file_num = 0;
-    FileType type;
-    if (ParseFileName(fname, &file_num, /*info_log_name_prefix=*/"", &type) &&
-        type == kBlobFile) {
-      if (file_num > max_blob_file_num) {
-        max_blob_file_num = file_num;
-        blob_file_path = dbname_ + "/" + fname;
-      }
-    }
-  }
-  ASSERT_OK(env_->DeleteFile(blob_file_path));
-
-  options.best_efforts_recovery = true;
-  Reopen(options);
-  std::string value;
-  ASSERT_OK(db_->Get(ReadOptions(), "a", &value));
-  ASSERT_EQ("value" + std::to_string(kNumTableFiles - 2), value);
-}
-
-TEST_F(DBBlobBasicTest, GetMergeBlobWithPut) {
-  Options options = GetDefaultOptions();
-  options.merge_operator = MergeOperators::CreateStringAppendOperator();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  ASSERT_OK(Put("Key1", "v1"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("Key1", "v2"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("Key1", "v3"));
-  ASSERT_OK(Flush());
-
-  std::string value;
-  ASSERT_OK(db_->Get(ReadOptions(), "Key1", &value));
-  ASSERT_EQ(Get("Key1"), "v1,v2,v3");
-}
-
-TEST_F(DBBlobBasicTest, MultiGetMergeBlobWithPut) {
-  constexpr size_t num_keys = 3;
-
-  Options options = GetDefaultOptions();
-  options.merge_operator = MergeOperators::CreateStringAppendOperator();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  ASSERT_OK(Put("Key0", "v0_0"));
-  ASSERT_OK(Put("Key1", "v1_0"));
-  ASSERT_OK(Put("Key2", "v2_0"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("Key0", "v0_1"));
-  ASSERT_OK(Merge("Key1", "v1_1"));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Merge("Key0", "v0_2"));
-  ASSERT_OK(Flush());
-
-  std::array<Slice, num_keys> keys{{"Key0", "Key1", "Key2"}};
-  std::array<PinnableSlice, num_keys> values;
-  std::array<Status, num_keys> statuses;
-
-  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0],
-                &values[0], &statuses[0]);
-
-  ASSERT_OK(statuses[0]);
-  ASSERT_EQ(values[0], "v0_0,v0_1,v0_2");
-
-  ASSERT_OK(statuses[1]);
-  ASSERT_EQ(values[1], "v1_0,v1_1");
-
-  ASSERT_OK(statuses[2]);
-  ASSERT_EQ(values[2], "v2_0");
-}
-
-class DBBlobBasicIOErrorTest : public DBBlobBasicTest,
-                               public testing::WithParamInterface<std::string> {
- protected:
-  DBBlobBasicIOErrorTest() : sync_point_(GetParam()) {
-    fault_injection_env_.reset(new FaultInjectionTestEnv(env_));
-  }
-  ~DBBlobBasicIOErrorTest() { Close(); }
-
-  std::unique_ptr<FaultInjectionTestEnv> fault_injection_env_;
-  std::string sync_point_;
-};
-
-INSTANTIATE_TEST_CASE_P(DBBlobBasicTest, DBBlobBasicIOErrorTest,
-                        ::testing::ValuesIn(std::vector<std::string>{
-                            "BlobFileReader::OpenFile:NewRandomAccessFile",
-                            "BlobFileReader::GetBlob:ReadFromFile"}));
-
-TEST_P(DBBlobBasicIOErrorTest, GetBlob_IOError) {
-  Options options;
-  options.env = fault_injection_env_.get();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  constexpr char key[] = "key";
-  constexpr char blob_value[] = "blob_value";
-
-  ASSERT_OK(Put(key, blob_value));
-
-  ASSERT_OK(Flush());
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
-    fault_injection_env_->SetFilesystemActive(false,
-                                              Status::IOError(sync_point_));
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  PinnableSlice result;
-  ASSERT_TRUE(db_->Get(ReadOptions(), db_->DefaultColumnFamily(), key, &result)
-                  .IsIOError());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_P(DBBlobBasicIOErrorTest, MultiGetBlobs_IOError) {
-  Options options = GetDefaultOptions();
-  options.env = fault_injection_env_.get();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  Reopen(options);
-
-  constexpr size_t num_keys = 2;
-
-  constexpr char first_key[] = "first_key";
-  constexpr char first_value[] = "first_value";
-
-  ASSERT_OK(Put(first_key, first_value));
-
-  constexpr char second_key[] = "second_key";
-  constexpr char second_value[] = "second_value";
-
-  ASSERT_OK(Put(second_key, second_value));
-
-  ASSERT_OK(Flush());
-
-  std::array<Slice, num_keys> keys{{first_key, second_key}};
-  std::array<PinnableSlice, num_keys> values;
-  std::array<Status, num_keys> statuses;
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
-    fault_injection_env_->SetFilesystemActive(false,
-                                              Status::IOError(sync_point_));
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  db_->MultiGet(ReadOptions(), db_->DefaultColumnFamily(), num_keys, &keys[0],
-                &values[0], &statuses[0]);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  ASSERT_TRUE(statuses[0].IsIOError());
-  ASSERT_TRUE(statuses[1].IsIOError());
-}
-
-namespace {
-
-class ReadBlobCompactionFilter : public CompactionFilter {
- public:
-  ReadBlobCompactionFilter() = default;
-  const char* Name() const override {
-    return "rocksdb.compaction.filter.read.blob";
-  }
-  CompactionFilter::Decision FilterV2(
-      int /*level*/, const Slice& /*key*/, ValueType value_type,
-      const Slice& existing_value, std::string* new_value,
-      std::string* /*skip_until*/) const override {
-    if (value_type != CompactionFilter::ValueType::kValue) {
-      return CompactionFilter::Decision::kKeep;
-    }
-    assert(new_value);
-    new_value->assign(existing_value.data(), existing_value.size());
-    return CompactionFilter::Decision::kChangeValue;
-  }
-};
-
-}  // anonymous namespace
-
-TEST_P(DBBlobBasicIOErrorTest, CompactionFilterReadBlob_IOError) {
-  Options options = GetDefaultOptions();
-  options.env = fault_injection_env_.get();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.create_if_missing = true;
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new ReadBlobCompactionFilter);
-  options.compaction_filter = compaction_filter_guard.get();
-
-  DestroyAndReopen(options);
-  constexpr char key[] = "foo";
-  constexpr char blob_value[] = "foo_blob_value";
-  ASSERT_OK(Put(key, blob_value));
-  ASSERT_OK(Flush());
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* /* arg */) {
-    fault_injection_env_->SetFilesystemActive(false,
-                                              Status::IOError(sync_point_));
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                                /*end=*/nullptr)
-                  .IsIOError());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/db_blob_compaction_test.cc

@@ -1,600 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "test_util/sync_point.h"
-#include "utilities/fault_injection_env.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class DBBlobCompactionTest : public DBTestBase {
- public:
-  explicit DBBlobCompactionTest()
-      : DBTestBase("/db_blob_compaction_test", /*env_do_fsync=*/false) {}
-
-#ifndef ROCKSDB_LITE
-  const std::vector<InternalStats::CompactionStats>& GetCompactionStats() {
-    VersionSet* const versions = dbfull()->TEST_GetVersionSet();
-    assert(versions);
-    assert(versions->GetColumnFamilySet());
-
-    ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
-    assert(cfd);
-
-    const InternalStats* const internal_stats = cfd->internal_stats();
-    assert(internal_stats);
-
-    return internal_stats->TEST_GetCompactionStats();
-  }
-#endif  // ROCKSDB_LITE
-};
-
-namespace {
-
-class FilterByKeyLength : public CompactionFilter {
- public:
-  explicit FilterByKeyLength(size_t len) : length_threshold_(len) {}
-  const char* Name() const override {
-    return "rocksdb.compaction.filter.by.key.length";
-  }
-  CompactionFilter::Decision FilterBlobByKey(
-      int /*level*/, const Slice& key, std::string* /*new_value*/,
-      std::string* /*skip_until*/) const override {
-    if (key.size() < length_threshold_) {
-      return CompactionFilter::Decision::kRemove;
-    }
-    return CompactionFilter::Decision::kKeep;
-  }
-
- private:
-  size_t length_threshold_;
-};
-
-class BadBlobCompactionFilter : public CompactionFilter {
- public:
-  explicit BadBlobCompactionFilter(std::string prefix,
-                                   CompactionFilter::Decision filter_by_key,
-                                   CompactionFilter::Decision filter_v2)
-      : prefix_(std::move(prefix)),
-        filter_blob_by_key_(filter_by_key),
-        filter_v2_(filter_v2) {}
-  const char* Name() const override { return "rocksdb.compaction.filter.bad"; }
-  CompactionFilter::Decision FilterBlobByKey(
-      int /*level*/, const Slice& key, std::string* /*new_value*/,
-      std::string* /*skip_until*/) const override {
-    if (key.size() >= prefix_.size() &&
-        0 == strncmp(prefix_.data(), key.data(), prefix_.size())) {
-      return CompactionFilter::Decision::kUndetermined;
-    }
-    return filter_blob_by_key_;
-  }
-  CompactionFilter::Decision FilterV2(
-      int /*level*/, const Slice& /*key*/, ValueType /*value_type*/,
-      const Slice& /*existing_value*/, std::string* /*new_value*/,
-      std::string* /*skip_until*/) const override {
-    return filter_v2_;
-  }
-
- private:
-  const std::string prefix_;
-  const CompactionFilter::Decision filter_blob_by_key_;
-  const CompactionFilter::Decision filter_v2_;
-};
-
-class ValueBlindWriteFilter : public CompactionFilter {
- public:
-  explicit ValueBlindWriteFilter(std::string new_val)
-      : new_value_(std::move(new_val)) {}
-  const char* Name() const override {
-    return "rocksdb.compaction.filter.blind.write";
-  }
-  CompactionFilter::Decision FilterBlobByKey(
-      int level, const Slice& key, std::string* new_value,
-      std::string* skip_until) const override;
-
- private:
-  const std::string new_value_;
-};
-
-CompactionFilter::Decision ValueBlindWriteFilter::FilterBlobByKey(
-    int /*level*/, const Slice& /*key*/, std::string* new_value,
-    std::string* /*skip_until*/) const {
-  assert(new_value);
-  new_value->assign(new_value_);
-  return CompactionFilter::Decision::kChangeValue;
-}
-
-class ValueMutationFilter : public CompactionFilter {
- public:
-  explicit ValueMutationFilter(std::string padding)
-      : padding_(std::move(padding)) {}
-  const char* Name() const override {
-    return "rocksdb.compaction.filter.value.mutation";
-  }
-  CompactionFilter::Decision FilterV2(int level, const Slice& key,
-                                      ValueType value_type,
-                                      const Slice& existing_value,
-                                      std::string* new_value,
-                                      std::string* skip_until) const override;
-
- private:
-  const std::string padding_;
-};
-
-CompactionFilter::Decision ValueMutationFilter::FilterV2(
-    int /*level*/, const Slice& /*key*/, ValueType value_type,
-    const Slice& existing_value, std::string* new_value,
-    std::string* /*skip_until*/) const {
-  assert(CompactionFilter::ValueType::kBlobIndex != value_type);
-  if (CompactionFilter::ValueType::kValue != value_type) {
-    return CompactionFilter::Decision::kKeep;
-  }
-  assert(new_value);
-  new_value->assign(existing_value.data(), existing_value.size());
-  new_value->append(padding_);
-  return CompactionFilter::Decision::kChangeValue;
-}
-
-class AlwaysKeepFilter : public CompactionFilter {
- public:
-  explicit AlwaysKeepFilter() = default;
-  const char* Name() const override {
-    return "rocksdb.compaction.filter.always.keep";
-  }
-  CompactionFilter::Decision FilterV2(
-      int /*level*/, const Slice& /*key*/, ValueType /*value_type*/,
-      const Slice& /*existing_value*/, std::string* /*new_value*/,
-      std::string* /*skip_until*/) const override {
-    return CompactionFilter::Decision::kKeep;
-  }
-};
-
-class SkipUntilFilter : public CompactionFilter {
- public:
-  explicit SkipUntilFilter(std::string skip_until)
-      : skip_until_(std::move(skip_until)) {}
-
-  const char* Name() const override {
-    return "rocksdb.compaction.filter.skip.until";
-  }
-
-  CompactionFilter::Decision FilterV2(int /* level */, const Slice& /* key */,
-                                      ValueType /* value_type */,
-                                      const Slice& /* existing_value */,
-                                      std::string* /* new_value */,
-                                      std::string* skip_until) const override {
-    assert(skip_until);
-    *skip_until = skip_until_;
-
-    return CompactionFilter::Decision::kRemoveAndSkipUntil;
-  }
-
- private:
-  std::string skip_until_;
-};
-
-}  // anonymous namespace
-
-class DBBlobBadCompactionFilterTest
-    : public DBBlobCompactionTest,
-      public testing::WithParamInterface<
-          std::tuple<std::string, CompactionFilter::Decision,
-                     CompactionFilter::Decision>> {
- public:
-  explicit DBBlobBadCompactionFilterTest()
-      : compaction_filter_guard_(new BadBlobCompactionFilter(
-            std::get<0>(GetParam()), std::get<1>(GetParam()),
-            std::get<2>(GetParam()))) {}
-
- protected:
-  std::unique_ptr<CompactionFilter> compaction_filter_guard_;
-};
-
-INSTANTIATE_TEST_CASE_P(
-    BadCompactionFilter, DBBlobBadCompactionFilterTest,
-    testing::Combine(
-        testing::Values("a"),
-        testing::Values(CompactionFilter::Decision::kChangeBlobIndex,
-                        CompactionFilter::Decision::kIOError),
-        testing::Values(CompactionFilter::Decision::kUndetermined,
-                        CompactionFilter::Decision::kChangeBlobIndex,
-                        CompactionFilter::Decision::kIOError)));
-
-TEST_F(DBBlobCompactionTest, FilterByKeyLength) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.create_if_missing = true;
-  constexpr size_t kKeyLength = 2;
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new FilterByKeyLength(kKeyLength));
-  options.compaction_filter = compaction_filter_guard.get();
-
-  constexpr char short_key[] = "a";
-  constexpr char long_key[] = "abc";
-  constexpr char blob_value[] = "value";
-
-  DestroyAndReopen(options);
-  ASSERT_OK(Put(short_key, blob_value));
-  ASSERT_OK(Put(long_key, blob_value));
-  ASSERT_OK(Flush());
-  CompactRangeOptions cro;
-  ASSERT_OK(db_->CompactRange(cro, /*begin=*/nullptr, /*end=*/nullptr));
-  std::string value;
-  ASSERT_TRUE(db_->Get(ReadOptions(), short_key, &value).IsNotFound());
-  value.clear();
-  ASSERT_OK(db_->Get(ReadOptions(), long_key, &value));
-  ASSERT_EQ("value", value);
-
-#ifndef ROCKSDB_LITE
-  const auto& compaction_stats = GetCompactionStats();
-  ASSERT_GE(compaction_stats.size(), 2);
-
-  // Filter decides between kKeep and kRemove solely based on key;
-  // this involves neither reading nor writing blobs
-  ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0);
-  ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0);
-#endif  // ROCKSDB_LITE
-
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, BlindWriteFilter) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.create_if_missing = true;
-  constexpr char new_blob_value[] = "new_blob_value";
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new ValueBlindWriteFilter(new_blob_value));
-  options.compaction_filter = compaction_filter_guard.get();
-  DestroyAndReopen(options);
-  const std::vector<std::string> keys = {"a", "b", "c"};
-  const std::vector<std::string> values = {"a_value", "b_value", "c_value"};
-  assert(keys.size() == values.size());
-  for (size_t i = 0; i < keys.size(); ++i) {
-    ASSERT_OK(Put(keys[i], values[i]));
-  }
-  ASSERT_OK(Flush());
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                              /*end=*/nullptr));
-  for (const auto& key : keys) {
-    ASSERT_EQ(new_blob_value, Get(key));
-  }
-
-#ifndef ROCKSDB_LITE
-  const auto& compaction_stats = GetCompactionStats();
-  ASSERT_GE(compaction_stats.size(), 2);
-
-  // Filter unconditionally changes value in FilterBlobByKey;
-  // this involves writing but not reading blobs
-  ASSERT_EQ(compaction_stats[1].bytes_read_blob, 0);
-  ASSERT_GT(compaction_stats[1].bytes_written_blob, 0);
-#endif  // ROCKSDB_LITE
-
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, SkipUntilFilter) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new SkipUntilFilter("z"));
-  options.compaction_filter = compaction_filter_guard.get();
-
-  Reopen(options);
-
-  const std::vector<std::string> keys{"a", "b", "c"};
-  const std::vector<std::string> values{"a_value", "b_value", "c_value"};
-  assert(keys.size() == values.size());
-
-  for (size_t i = 0; i < keys.size(); ++i) {
-    ASSERT_OK(Put(keys[i], values[i]));
-  }
-
-  ASSERT_OK(Flush());
-
-  int process_in_flow_called = 0;
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobCountingIterator::UpdateAndCountBlobIfNeeded:ProcessInFlow",
-      [&process_in_flow_called](void* /* arg */) { ++process_in_flow_called; });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /* begin */ nullptr,
-                              /* end */ nullptr));
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  for (const auto& key : keys) {
-    ASSERT_EQ(Get(key), "NOT_FOUND");
-  }
-
-  // Make sure SkipUntil was performed using iteration rather than Seek
-  ASSERT_EQ(process_in_flow_called, keys.size());
-
-  Close();
-}
-
-TEST_P(DBBlobBadCompactionFilterTest, BadDecisionFromCompactionFilter) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.create_if_missing = true;
-  options.compaction_filter = compaction_filter_guard_.get();
-  DestroyAndReopen(options);
-  ASSERT_OK(Put("b", "value"));
-  ASSERT_OK(Flush());
-  ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                                /*end=*/nullptr)
-                  .IsNotSupported());
-  Close();
-
-  DestroyAndReopen(options);
-  std::string key(std::get<0>(GetParam()));
-  ASSERT_OK(Put(key, "value"));
-  ASSERT_OK(Flush());
-  ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                                /*end=*/nullptr)
-                  .IsNotSupported());
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, CompactionFilter_InlinedTTLIndex) {
-  Options options = GetDefaultOptions();
-  options.create_if_missing = true;
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new ValueMutationFilter(""));
-  options.compaction_filter = compaction_filter_guard.get();
-  DestroyAndReopen(options);
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-  // Fake an inlined TTL blob index.
-  std::string blob_index;
-  constexpr uint64_t expiration = 1234567890;
-  BlobIndex::EncodeInlinedTTL(&blob_index, expiration, blob);
-  WriteBatch batch;
-  ASSERT_OK(WriteBatchInternal::PutBlobIndex(&batch, 0, key, blob_index));
-  ASSERT_OK(db_->Write(WriteOptions(), &batch));
-  ASSERT_OK(Flush());
-  ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                                /*end=*/nullptr)
-                  .IsCorruption());
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, CompactionFilter) {
-  Options options = GetDefaultOptions();
-  options.create_if_missing = true;
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  constexpr char padding[] = "_delta";
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new ValueMutationFilter(padding));
-  options.compaction_filter = compaction_filter_guard.get();
-  DestroyAndReopen(options);
-  const std::vector<std::pair<std::string, std::string>> kvs = {
-      {"a", "a_value"}, {"b", "b_value"}, {"c", "c_value"}};
-  for (const auto& kv : kvs) {
-    ASSERT_OK(Put(kv.first, kv.second));
-  }
-  ASSERT_OK(Flush());
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                              /*end=*/nullptr));
-  for (const auto& kv : kvs) {
-    ASSERT_EQ(kv.second + std::string(padding), Get(kv.first));
-  }
-
-#ifndef ROCKSDB_LITE
-  const auto& compaction_stats = GetCompactionStats();
-  ASSERT_GE(compaction_stats.size(), 2);
-
-  // Filter changes the value using the previous value in FilterV2;
-  // this involves reading and writing blobs
-  ASSERT_GT(compaction_stats[1].bytes_read_blob, 0);
-  ASSERT_GT(compaction_stats[1].bytes_written_blob, 0);
-#endif  // ROCKSDB_LITE
-
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, CorruptedBlobIndex) {
-  Options options = GetDefaultOptions();
-  options.create_if_missing = true;
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new ValueMutationFilter(""));
-  options.compaction_filter = compaction_filter_guard.get();
-  DestroyAndReopen(options);
-  // Mock a corrupted blob index
-  constexpr char key[] = "key";
-  std::string blob_idx("blob_idx");
-  WriteBatch write_batch;
-  ASSERT_OK(WriteBatchInternal::PutBlobIndex(&write_batch, 0, key, blob_idx));
-  ASSERT_OK(db_->Write(WriteOptions(), &write_batch));
-  ASSERT_OK(Flush());
-  ASSERT_TRUE(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                                /*end=*/nullptr)
-                  .IsCorruption());
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, CompactionFilterReadBlobAndKeep) {
-  Options options = GetDefaultOptions();
-  options.create_if_missing = true;
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  std::unique_ptr<CompactionFilter> compaction_filter_guard(
-      new AlwaysKeepFilter());
-  options.compaction_filter = compaction_filter_guard.get();
-  DestroyAndReopen(options);
-  ASSERT_OK(Put("foo", "foo_value"));
-  ASSERT_OK(Flush());
-  std::vector<uint64_t> blob_files = GetBlobFileNumbers();
-  ASSERT_EQ(1, blob_files.size());
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                              /*end=*/nullptr));
-  ASSERT_EQ(blob_files, GetBlobFileNumbers());
-
-#ifndef ROCKSDB_LITE
-  const auto& compaction_stats = GetCompactionStats();
-  ASSERT_GE(compaction_stats.size(), 2);
-
-  // Filter decides to keep the existing value in FilterV2;
-  // this involves reading but not writing blobs
-  ASSERT_GT(compaction_stats[1].bytes_read_blob, 0);
-  ASSERT_EQ(compaction_stats[1].bytes_written_blob, 0);
-#endif  // ROCKSDB_LITE
-
-  Close();
-}
-
-TEST_F(DBBlobCompactionTest, TrackGarbage) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-
-  Reopen(options);
-
-  // First table+blob file pair: 4 blobs with different keys
-  constexpr char first_key[] = "first_key";
-  constexpr char first_value[] = "first_value";
-  constexpr char second_key[] = "second_key";
-  constexpr char second_value[] = "second_value";
-  constexpr char third_key[] = "third_key";
-  constexpr char third_value[] = "third_value";
-  constexpr char fourth_key[] = "fourth_key";
-  constexpr char fourth_value[] = "fourth_value";
-
-  ASSERT_OK(Put(first_key, first_value));
-  ASSERT_OK(Put(second_key, second_value));
-  ASSERT_OK(Put(third_key, third_value));
-  ASSERT_OK(Put(fourth_key, fourth_value));
-  ASSERT_OK(Flush());
-
-  // Second table+blob file pair: overwrite 2 existing keys
-  constexpr char new_first_value[] = "new_first_value";
-  constexpr char new_second_value[] = "new_second_value";
-
-  ASSERT_OK(Put(first_key, new_first_value));
-  ASSERT_OK(Put(second_key, new_second_value));
-  ASSERT_OK(Flush());
-
-  // Compact them together. The first blob file should have 2 garbage blobs
-  // corresponding to the 2 overwritten keys.
-  constexpr Slice* begin = nullptr;
-  constexpr Slice* end = nullptr;
-
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), begin, end));
-
-  VersionSet* const versions = dbfull()->TEST_GetVersionSet();
-  assert(versions);
-  assert(versions->GetColumnFamilySet());
-
-  ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault();
-  assert(cfd);
-
-  Version* const current = cfd->current();
-  assert(current);
-
-  const VersionStorageInfo* const storage_info = current->storage_info();
-  assert(storage_info);
-
-  const auto& blob_files = storage_info->GetBlobFiles();
-  ASSERT_EQ(blob_files.size(), 2);
-
-  {
-    auto it = blob_files.begin();
-    const auto& meta = it->second;
-    assert(meta);
-
-    constexpr uint64_t first_expected_bytes =
-        sizeof(first_value) - 1 +
-        BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(first_key) -
-                                                          1);
-    constexpr uint64_t second_expected_bytes =
-        sizeof(second_value) - 1 +
-        BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(second_key) -
-                                                          1);
-    constexpr uint64_t third_expected_bytes =
-        sizeof(third_value) - 1 +
-        BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(third_key) -
-                                                          1);
-    constexpr uint64_t fourth_expected_bytes =
-        sizeof(fourth_value) - 1 +
-        BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(fourth_key) -
-                                                          1);
-
-    ASSERT_EQ(meta->GetTotalBlobCount(), 4);
-    ASSERT_EQ(meta->GetTotalBlobBytes(),
-              first_expected_bytes + second_expected_bytes +
-                  third_expected_bytes + fourth_expected_bytes);
-    ASSERT_EQ(meta->GetGarbageBlobCount(), 2);
-    ASSERT_EQ(meta->GetGarbageBlobBytes(),
-              first_expected_bytes + second_expected_bytes);
-  }
-
-  {
-    auto it = blob_files.rbegin();
-    const auto& meta = it->second;
-    assert(meta);
-
-    constexpr uint64_t new_first_expected_bytes =
-        sizeof(new_first_value) - 1 +
-        BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(first_key) -
-                                                          1);
-    constexpr uint64_t new_second_expected_bytes =
-        sizeof(new_second_value) - 1 +
-        BlobLogRecord::CalculateAdjustmentForRecordHeader(sizeof(second_key) -
-                                                          1);
-
-    ASSERT_EQ(meta->GetTotalBlobCount(), 2);
-    ASSERT_EQ(meta->GetTotalBlobBytes(),
-              new_first_expected_bytes + new_second_expected_bytes);
-    ASSERT_EQ(meta->GetGarbageBlobCount(), 0);
-    ASSERT_EQ(meta->GetGarbageBlobBytes(), 0);
-  }
-}
-
-TEST_F(DBBlobCompactionTest, MergeBlobWithBase) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.merge_operator = MergeOperators::CreateStringAppendOperator();
-  options.disable_auto_compactions = true;
-
-  Reopen(options);
-  ASSERT_OK(Put("Key1", "v1_1"));
-  ASSERT_OK(Put("Key2", "v2_1"));
-  ASSERT_OK(Flush());
-
-  ASSERT_OK(Merge("Key1", "v1_2"));
-  ASSERT_OK(Merge("Key2", "v2_2"));
-  ASSERT_OK(Flush());
-
-  ASSERT_OK(Merge("Key1", "v1_3"));
-  ASSERT_OK(Flush());
-
-  ASSERT_OK(db_->CompactRange(CompactRangeOptions(), /*begin=*/nullptr,
-                              /*end=*/nullptr));
-  ASSERT_EQ(Get("Key1"), "v1_1,v1_2,v1_3");
-  ASSERT_EQ(Get("Key2"), "v2_1,v2_2");
-  Close();
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/db_blob_corruption_test.cc

@@ -1,81 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/db_test_util.h"
-#include "port/stack_trace.h"
-#include "test_util/sync_point.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class DBBlobCorruptionTest : public DBTestBase {
- protected:
-  DBBlobCorruptionTest()
-      : DBTestBase("/db_blob_corruption_test", /* env_do_fsync */ false) {}
-
-  void Corrupt(FileType filetype, int offset, int bytes_to_corrupt) {
-    // Pick file to corrupt
-    std::vector<std::string> filenames;
-    ASSERT_OK(env_->GetChildren(dbname_, &filenames));
-    uint64_t number;
-    FileType type;
-    std::string fname;
-    uint64_t picked_number = kInvalidBlobFileNumber;
-    for (size_t i = 0; i < filenames.size(); i++) {
-      if (ParseFileName(filenames[i], &number, &type) && type == filetype &&
-          number > picked_number) {  // Pick latest file
-        fname = dbname_ + "/" + filenames[i];
-        picked_number = number;
-      }
-    }
-    ASSERT_TRUE(!fname.empty()) << filetype;
-    ASSERT_OK(test::CorruptFile(env_, fname, offset, bytes_to_corrupt));
-  }
-};
-
-#ifndef ROCKSDB_LITE
-TEST_F(DBBlobCorruptionTest, VerifyWholeBlobFileChecksum) {
-  Options options = GetDefaultOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-  options.create_if_missing = true;
-  options.file_checksum_gen_factory =
-      ROCKSDB_NAMESPACE::GetFileChecksumGenCrc32cFactory();
-  Reopen(options);
-
-  ASSERT_OK(Put(Slice("key_1"), Slice("blob_value_1")));
-  ASSERT_OK(Flush());
-  ASSERT_OK(Put(Slice("key_2"), Slice("blob_value_2")));
-  ASSERT_OK(Flush());
-  ASSERT_OK(db_->VerifyFileChecksums(ReadOptions()));
-  Close();
-
-  Corrupt(kBlobFile, 0, 2);
-
-  ASSERT_OK(TryReopen(options));
-
-  int count{0};
-  SyncPoint::GetInstance()->SetCallBack(
-      "DBImpl::VerifyFullFileChecksum:mismatch", [&](void* arg) {
-        const Status* s = static_cast<Status*>(arg);
-        ASSERT_NE(s, nullptr);
-        ++count;
-        ASSERT_NOK(*s);
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_TRUE(db_->VerifyFileChecksums(ReadOptions()).IsCorruption());
-  ASSERT_EQ(1, count);
-
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
-  ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-#endif  // !ROCKSDB_LITE
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ROCKSDB_NAMESPACE::port::InstallStackTraceHandler();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/db_blob_index_test.cc

@@ -41,13 +41,12 @@ class DBBlobIndexTest : public DBTestBase {
                                        Tier::kImmutableMemtables,
                                        Tier::kL0SstFile, Tier::kLnSstFile};
 
-  DBBlobIndexTest()
-      : DBTestBase("/db_blob_index_test", /*env_do_fsync=*/true) {}
+  DBBlobIndexTest() : DBTestBase("/db_blob_index_test") {}
 
   ColumnFamilyHandle* cfh() { return dbfull()->DefaultColumnFamily(); }
 
   ColumnFamilyData* cfd() {
-    return static_cast_with_check<ColumnFamilyHandleImpl>(cfh())->cfd();
+    return reinterpret_cast<ColumnFamilyHandleImpl*>(cfh())->cfd();
   }
 
   Status PutBlobIndex(WriteBatch* batch, const Slice& key,
@@ -73,9 +72,6 @@ class DBBlobIndexTest : public DBTestBase {
     if (s.IsNotFound()) {
       return "NOT_FOUND";
     }
-    if (s.IsCorruption()) {
-      return "CORRUPTION";
-    }
     if (s.IsNotSupported()) {
       return "NOT_SUPPORTED";
     }
@@ -98,12 +94,11 @@ class DBBlobIndexTest : public DBTestBase {
   ArenaWrappedDBIter* GetBlobIterator() {
     return dbfull()->NewIteratorImpl(
         ReadOptions(), cfd(), dbfull()->GetLatestSequenceNumber(),
-        nullptr /*read_callback*/, true /*expose_blob_index*/);
+        nullptr /*read_callback*/, true /*allow_blob*/);
   }
 
   Options GetTestOptions() {
     Options options;
-    options.env = CurrentOptions().env;
     options.create_if_missing = true;
     options.num_levels = 2;
     options.disable_auto_compactions = true;
@@ -157,13 +152,8 @@ TEST_F(DBBlobIndexTest, Write) {
   }
 }
 
-// Note: the following test case pertains to the StackableDB-based BlobDB
-// implementation. Get should be able to return blob index if is_blob_index is
-// provided, otherwise it should return Status::NotSupported (when reading from
-// memtable) or Status::Corruption (when reading from SST). Reading from SST
-// returns Corruption because we can't differentiate between the application
-// accidentally opening the base DB of a stacked BlobDB and actual corruption
-// when using the integrated BlobDB.
+// Get should be able to return blob index if is_blob_index is provided,
+// otherwise return Status::NotSupported status.
 TEST_F(DBBlobIndexTest, Get) {
   for (auto tier : kAllTiers) {
     DestroyAndReopen(GetTestOptions());
@@ -180,22 +170,15 @@ TEST_F(DBBlobIndexTest, Get) {
     ASSERT_EQ("value", GetImpl("key", &is_blob_index));
     ASSERT_FALSE(is_blob_index);
     // Verify blob index
-    if (tier <= kImmutableMemtables) {
-      ASSERT_TRUE(Get("blob_key", &value).IsNotSupported());
-      ASSERT_EQ("NOT_SUPPORTED", GetImpl("blob_key"));
-    } else {
-      ASSERT_TRUE(Get("blob_key", &value).IsCorruption());
-      ASSERT_EQ("CORRUPTION", GetImpl("blob_key"));
-    }
+    ASSERT_TRUE(Get("blob_key", &value).IsNotSupported());
+    ASSERT_EQ("NOT_SUPPORTED", GetImpl("blob_key"));
     ASSERT_EQ("blob_index", GetImpl("blob_key", &is_blob_index));
     ASSERT_TRUE(is_blob_index);
   }
 }
 
-// Note: the following test case pertains to the StackableDB-based BlobDB
-// implementation. Get should NOT return Status::NotSupported/Status::Corruption
-// if blob index is updated with a normal value. See the test case above for
-// more details.
+// Get should NOT return Status::NotSupported if blob index is updated with
+// a normal value.
 TEST_F(DBBlobIndexTest, Updated) {
   for (auto tier : kAllTiers) {
     DestroyAndReopen(GetTestOptions());
@@ -222,11 +205,7 @@ TEST_F(DBBlobIndexTest, Updated) {
       ASSERT_EQ("blob_index", GetBlobIndex("key" + ToString(i), snapshot));
     }
     ASSERT_EQ("new_value", Get("key1"));
-    if (tier <= kImmutableMemtables) {
-      ASSERT_EQ("NOT_SUPPORTED", GetImpl("key2"));
-    } else {
-      ASSERT_EQ("CORRUPTION", GetImpl("key2"));
-    }
+    ASSERT_EQ("NOT_SUPPORTED", GetImpl("key2"));
     ASSERT_EQ("NOT_FOUND", Get("key3"));
     ASSERT_EQ("NOT_FOUND", Get("key4"));
     ASSERT_EQ("a,b,c", GetImpl("key5"));
@@ -238,11 +217,8 @@ TEST_F(DBBlobIndexTest, Updated) {
   }
 }
 
-// Note: the following test case pertains to the StackableDB-based BlobDB
-// implementation. When a blob iterator is used, it should set the
-// expose_blob_index flag for the underlying DBIter, and retrieve/return the
-// corresponding blob value. If a regular DBIter is created (i.e.
-// expose_blob_index is not set), it should return Status::Corruption.
+// Iterator should get blob value if allow_blob flag is set,
+// otherwise return Status::NotSupported status.
 TEST_F(DBBlobIndexTest, Iterate) {
   const std::vector<std::vector<ValueType>> data = {
       /*00*/ {kTypeValue},
@@ -305,7 +281,6 @@ TEST_F(DBBlobIndexTest, Iterate) {
                     std::function<void(Iterator*)> extra_check = nullptr) {
     // Seek
     auto* iterator = create_iterator();
-    ASSERT_OK(iterator->status());
     ASSERT_OK(iterator->Refresh());
     iterator->Seek(get_key(index));
     check_iterator(iterator, expected_status, forward_value);
@@ -319,7 +294,6 @@ TEST_F(DBBlobIndexTest, Iterate) {
     ASSERT_OK(iterator->Refresh());
     iterator->Seek(get_key(index - 1));
     ASSERT_TRUE(iterator->Valid());
-    ASSERT_OK(iterator->status());
     iterator->Next();
     check_iterator(iterator, expected_status, forward_value);
     if (extra_check) {
@@ -329,7 +303,6 @@ TEST_F(DBBlobIndexTest, Iterate) {
 
     // SeekForPrev
     iterator = create_iterator();
-    ASSERT_OK(iterator->status());
     ASSERT_OK(iterator->Refresh());
     iterator->SeekForPrev(get_key(index));
     check_iterator(iterator, expected_status, backward_value);
@@ -342,7 +315,6 @@ TEST_F(DBBlobIndexTest, Iterate) {
     iterator = create_iterator();
     iterator->Seek(get_key(index + 1));
     ASSERT_TRUE(iterator->Valid());
-    ASSERT_OK(iterator->status());
     iterator->Prev();
     check_iterator(iterator, expected_status, backward_value);
     if (extra_check) {
@@ -380,7 +352,7 @@ TEST_F(DBBlobIndexTest, Iterate) {
             ASSERT_OK(Write(&batch));
             break;
           default:
-            FAIL();
+            assert(false);
         };
       }
       snapshots.push_back(dbfull()->GetSnapshot());
@@ -391,15 +363,15 @@ TEST_F(DBBlobIndexTest, Iterate) {
     MoveDataTo(tier);
 
     // Normal iterator
-    verify(1, Status::kCorruption, "", "", create_normal_iterator);
-    verify(3, Status::kCorruption, "", "", create_normal_iterator);
+    verify(1, Status::kNotSupported, "", "", create_normal_iterator);
+    verify(3, Status::kNotSupported, "", "", create_normal_iterator);
     verify(5, Status::kOk, get_value(5, 0), get_value(5, 0),
            create_normal_iterator);
     verify(7, Status::kOk, get_value(8, 0), get_value(6, 0),
            create_normal_iterator);
     verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
            create_normal_iterator);
-    verify(11, Status::kCorruption, "", "", create_normal_iterator);
+    verify(11, Status::kNotSupported, "", "", create_normal_iterator);
     verify(13, Status::kOk,
            get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
            get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
@@ -418,11 +390,7 @@ TEST_F(DBBlobIndexTest, Iterate) {
            create_blob_iterator, check_is_blob(false));
     verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
            create_blob_iterator, check_is_blob(false));
-    if (tier <= kImmutableMemtables) {
-      verify(11, Status::kNotSupported, "", "", create_blob_iterator);
-    } else {
-      verify(11, Status::kCorruption, "", "", create_blob_iterator);
-    }
+    verify(11, Status::kNotSupported, "", "", create_blob_iterator);
     verify(13, Status::kOk,
            get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
            get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
@@ -444,11 +412,7 @@ TEST_F(DBBlobIndexTest, Iterate) {
            create_blob_iterator, check_is_blob(false));
     verify(9, Status::kOk, get_value(10, 0), get_value(8, 0),
            create_blob_iterator, check_is_blob(false));
-    if (tier <= kImmutableMemtables) {
-      verify(11, Status::kNotSupported, "", "", create_blob_iterator);
-    } else {
-      verify(11, Status::kCorruption, "", "", create_blob_iterator);
-    }
+    verify(11, Status::kNotSupported, "", "", create_blob_iterator);
     verify(13, Status::kOk,
            get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
            get_value(13, 2) + "," + get_value(13, 1) + "," + get_value(13, 0),
@@ -463,106 +427,6 @@ TEST_F(DBBlobIndexTest, Iterate) {
   }
 }
 
-TEST_F(DBBlobIndexTest, IntegratedBlobIterate) {
-  const std::vector<std::vector<std::string>> data = {
-      /*00*/ {"Put"},
-      /*01*/ {"Put", "Merge", "Merge", "Merge"},
-      /*02*/ {"Put"}};
-
-  auto get_key = [](size_t index) { return ("key" + std::to_string(index)); };
-
-  auto get_value = [&](size_t index, size_t version) {
-    return get_key(index) + "_value" + ToString(version);
-  };
-
-  auto check_iterator = [&](Iterator* iterator, Status expected_status,
-                            const Slice& expected_value) {
-    ASSERT_EQ(expected_status, iterator->status());
-    if (expected_status.ok()) {
-      ASSERT_TRUE(iterator->Valid());
-      ASSERT_EQ(expected_value, iterator->value());
-    } else {
-      ASSERT_FALSE(iterator->Valid());
-    }
-  };
-
-  auto verify = [&](size_t index, Status expected_status,
-                    const Slice& expected_value) {
-    // Seek
-    {
-      Iterator* iterator = db_->NewIterator(ReadOptions());
-      std::unique_ptr<Iterator> iterator_guard(iterator);
-      ASSERT_OK(iterator->status());
-      ASSERT_OK(iterator->Refresh());
-      iterator->Seek(get_key(index));
-      check_iterator(iterator, expected_status, expected_value);
-    }
-    // Next
-    {
-      Iterator* iterator = db_->NewIterator(ReadOptions());
-      std::unique_ptr<Iterator> iterator_guard(iterator);
-      ASSERT_OK(iterator->Refresh());
-      iterator->Seek(get_key(index - 1));
-      ASSERT_TRUE(iterator->Valid());
-      ASSERT_OK(iterator->status());
-      iterator->Next();
-      check_iterator(iterator, expected_status, expected_value);
-    }
-    // SeekForPrev
-    {
-      Iterator* iterator = db_->NewIterator(ReadOptions());
-      std::unique_ptr<Iterator> iterator_guard(iterator);
-      ASSERT_OK(iterator->status());
-      ASSERT_OK(iterator->Refresh());
-      iterator->SeekForPrev(get_key(index));
-      check_iterator(iterator, expected_status, expected_value);
-    }
-    // Prev
-    {
-      Iterator* iterator = db_->NewIterator(ReadOptions());
-      std::unique_ptr<Iterator> iterator_guard(iterator);
-      iterator->Seek(get_key(index + 1));
-      ASSERT_TRUE(iterator->Valid());
-      ASSERT_OK(iterator->status());
-      iterator->Prev();
-      check_iterator(iterator, expected_status, expected_value);
-    }
-  };
-
-  Options options = GetTestOptions();
-  options.enable_blob_files = true;
-  options.min_blob_size = 0;
-
-  DestroyAndReopen(options);
-
-  // fill data
-  for (size_t i = 0; i < data.size(); i++) {
-    for (size_t j = 0; j < data[i].size(); j++) {
-      std::string key = get_key(i);
-      std::string value = get_value(i, j);
-      if (data[i][j] == "Put") {
-        ASSERT_OK(Put(key, value));
-        ASSERT_OK(Flush());
-      } else if (data[i][j] == "Merge") {
-        ASSERT_OK(Merge(key, value));
-        ASSERT_OK(Flush());
-      }
-    }
-  }
-
-  std::string expected_value = get_value(1, 0) + "," + get_value(1, 1) + "," +
-                               get_value(1, 2) + "," + get_value(1, 3);
-  Status expected_status;
-  verify(1, expected_status, expected_value);
-
-#ifndef ROCKSDB_LITE
-  // Test DBIter::FindValueForCurrentKeyUsingSeek flow.
-  ASSERT_OK(dbfull()->SetOptions(cfh(),
-                                 {{"max_sequential_skip_in_iterations", "0"}}));
-  verify(1, expected_status, expected_value);
-#endif  // !ROCKSDB_LITE
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 int main(int argc, char** argv) {

db/builder.cc

@@ -13,23 +13,19 @@
 #include <deque>
 #include <vector>
 
-#include "db/blob/blob_file_builder.h"
 #include "db/compaction/compaction_iterator.h"
 #include "db/dbformat.h"
 #include "db/event_helpers.h"
 #include "db/internal_stats.h"
 #include "db/merge_helper.h"
-#include "db/output_validator.h"
 #include "db/range_del_aggregator.h"
 #include "db/table_cache.h"
 #include "db/version_edit.h"
-#include "file/file_util.h"
 #include "file/filename.h"
 #include "file/read_write_util.h"
 #include "file/writable_file_writer.h"
 #include "monitoring/iostats_context_imp.h"
 #include "monitoring/thread_status_util.h"
-#include "options/options_helper.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
@@ -45,171 +41,125 @@ namespace ROCKSDB_NAMESPACE {
 
 class TableFactory;
 
-TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions,
-                              WritableFileWriter* file) {
-  assert((tboptions.column_family_id ==
+TableBuilder* NewTableBuilder(
+    const ImmutableCFOptions& ioptions, const MutableCFOptions& moptions,
+    const InternalKeyComparator& internal_comparator,
+    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+        int_tbl_prop_collector_factories,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
+    uint64_t sample_for_compression, const CompressionOptions& compression_opts,
+    int level, const bool skip_filters, const uint64_t creation_time,
+    const uint64_t oldest_key_time, const uint64_t target_file_size,
+    const uint64_t file_creation_time) {
+  assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         tboptions.column_family_name.empty());
-  return tboptions.ioptions.table_factory->NewTableBuilder(tboptions, file);
+         column_family_name.empty());
+  return ioptions.table_factory->NewTableBuilder(
+      TableBuilderOptions(ioptions, moptions, internal_comparator,
+                          int_tbl_prop_collector_factories, compression_type,
+                          sample_for_compression, compression_opts,
+                          skip_filters, column_family_name, level,
+                          creation_time, oldest_key_time, target_file_size,
+                          file_creation_time),
+      column_family_id, file);
 }
 
 Status BuildTable(
-    const std::string& dbname, VersionSet* versions,
-    const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions,
-    const FileOptions& file_options, TableCache* table_cache,
-    InternalIterator* iter,
+    const std::string& dbname, Env* env, FileSystem* fs,
+    const ImmutableCFOptions& ioptions,
+    const MutableCFOptions& mutable_cf_options, const FileOptions& file_options,
+    TableCache* table_cache, InternalIterator* iter,
     std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
         range_del_iters,
-    FileMetaData* meta, std::vector<BlobFileAddition>* blob_file_additions,
+    FileMetaData* meta, const InternalKeyComparator& internal_comparator,
+    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+        int_tbl_prop_collector_factories,
+    uint32_t column_family_id, const std::string& column_family_name,
     std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
-    SnapshotChecker* snapshot_checker, bool paranoid_file_checks,
-    InternalStats* internal_stats, IOStatus* io_status,
-    const std::shared_ptr<IOTracer>& io_tracer, EventLogger* event_logger,
-    int job_id, const Env::IOPriority io_priority,
-    TableProperties* table_properties, Env::WriteLifeTimeHint write_hint,
-    const std::string* full_history_ts_low,
-    BlobFileCompletionCallback* blob_callback, uint64_t* num_input_entries,
-    uint64_t* memtable_payload_bytes, uint64_t* memtable_garbage_bytes) {
-  assert((tboptions.column_family_id ==
+    SnapshotChecker* snapshot_checker, const CompressionType compression,
+    uint64_t sample_for_compression, const CompressionOptions& compression_opts,
+    bool paranoid_file_checks, InternalStats* internal_stats,
+    TableFileCreationReason reason, EventLogger* event_logger, int job_id,
+    const Env::IOPriority io_priority, TableProperties* table_properties,
+    int level, const uint64_t creation_time, const uint64_t oldest_key_time,
+    Env::WriteLifeTimeHint write_hint, const uint64_t file_creation_time) {
+  assert((column_family_id ==
           TablePropertiesCollectorFactory::Context::kUnknownColumnFamily) ==
-         tboptions.column_family_name.empty());
-  auto& mutable_cf_options = tboptions.moptions;
-  auto& ioptions = tboptions.ioptions;
+         column_family_name.empty());
   // Reports the IOStats for flush for every following bytes.
   const size_t kReportFlushIOStatsEvery = 1048576;
-  OutputValidator output_validator(
-      tboptions.internal_comparator,
-      /*enable_order_check=*/
-      mutable_cf_options.check_flush_compaction_key_order,
-      /*enable_hash=*/paranoid_file_checks);
   Status s;
   meta->fd.file_size = 0;
   iter->SeekToFirst();
   std::unique_ptr<CompactionRangeDelAggregator> range_del_agg(
-      new CompactionRangeDelAggregator(&tboptions.internal_comparator,
-                                       snapshots));
-  uint64_t num_unfragmented_tombstones = 0;
-  uint64_t total_tombstone_payload_bytes = 0;
+      new CompactionRangeDelAggregator(&internal_comparator, snapshots));
   for (auto& range_del_iter : range_del_iters) {
-    num_unfragmented_tombstones +=
-        range_del_iter->num_unfragmented_tombstones();
-    total_tombstone_payload_bytes +=
-        range_del_iter->total_tombstone_payload_bytes();
     range_del_agg->AddTombstones(std::move(range_del_iter));
   }
 
   std::string fname = TableFileName(ioptions.cf_paths, meta->fd.GetNumber(),
                                     meta->fd.GetPathId());
-  std::vector<std::string> blob_file_paths;
-  std::string file_checksum = kUnknownFileChecksum;
-  std::string file_checksum_func_name = kUnknownFileChecksumFuncName;
 #ifndef ROCKSDB_LITE
-  EventHelpers::NotifyTableFileCreationStarted(ioptions.listeners, dbname,
-                                               tboptions.column_family_name,
-                                               fname, job_id, tboptions.reason);
+  EventHelpers::NotifyTableFileCreationStarted(
+      ioptions.listeners, dbname, column_family_name, fname, job_id, reason);
 #endif  // !ROCKSDB_LITE
-  Env* env = db_options.env;
-  assert(env);
-  FileSystem* fs = db_options.fs.get();
-  assert(fs);
-
   TableProperties tp;
-  if (iter->Valid() || !range_del_agg->IsEmpty()) {
-    std::unique_ptr<CompactionFilter> compaction_filter;
-    if (ioptions.compaction_filter_factory != nullptr &&
-        ioptions.compaction_filter_factory->ShouldFilterTableFileCreation(
-            tboptions.reason)) {
-      CompactionFilter::Context context;
-      context.is_full_compaction = false;
-      context.is_manual_compaction = false;
-      context.column_family_id = tboptions.column_family_id;
-      context.reason = tboptions.reason;
-      compaction_filter =
-          ioptions.compaction_filter_factory->CreateCompactionFilter(context);
-      if (compaction_filter != nullptr &&
-          !compaction_filter->IgnoreSnapshots()) {
-        s.PermitUncheckedError();
-        return Status::NotSupported(
-            "CompactionFilter::IgnoreSnapshots() = false is not supported "
-            "anymore.");
-      }
-    }
 
+  if (iter->Valid() || !range_del_agg->IsEmpty()) {
     TableBuilder* builder;
     std::unique_ptr<WritableFileWriter> file_writer;
+    // Currently we only enable dictionary compression during compaction to the
+    // bottommost level.
+    CompressionOptions compression_opts_for_flush(compression_opts);
+    compression_opts_for_flush.max_dict_bytes = 0;
+    compression_opts_for_flush.zstd_max_train_bytes = 0;
     {
       std::unique_ptr<FSWritableFile> file;
 #ifndef NDEBUG
       bool use_direct_writes = file_options.use_direct_writes;
       TEST_SYNC_POINT_CALLBACK("BuildTable:create_file", &use_direct_writes);
 #endif  // !NDEBUG
-      IOStatus io_s = NewWritableFile(fs, fname, &file, file_options);
-      assert(s.ok());
-      s = io_s;
-      if (io_status->ok()) {
-        *io_status = io_s;
-      }
+      s = NewWritableFile(fs, fname, &file, file_options);
       if (!s.ok()) {
         EventHelpers::LogAndNotifyTableFileCreationFinished(
-            event_logger, ioptions.listeners, dbname,
-            tboptions.column_family_name, fname, job_id, meta->fd,
-            kInvalidBlobFileNumber, tp, tboptions.reason, s, file_checksum,
-            file_checksum_func_name);
+            event_logger, ioptions.listeners, dbname, column_family_name, fname,
+            job_id, meta->fd, kInvalidBlobFileNumber, tp, reason, s);
         return s;
       }
-      FileTypeSet tmp_set = ioptions.checksum_handoff_file_types;
       file->SetIOPriority(io_priority);
       file->SetWriteLifeTimeHint(write_hint);
-      file_writer.reset(new WritableFileWriter(
-          std::move(file), fname, file_options, ioptions.clock, io_tracer,
-          ioptions.stats, ioptions.listeners,
-          ioptions.file_checksum_gen_factory.get(),
-          tmp_set.Contains(FileType::kTableFile), false));
 
-      builder = NewTableBuilder(tboptions, file_writer.get());
+      file_writer.reset(new WritableFileWriter(
+          std::move(file), fname, file_options, env, ioptions.statistics,
+          ioptions.listeners, ioptions.sst_file_checksum_func));
+
+      builder = NewTableBuilder(
+          ioptions, mutable_cf_options, internal_comparator,
+          int_tbl_prop_collector_factories, column_family_id,
+          column_family_name, file_writer.get(), compression,
+          sample_for_compression, compression_opts_for_flush, level,
+          false /* skip_filters */, creation_time, oldest_key_time,
+          0 /*target_file_size*/, file_creation_time);
     }
 
-    MergeHelper merge(
-        env, tboptions.internal_comparator.user_comparator(),
-        ioptions.merge_operator.get(), compaction_filter.get(), ioptions.logger,
-        true /* internal key corruption is not ok */,
-        snapshots.empty() ? 0 : snapshots.back(), snapshot_checker);
-
-    std::unique_ptr<BlobFileBuilder> blob_file_builder(
-        (mutable_cf_options.enable_blob_files && blob_file_additions)
-            ? new BlobFileBuilder(versions, fs, &ioptions, &mutable_cf_options,
-                                  &file_options, job_id,
-                                  tboptions.column_family_id,
-                                  tboptions.column_family_name, io_priority,
-                                  write_hint, io_tracer, blob_callback,
-                                  &blob_file_paths, blob_file_additions)
-            : nullptr);
+    MergeHelper merge(env, internal_comparator.user_comparator(),
+                      ioptions.merge_operator, nullptr, ioptions.info_log,
+                      true /* internal key corruption is not ok */,
+                      snapshots.empty() ? 0 : snapshots.back(),
+                      snapshot_checker);
 
     CompactionIterator c_iter(
-        iter, tboptions.internal_comparator.user_comparator(), &merge,
-        kMaxSequenceNumber, &snapshots, earliest_write_conflict_snapshot,
-        snapshot_checker, env, ShouldReportDetailedTime(env, ioptions.stats),
-        true /* internal key corruption is not ok */, range_del_agg.get(),
-        blob_file_builder.get(), ioptions.allow_data_in_errors,
-        /*compaction=*/nullptr, compaction_filter.get(),
-        /*shutting_down=*/nullptr,
-        /*preserve_deletes_seqnum=*/0, /*manual_compaction_paused=*/nullptr,
-        /*manual_compaction_canceled=*/nullptr, db_options.info_log,
-        full_history_ts_low);
-
+        iter, internal_comparator.user_comparator(), &merge, kMaxSequenceNumber,
+        &snapshots, earliest_write_conflict_snapshot, snapshot_checker, env,
+        ShouldReportDetailedTime(env, ioptions.statistics),
+        true /* internal key corruption is not ok */, range_del_agg.get());
     c_iter.SeekToFirst();
     for (; c_iter.Valid(); c_iter.Next()) {
       const Slice& key = c_iter.key();
       const Slice& value = c_iter.value();
       const ParsedInternalKey& ikey = c_iter.ikey();
-      // Generate a rolling 64-bit hash of the key and values
-      // Note :
-      // Here "key" integrates 'sequence_number'+'kType'+'user key'.
-      s = output_validator.Add(key, value);
-      if (!s.ok()) {
-        break;
-      }
       builder->Add(key, value);
       meta->UpdateBoundaries(key, value, ikey.sequence, ikey.type);
 
@@ -220,39 +170,26 @@ Status BuildTable(
             ThreadStatus::FLUSH_BYTES_WRITTEN, IOSTATS(bytes_written));
       }
     }
-    if (!s.ok()) {
-      c_iter.status().PermitUncheckedError();
-    } else if (!c_iter.status().ok()) {
-      s = c_iter.status();
+
+    auto range_del_it = range_del_agg->NewIterator();
+    for (range_del_it->SeekToFirst(); range_del_it->Valid();
+         range_del_it->Next()) {
+      auto tombstone = range_del_it->Tombstone();
+      auto kv = tombstone.Serialize();
+      builder->Add(kv.first.Encode(), kv.second);
+      meta->UpdateBoundariesForRange(kv.first, tombstone.SerializeEndKey(),
+                                     tombstone.seq_, internal_comparator);
     }
 
-    if (s.ok()) {
-      auto range_del_it = range_del_agg->NewIterator();
-      for (range_del_it->SeekToFirst(); range_del_it->Valid();
-           range_del_it->Next()) {
-        auto tombstone = range_del_it->Tombstone();
-        auto kv = tombstone.Serialize();
-        builder->Add(kv.first.Encode(), kv.second);
-        meta->UpdateBoundariesForRange(kv.first, tombstone.SerializeEndKey(),
-                                       tombstone.seq_,
-                                       tboptions.internal_comparator);
-      }
-    }
-
-    TEST_SYNC_POINT("BuildTable:BeforeFinishBuildTable");
-    const bool empty = builder->IsEmpty();
-    if (num_input_entries != nullptr) {
-      *num_input_entries =
-          c_iter.num_input_entry_scanned() + num_unfragmented_tombstones;
-    }
+    // Finish and check for builder errors
+    tp = builder->GetTableProperties();
+    bool empty = builder->NumEntries() == 0 && tp.num_range_deletions == 0;
+    s = c_iter.status();
     if (!s.ok() || empty) {
       builder->Abandon();
     } else {
       s = builder->Finish();
     }
-    if (io_status->ok()) {
-      *io_status = builder->io_status();
-    }
 
     if (s.ok() && !empty) {
       uint64_t file_size = builder->FileSize();
@@ -260,63 +197,23 @@ Status BuildTable(
       meta->marked_for_compaction = builder->NeedCompact();
       assert(meta->fd.GetFileSize() > 0);
       tp = builder->GetTableProperties(); // refresh now that builder is finished
-      if (memtable_payload_bytes != nullptr &&
-          memtable_garbage_bytes != nullptr) {
-        const CompactionIterationStats& ci_stats = c_iter.iter_stats();
-        uint64_t total_payload_bytes = ci_stats.total_input_raw_key_bytes +
-                                       ci_stats.total_input_raw_value_bytes +
-                                       total_tombstone_payload_bytes;
-        uint64_t total_payload_bytes_written =
-            (tp.raw_key_size + tp.raw_value_size);
-        // Prevent underflow, which may still happen at this point
-        // since we only support inserts, deletes, and deleteRanges.
-        if (total_payload_bytes_written <= total_payload_bytes) {
-          *memtable_payload_bytes = total_payload_bytes;
-          *memtable_garbage_bytes =
-              total_payload_bytes - total_payload_bytes_written;
-        } else {
-          *memtable_payload_bytes = 0;
-          *memtable_garbage_bytes = 0;
-        }
-      }
       if (table_properties) {
         *table_properties = tp;
       }
+      // Add the checksum information to file metadata.
+      meta->file_checksum = builder->GetFileChecksum();
+      meta->file_checksum_func_name = builder->GetFileChecksumFuncName();
     }
     delete builder;
 
     // Finish and check for file errors
-    TEST_SYNC_POINT("BuildTable:BeforeSyncTable");
     if (s.ok() && !empty) {
-      StopWatch sw(ioptions.clock, ioptions.stats, TABLE_SYNC_MICROS);
-      *io_status = file_writer->Sync(ioptions.use_fsync);
+      StopWatch sw(env, ioptions.statistics, TABLE_SYNC_MICROS);
+      s = file_writer->Sync(ioptions.use_fsync);
     }
-    TEST_SYNC_POINT("BuildTable:BeforeCloseTableFile");
-    if (s.ok() && io_status->ok() && !empty) {
-      *io_status = file_writer->Close();
+    if (s.ok() && !empty) {
+      s = file_writer->Close();
     }
-    if (s.ok() && io_status->ok() && !empty) {
-      // Add the checksum information to file metadata.
-      meta->file_checksum = file_writer->GetFileChecksum();
-      meta->file_checksum_func_name = file_writer->GetFileChecksumFuncName();
-      file_checksum = meta->file_checksum;
-      file_checksum_func_name = meta->file_checksum_func_name;
-    }
-
-    if (s.ok()) {
-      s = *io_status;
-    }
-
-    if (blob_file_builder) {
-      if (s.ok()) {
-        s = blob_file_builder->Finish();
-      } else {
-        blob_file_builder->Abandon();
-      }
-      blob_file_builder.reset();
-    }
-
-    // TODO Also check the IO status when create the Iterator.
 
     if (s.ok() && !empty) {
       // Verify that the table is usable
@@ -325,32 +222,20 @@ Status BuildTable(
       // No matter whether use_direct_io_for_flush_and_compaction is true,
       // we will regrad this verification as user reads since the goal is
       // to cache it here for further user reads
-      ReadOptions read_options;
       std::unique_ptr<InternalIterator> it(table_cache->NewIterator(
-          read_options, file_options, tboptions.internal_comparator, *meta,
+          ReadOptions(), file_options, internal_comparator, *meta,
           nullptr /* range_del_agg */,
           mutable_cf_options.prefix_extractor.get(), nullptr,
           (internal_stats == nullptr) ? nullptr
                                       : internal_stats->GetFileReadHist(0),
           TableReaderCaller::kFlush, /*arena=*/nullptr,
-          /*skip_filter=*/false, tboptions.level_at_creation,
-          MaxFileSizeForL0MetaPin(mutable_cf_options),
-          /*smallest_compaction_key=*/nullptr,
-          /*largest_compaction_key*/ nullptr,
-          /*allow_unprepared_value*/ false));
+          /*skip_filter=*/false, level, /*smallest_compaction_key=*/nullptr,
+          /*largest_compaction_key*/ nullptr));
       s = it->status();
       if (s.ok() && paranoid_file_checks) {
-        OutputValidator file_validator(tboptions.internal_comparator,
-                                       /*enable_order_check=*/true,
-                                       /*enable_hash=*/true);
         for (it->SeekToFirst(); it->Valid(); it->Next()) {
-          // Generate a rolling 64-bit hash of the key and values
-          file_validator.Add(it->key(), it->value()).PermitUncheckedError();
         }
         s = it->status();
-        if (s.ok() && !output_validator.CompareValidator(file_validator)) {
-          s = Status::Corruption("Paranoid checksums do not match");
-        }
       }
     }
   }
@@ -361,23 +246,7 @@ Status BuildTable(
   }
 
   if (!s.ok() || meta->fd.GetFileSize() == 0) {
-    TEST_SYNC_POINT("BuildTable:BeforeDeleteFile");
-
-    constexpr IODebugContext* dbg = nullptr;
-
-    Status ignored = fs->DeleteFile(fname, IOOptions(), dbg);
-    ignored.PermitUncheckedError();
-
-    assert(blob_file_additions || blob_file_paths.empty());
-
-    if (blob_file_additions) {
-      for (const std::string& blob_file_path : blob_file_paths) {
-        ignored = DeleteDBFile(&db_options, blob_file_path, dbname,
-                               /*force_bg=*/false, /*force_fg=*/false);
-        ignored.PermitUncheckedError();
-        TEST_SYNC_POINT("BuildTable::AfterDeleteFile");
-      }
-    }
+    fs->DeleteFile(fname, IOOptions(), nullptr);
   }
 
   if (meta->fd.GetFileSize() == 0) {
@@ -385,9 +254,8 @@ Status BuildTable(
   }
   // Output to event logger and fire events.
   EventHelpers::LogAndNotifyTableFileCreationFinished(
-      event_logger, ioptions.listeners, dbname, tboptions.column_family_name,
-      fname, job_id, meta->fd, meta->oldest_blob_file_number, tp,
-      tboptions.reason, s, file_checksum, file_checksum_func_name);
+      event_logger, ioptions.listeners, dbname, column_family_name, fname,
+      job_id, meta->fd, meta->oldest_blob_file_number, tp, reason, s);
 
   return s;
 }

db/builder.h

@@ -24,20 +24,34 @@
 
 namespace ROCKSDB_NAMESPACE {
 
+struct Options;
 struct FileMetaData;
 
-class VersionSet;
-class BlobFileAddition;
+class Env;
+struct EnvOptions;
+class Iterator;
 class SnapshotChecker;
 class TableCache;
+class VersionEdit;
 class TableBuilder;
 class WritableFileWriter;
 class InternalStats;
-class BlobFileCompletionCallback;
 
-// Convenience function for NewTableBuilder on the embedded table_factory.
-TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions,
-                              WritableFileWriter* file);
+// @param column_family_name Name of the column family that is also identified
+//    by column_family_id, or empty string if unknown. It must outlive the
+//    TableBuilder returned by this function.
+TableBuilder* NewTableBuilder(
+    const ImmutableCFOptions& options, const MutableCFOptions& moptions,
+    const InternalKeyComparator& internal_comparator,
+    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+        int_tbl_prop_collector_factories,
+    uint32_t column_family_id, const std::string& column_family_name,
+    WritableFileWriter* file, const CompressionType compression_type,
+    const uint64_t sample_for_compression,
+    const CompressionOptions& compression_opts, int level,
+    const bool skip_filters = false, const uint64_t creation_time = 0,
+    const uint64_t oldest_key_time = 0, const uint64_t target_file_size = 0,
+    const uint64_t file_creation_time = 0);
 
 // Build a Table file from the contents of *iter.  The generated file
 // will be named according to number specified in meta. On success, the rest of
@@ -48,26 +62,27 @@ TableBuilder* NewTableBuilder(const TableBuilderOptions& tboptions,
 // @param column_family_name Name of the column family that is also identified
 //    by column_family_id, or empty string if unknown.
 extern Status BuildTable(
-    const std::string& dbname, VersionSet* versions,
-    const ImmutableDBOptions& db_options, const TableBuilderOptions& tboptions,
-    const FileOptions& file_options, TableCache* table_cache,
-    InternalIterator* iter,
+    const std::string& dbname, Env* env, FileSystem* fs,
+    const ImmutableCFOptions& options,
+    const MutableCFOptions& mutable_cf_options, const FileOptions& file_options,
+    TableCache* table_cache, InternalIterator* iter,
     std::vector<std::unique_ptr<FragmentedRangeTombstoneIterator>>
         range_del_iters,
-    FileMetaData* meta, std::vector<BlobFileAddition>* blob_file_additions,
+    FileMetaData* meta, const InternalKeyComparator& internal_comparator,
+    const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+        int_tbl_prop_collector_factories,
+    uint32_t column_family_id, const std::string& column_family_name,
     std::vector<SequenceNumber> snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
-    SnapshotChecker* snapshot_checker, bool paranoid_file_checks,
-    InternalStats* internal_stats, IOStatus* io_status,
-    const std::shared_ptr<IOTracer>& io_tracer,
+    SnapshotChecker* snapshot_checker, const CompressionType compression,
+    const uint64_t sample_for_compression,
+    const CompressionOptions& compression_opts, bool paranoid_file_checks,
+    InternalStats* internal_stats, TableFileCreationReason reason,
     EventLogger* event_logger = nullptr, int job_id = 0,
     const Env::IOPriority io_priority = Env::IO_HIGH,
-    TableProperties* table_properties = nullptr,
+    TableProperties* table_properties = nullptr, int level = -1,
+    const uint64_t creation_time = 0, const uint64_t oldest_key_time = 0,
     Env::WriteLifeTimeHint write_hint = Env::WLTH_NOT_SET,
-    const std::string* full_history_ts_low = nullptr,
-    BlobFileCompletionCallback* blob_callback = nullptr,
-    uint64_t* num_input_entries = nullptr,
-    uint64_t* memtable_payload_bytes = nullptr,
-    uint64_t* memtable_garbage_bytes = nullptr);
+    const uint64_t file_creation_time = 0);
 
 }  // namespace ROCKSDB_NAMESPACE

db/column_family.cc

@@ -12,11 +12,9 @@
 #include <algorithm>
 #include <cinttypes>
 #include <limits>
-#include <sstream>
 #include <string>
 #include <vector>
 
-#include "db/blob/blob_file_cache.h"
 #include "db/compaction/compaction_picker.h"
 #include "db/compaction/compaction_picker_fifo.h"
 #include "db/compaction/compaction_picker_level.h"
@@ -33,11 +31,9 @@
 #include "monitoring/thread_status_util.h"
 #include "options/options_helper.h"
 #include "port/port.h"
-#include "rocksdb/convenience.h"
-#include "rocksdb/table.h"
+#include "table/block_based/block_based_table_factory.h"
 #include "table/merging_iterator.h"
 #include "util/autovector.h"
-#include "util/cast_util.h"
 #include "util/compression.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -109,9 +105,8 @@ const Comparator* ColumnFamilyHandleImpl::GetComparator() const {
 
 void GetIntTblPropCollectorFactory(
     const ImmutableCFOptions& ioptions,
-    IntTblPropCollectorFactories* int_tbl_prop_collector_factories) {
-  assert(int_tbl_prop_collector_factories);
-
+    std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+        int_tbl_prop_collector_factories) {
   auto& collector_factories = ioptions.table_properties_collector_factories;
   for (size_t i = 0; i < ioptions.table_properties_collector_factories.size();
        ++i) {
@@ -152,16 +147,6 @@ Status CheckCompressionSupported(const ColumnFamilyOptions& cf_options) {
           "should be nonzero if we're using zstd's dictionary generator.");
     }
   }
-
-  if (!CompressionTypeSupported(cf_options.blob_compression_type)) {
-    std::ostringstream oss;
-    oss << "The specified blob compression type "
-        << CompressionTypeToString(cf_options.blob_compression_type)
-        << " is not available.";
-
-    return Status::InvalidArgument(oss.str());
-  }
-
   return Status::OK();
 }
 
@@ -215,8 +200,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   // if user sets arena_block_size, we trust user to use this value. Otherwise,
   // calculate a proper value from writer_buffer_size;
   if (result.arena_block_size <= 0) {
-    result.arena_block_size =
-        std::min(size_t{1024 * 1024}, result.write_buffer_size / 8);
+    result.arena_block_size = result.write_buffer_size / 8;
 
     // Align up to 4k
     const size_t align = 4 * 1024;
@@ -285,7 +269,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   }
 
   if (result.level0_file_num_compaction_trigger == 0) {
-    ROCKS_LOG_WARN(db_options.logger,
+    ROCKS_LOG_WARN(db_options.info_log.get(),
                    "level0_file_num_compaction_trigger cannot be 0");
     result.level0_file_num_compaction_trigger = 1;
   }
@@ -294,7 +278,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
           result.level0_slowdown_writes_trigger ||
       result.level0_slowdown_writes_trigger <
           result.level0_file_num_compaction_trigger) {
-    ROCKS_LOG_WARN(db_options.logger,
+    ROCKS_LOG_WARN(db_options.info_log.get(),
                    "This condition must be satisfied: "
                    "level0_stop_writes_trigger(%d) >= "
                    "level0_slowdown_writes_trigger(%d) >= "
@@ -311,7 +295,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
         result.level0_slowdown_writes_trigger) {
       result.level0_stop_writes_trigger = result.level0_slowdown_writes_trigger;
     }
-    ROCKS_LOG_WARN(db_options.logger,
+    ROCKS_LOG_WARN(db_options.info_log.get(),
                    "Adjust the value to "
                    "level0_stop_writes_trigger(%d)"
                    "level0_slowdown_writes_trigger(%d)"
@@ -338,9 +322,7 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
   // was not used)
   auto sfm = static_cast<SstFileManagerImpl*>(db_options.sst_file_manager.get());
   for (size_t i = 0; i < result.cf_paths.size(); i++) {
-    DeleteScheduler::CleanupDirectory(db_options.env, sfm,
-                                      result.cf_paths[i].path)
-        .PermitUncheckedError();
+    DeleteScheduler::CleanupDirectory(db_options.env, sfm, result.cf_paths[i].path);
   }
 #endif
 
@@ -363,8 +345,8 @@ ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
     result.max_compaction_bytes = result.target_file_size_base * 25;
   }
 
-  bool is_block_based_table = (result.table_factory->IsInstanceOf(
-      TableFactory::kBlockBasedTableName()));
+  bool is_block_based_table =
+      (result.table_factory->Name() == BlockBasedTableFactory().Name());
 
   const uint64_t kAdjustedTtl = 30 * 24 * 60 * 60;
   if (result.ttl == kDefaultTtl) {
@@ -445,9 +427,6 @@ bool SuperVersion::Unref() {
 
 void SuperVersion::Cleanup() {
   assert(refs.load(std::memory_order_relaxed) == 0);
-  // Since this SuperVersion object is being deleted,
-  // decrement reference to the immutable MemtableList
-  // this SV object was pointing to.
   imm->Unref(&to_delete);
   MemTable* m = mem->Unref();
   if (m != nullptr) {
@@ -457,7 +436,9 @@ void SuperVersion::Cleanup() {
     to_delete.push_back(m);
   }
   current->Unref();
-  cfd->UnrefAndTryDelete();
+  if (cfd->Unref()) {
+    delete cfd;
+  }
 }
 
 void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
@@ -475,10 +456,10 @@ void SuperVersion::Init(ColumnFamilyData* new_cfd, MemTable* new_mem,
 
 namespace {
 void SuperVersionUnrefHandle(void* ptr) {
-  // UnrefHandle is called when a thread exits or a ThreadLocalPtr gets
-  // destroyed. When the former happens, the thread shouldn't see kSVInUse.
-  // When the latter happens, only super_version_ holds a reference
-  // to ColumnFamilyData, so no further queries are possible.
+  // UnrefHandle is called when a thread exists or a ThreadLocalPtr gets
+  // destroyed. When former happens, the thread shouldn't see kSVInUse.
+  // When latter happens, we are in ~ColumnFamilyData(), no get should happen as
+  // well.
   SuperVersion* sv = static_cast<SuperVersion*>(ptr);
   bool was_last_ref __attribute__((__unused__));
   was_last_ref = sv->Unref();
@@ -506,9 +487,7 @@ ColumnFamilyData::ColumnFamilyData(
     Cache* _table_cache, WriteBufferManager* write_buffer_manager,
     const ColumnFamilyOptions& cf_options, const ImmutableDBOptions& db_options,
     const FileOptions& file_options, ColumnFamilySet* column_family_set,
-    BlockCacheTracer* const block_cache_tracer,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    const std::string& db_session_id)
+    BlockCacheTracer* const block_cache_tracer)
     : id_(id),
       name_(name),
       dummy_versions_(_dummy_versions),
@@ -551,7 +530,7 @@ ColumnFamilyData::ColumnFamilyData(
       db_paths_registered_ = true;
     } else {
       ROCKS_LOG_ERROR(
-          ioptions_.logger,
+          ioptions_.info_log,
           "Failed to register data paths of column family (id: %d, name: %s)",
           id_, name_.c_str());
     }
@@ -564,14 +543,9 @@ ColumnFamilyData::ColumnFamilyData(
   // if _dummy_versions is nullptr, then this is a dummy column family.
   if (_dummy_versions != nullptr) {
     internal_stats_.reset(
-        new InternalStats(ioptions_.num_levels, ioptions_.clock, this));
+        new InternalStats(ioptions_.num_levels, db_options.env, this));
     table_cache_.reset(new TableCache(ioptions_, file_options, _table_cache,
-                                      block_cache_tracer, io_tracer,
-                                      db_session_id));
-    blob_file_cache_.reset(
-        new BlobFileCache(_table_cache, ioptions(), soptions(), id_,
-                          internal_stats_->GetBlobFileReadHist(), io_tracer));
-
+                                      block_cache_tracer));
     if (ioptions_.compaction_style == kCompactionStyleLevel) {
       compaction_picker_.reset(
           new LevelCompactionPicker(ioptions_, &internal_comparator_));
@@ -585,13 +559,13 @@ ColumnFamilyData::ColumnFamilyData(
     } else if (ioptions_.compaction_style == kCompactionStyleNone) {
       compaction_picker_.reset(new NullCompactionPicker(
           ioptions_, &internal_comparator_));
-      ROCKS_LOG_WARN(ioptions_.logger,
+      ROCKS_LOG_WARN(ioptions_.info_log,
                      "Column family %s does not use any background compaction. "
                      "Compactions can only be done via CompactFiles\n",
                      GetName().c_str());
 #endif  // !ROCKSDB_LITE
     } else {
-      ROCKS_LOG_ERROR(ioptions_.logger,
+      ROCKS_LOG_ERROR(ioptions_.info_log,
                       "Unable to recognize the specified compaction style %d. "
                       "Column family %s will use kCompactionStyleLevel.\n",
                       ioptions_.compaction_style, GetName().c_str());
@@ -600,12 +574,12 @@ ColumnFamilyData::ColumnFamilyData(
     }
 
     if (column_family_set_->NumberOfColumnFamilies() < 10) {
-      ROCKS_LOG_INFO(ioptions_.logger,
+      ROCKS_LOG_INFO(ioptions_.info_log,
                      "--------------- Options for column family [%s]:\n",
                      name.c_str());
-      initial_cf_options_.Dump(ioptions_.logger);
+      initial_cf_options_.Dump(ioptions_.info_log);
     } else {
-      ROCKS_LOG_INFO(ioptions_.logger, "\t(skipping printing options)\n");
+      ROCKS_LOG_INFO(ioptions_.info_log, "\t(skipping printing options)\n");
     }
   }
 
@@ -640,7 +614,7 @@ ColumnFamilyData::~ColumnFamilyData() {
 
   if (dummy_versions_ != nullptr) {
     // List must be empty
-    assert(dummy_versions_->Next() == dummy_versions_);
+    assert(dummy_versions_->TEST_Next() == dummy_versions_);
     bool deleted __attribute__((__unused__));
     deleted = dummy_versions_->Unref();
     assert(deleted);
@@ -661,7 +635,7 @@ ColumnFamilyData::~ColumnFamilyData() {
     Status s = ioptions_.env->UnregisterDbPaths(GetDbPaths());
     if (!s.ok()) {
       ROCKS_LOG_ERROR(
-          ioptions_.logger,
+          ioptions_.info_log,
           "Failed to unregister data paths of column family (id: %d, name: %s)",
           id_, name_.c_str());
     }
@@ -682,13 +656,14 @@ bool ColumnFamilyData::UnrefAndTryDelete() {
     // Only the super_version_ holds me
     SuperVersion* sv = super_version_;
     super_version_ = nullptr;
-
-    // Release SuperVersion references kept in ThreadLocalPtr.
+    // Release SuperVersion reference kept in ThreadLocalPtr.
+    // This must be done outside of mutex_ since unref handler can lock mutex.
+    sv->db_mutex->Unlock();
     local_sv_.reset();
+    sv->db_mutex->Lock();
 
     if (sv->Unref()) {
-      // Note: sv will delete this ColumnFamilyData during Cleanup()
-      assert(sv->cfd == this);
+      // May delete this ColumnFamilyData after calling Cleanup()
       sv->Cleanup();
       delete sv;
       return true;
@@ -715,7 +690,9 @@ uint64_t ColumnFamilyData::OldestLogToKeep() {
   auto current_log = GetLogNumber();
 
   if (allow_2pc_) {
-    auto imm_prep_log = imm()->PrecomputeMinLogContainingPrepSection();
+    autovector<MemTable*> empty_list;
+    auto imm_prep_log =
+        imm()->PrecomputeMinLogContainingPrepSection(empty_list);
     auto mem_prep_log = mem()->GetMinLogContainingPrepSection();
 
     if (imm_prep_log > 0 && imm_prep_log < current_log) {
@@ -837,8 +814,7 @@ std::pair<WriteStallCondition, ColumnFamilyData::WriteStallCause>
 ColumnFamilyData::GetWriteStallConditionAndCause(
     int num_unflushed_memtables, int num_l0_files,
     uint64_t num_compaction_needed_bytes,
-    const MutableCFOptions& mutable_cf_options,
-    const ImmutableCFOptions& immutable_cf_options) {
+    const MutableCFOptions& mutable_cf_options) {
   if (num_unflushed_memtables >= mutable_cf_options.max_write_buffer_number) {
     return {WriteStallCondition::kStopped, WriteStallCause::kMemtableLimit};
   } else if (!mutable_cf_options.disable_auto_compactions &&
@@ -852,9 +828,7 @@ ColumnFamilyData::GetWriteStallConditionAndCause(
             WriteStallCause::kPendingCompactionBytes};
   } else if (mutable_cf_options.max_write_buffer_number > 3 &&
              num_unflushed_memtables >=
-                 mutable_cf_options.max_write_buffer_number - 1 &&
-             num_unflushed_memtables - 1 >=
-                 immutable_cf_options.min_write_buffer_number_to_merge) {
+                 mutable_cf_options.max_write_buffer_number - 1) {
     return {WriteStallCondition::kDelayed, WriteStallCause::kMemtableLimit};
   } else if (!mutable_cf_options.disable_auto_compactions &&
              mutable_cf_options.level0_slowdown_writes_trigger >= 0 &&
@@ -882,8 +856,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
 
     auto write_stall_condition_and_cause = GetWriteStallConditionAndCause(
         imm()->NumNotFlushed(), vstorage->l0_delay_trigger_count(),
-        vstorage->estimated_compaction_needed_bytes(), mutable_cf_options,
-        *ioptions());
+        vstorage->estimated_compaction_needed_bytes(), mutable_cf_options);
     write_stall_condition = write_stall_condition_and_cause.first;
     auto write_stall_cause = write_stall_condition_and_cause.second;
 
@@ -895,7 +868,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
       write_controller_token_ = write_controller->GetStopToken();
       internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_STOPS, 1);
       ROCKS_LOG_WARN(
-          ioptions_.logger,
+          ioptions_.info_log,
           "[%s] Stopping writes because we have %d immutable memtables "
           "(waiting for flush), max_write_buffer_number is set to %d",
           name_.c_str(), imm()->NumNotFlushed(),
@@ -908,7 +881,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
         internal_stats_->AddCFStats(
             InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_STOPS, 1);
       }
-      ROCKS_LOG_WARN(ioptions_.logger,
+      ROCKS_LOG_WARN(ioptions_.info_log,
                      "[%s] Stopping writes because we have %d level-0 files",
                      name_.c_str(), vstorage->l0_delay_trigger_count());
     } else if (write_stall_condition == WriteStallCondition::kStopped &&
@@ -917,7 +890,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
       internal_stats_->AddCFStats(
           InternalStats::PENDING_COMPACTION_BYTES_LIMIT_STOPS, 1);
       ROCKS_LOG_WARN(
-          ioptions_.logger,
+          ioptions_.info_log,
           "[%s] Stopping writes because of estimated pending compaction "
           "bytes %" PRIu64,
           name_.c_str(), compaction_needed_bytes);
@@ -929,7 +902,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
                      mutable_cf_options.disable_auto_compactions);
       internal_stats_->AddCFStats(InternalStats::MEMTABLE_LIMIT_SLOWDOWNS, 1);
       ROCKS_LOG_WARN(
-          ioptions_.logger,
+          ioptions_.info_log,
           "[%s] Stalling writes because we have %d immutable memtables "
           "(waiting for flush), max_write_buffer_number is set to %d "
           "rate %" PRIu64,
@@ -951,7 +924,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
         internal_stats_->AddCFStats(
             InternalStats::LOCKED_L0_FILE_COUNT_LIMIT_SLOWDOWNS, 1);
       }
-      ROCKS_LOG_WARN(ioptions_.logger,
+      ROCKS_LOG_WARN(ioptions_.info_log,
                      "[%s] Stalling writes because we have %d level-0 files "
                      "rate %" PRIu64,
                      name_.c_str(), vstorage->l0_delay_trigger_count(),
@@ -976,7 +949,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
       internal_stats_->AddCFStats(
           InternalStats::PENDING_COMPACTION_BYTES_LIMIT_SLOWDOWNS, 1);
       ROCKS_LOG_WARN(
-          ioptions_.logger,
+          ioptions_.info_log,
           "[%s] Stalling writes because of estimated pending compaction "
           "bytes %" PRIu64 " rate %" PRIu64,
           name_.c_str(), vstorage->estimated_compaction_needed_bytes(),
@@ -990,7 +963,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
         write_controller_token_ =
             write_controller->GetCompactionPressureToken();
         ROCKS_LOG_INFO(
-            ioptions_.logger,
+            ioptions_.info_log,
             "[%s] Increasing compaction threads because we have %d level-0 "
             "files ",
             name_.c_str(), vstorage->l0_delay_trigger_count());
@@ -1004,7 +977,7 @@ WriteStallCondition ColumnFamilyData::RecalculateWriteStallConditions(
             write_controller->GetCompactionPressureToken();
         if (mutable_cf_options.soft_pending_compaction_bytes_limit > 0) {
           ROCKS_LOG_INFO(
-              ioptions_.logger,
+              ioptions_.info_log,
               "[%s] Increasing compaction threads because of estimated pending "
               "compaction "
               "bytes %" PRIu64,
@@ -1054,37 +1027,32 @@ uint64_t ColumnFamilyData::GetLiveSstFilesSize() const {
 }
 
 MemTable* ColumnFamilyData::ConstructNewMemtable(
-    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq,
-    uint64_t log_number) {
+    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
   return new MemTable(internal_comparator_, ioptions_, mutable_cf_options,
-                      write_buffer_manager_, earliest_seq, id_, log_number);
+                      write_buffer_manager_, earliest_seq, id_);
 }
 
 void ColumnFamilyData::CreateNewMemtable(
-    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq,
-    uint64_t log_number) {
+    const MutableCFOptions& mutable_cf_options, SequenceNumber earliest_seq) {
   if (mem_ != nullptr) {
     delete mem_->Unref();
   }
-  SetMemtable(
-      ConstructNewMemtable(mutable_cf_options, earliest_seq, log_number));
+  SetMemtable(ConstructNewMemtable(mutable_cf_options, earliest_seq));
   mem_->Ref();
 }
 
 bool ColumnFamilyData::NeedsCompaction() const {
-  return !mutable_cf_options_.disable_auto_compactions &&
-         compaction_picker_->NeedsCompaction(current_->storage_info());
+  return compaction_picker_->NeedsCompaction(current_->storage_info());
 }
 
 Compaction* ColumnFamilyData::PickCompaction(
-    const MutableCFOptions& mutable_options,
-    const MutableDBOptions& mutable_db_options, LogBuffer* log_buffer) {
+    const MutableCFOptions& mutable_options, LogBuffer* log_buffer) {
   SequenceNumber earliest_mem_seqno =
       std::min(mem_->GetEarliestSequenceNumber(),
                imm_.current()->GetEarliestSequenceNumber(false));
   auto* result = compaction_picker_->PickCompaction(
-      GetName(), mutable_options, mutable_db_options, current_->storage_info(),
-      log_buffer, earliest_mem_seqno);
+      GetName(), mutable_options, current_->storage_info(), log_buffer,
+      earliest_mem_seqno);
   if (result != nullptr) {
     result->SetInputVersion(current_);
   }
@@ -1100,7 +1068,7 @@ bool ColumnFamilyData::RangeOverlapWithCompaction(
 
 Status ColumnFamilyData::RangesOverlapWithMemtables(
     const autovector<Range>& ranges, SuperVersion* super_version,
-    bool allow_data_in_errors, bool* overlap) {
+    bool* overlap) {
   assert(overlap != nullptr);
   *overlap = false;
   // Create an InternalIterator over all unflushed memtables
@@ -1119,12 +1087,10 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(
       super_version->mem->NewRangeTombstoneIterator(read_opts, read_seq);
   range_del_agg.AddTombstones(
       std::unique_ptr<FragmentedRangeTombstoneIterator>(active_range_del_iter));
-  Status status;
-  status = super_version->imm->AddRangeTombstoneIterators(
-      read_opts, nullptr /* arena */, &range_del_agg);
-  // AddRangeTombstoneIterators always return Status::OK.
-  assert(status.ok());
+  super_version->imm->AddRangeTombstoneIterators(read_opts, nullptr /* arena */,
+                                                 &range_del_agg);
 
+  Status status;
   for (size_t i = 0; i < ranges.size() && status.ok() && !*overlap; ++i) {
     auto* vstorage = super_version->current->storage_info();
     auto* ucmp = vstorage->InternalComparator()->user_comparator();
@@ -1133,12 +1099,12 @@ Status ColumnFamilyData::RangesOverlapWithMemtables(
     memtable_iter->Seek(range_start.Encode());
     status = memtable_iter->status();
     ParsedInternalKey seek_result;
-
-    if (status.ok() && memtable_iter->Valid()) {
-      status = ParseInternalKey(memtable_iter->key(), &seek_result,
-                                allow_data_in_errors);
+    if (status.ok()) {
+      if (memtable_iter->Valid() &&
+          !ParseInternalKey(memtable_iter->key(), &seek_result)) {
+        status = Status::Corruption("DB have corrupted keys");
+      }
     }
-
     if (status.ok()) {
       if (memtable_iter->Valid() &&
           ucmp->Compare(seek_result.user_key, ranges[i].limit) <= 0) {
@@ -1156,16 +1122,14 @@ const int ColumnFamilyData::kCompactAllLevels = -1;
 const int ColumnFamilyData::kCompactToBaseLevel = -2;
 
 Compaction* ColumnFamilyData::CompactRange(
-    const MutableCFOptions& mutable_cf_options,
-    const MutableDBOptions& mutable_db_options, int input_level,
+    const MutableCFOptions& mutable_cf_options, int input_level,
     int output_level, const CompactRangeOptions& compact_range_options,
     const InternalKey* begin, const InternalKey* end,
     InternalKey** compaction_end, bool* conflict,
     uint64_t max_file_num_to_ignore) {
   auto* result = compaction_picker_->CompactRange(
-      GetName(), mutable_cf_options, mutable_db_options,
-      current_->storage_info(), input_level, output_level,
-      compact_range_options, begin, end, compaction_end, conflict,
+      GetName(), mutable_cf_options, current_->storage_info(), input_level,
+      output_level, compact_range_options, begin, end, compaction_end, conflict,
       max_file_num_to_ignore);
   if (result != nullptr) {
     result->SetInputVersion(current_);
@@ -1208,11 +1172,11 @@ SuperVersion* ColumnFamilyData::GetThreadLocalSuperVersion(DBImpl* db) {
   SuperVersion* sv = static_cast<SuperVersion*>(ptr);
   if (sv == SuperVersion::kSVObsolete ||
       sv->version_number != super_version_number_.load()) {
-    RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_ACQUIRES);
+    RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_ACQUIRES);
     SuperVersion* sv_to_delete = nullptr;
 
     if (sv && sv->Unref()) {
-      RecordTick(ioptions_.stats, NUMBER_SUPERVERSION_CLEANUPS);
+      RecordTick(ioptions_.statistics, NUMBER_SUPERVERSION_CLEANUPS);
       db->mutex()->Lock();
       // NOTE: underlying resources held by superversion (sst files) might
       // not be released until the next background job.
@@ -1256,13 +1220,14 @@ bool ColumnFamilyData::ReturnThreadLocalSuperVersion(SuperVersion* sv) {
 void ColumnFamilyData::InstallSuperVersion(
     SuperVersionContext* sv_context, InstrumentedMutex* db_mutex) {
   db_mutex->AssertHeld();
-  return InstallSuperVersion(sv_context, mutable_cf_options_);
+  return InstallSuperVersion(sv_context, db_mutex, mutable_cf_options_);
 }
 
 void ColumnFamilyData::InstallSuperVersion(
-    SuperVersionContext* sv_context,
+    SuperVersionContext* sv_context, InstrumentedMutex* db_mutex,
     const MutableCFOptions& mutable_cf_options) {
   SuperVersion* new_superversion = sv_context->new_superversion.release();
+  new_superversion->db_mutex = db_mutex;
   new_superversion->mutable_cf_options = mutable_cf_options;
   new_superversion->Init(this, mem_, imm_.current(), current_);
   SuperVersion* old_superversion = super_version_;
@@ -1334,8 +1299,7 @@ Status ColumnFamilyData::ValidateOptions(
   }
 
   if (cf_options.ttl > 0 && cf_options.ttl != kDefaultTtl) {
-    if (!cf_options.table_factory->IsInstanceOf(
-            TableFactory::kBlockBasedTableName())) {
+    if (cf_options.table_factory->Name() != BlockBasedTableFactory().Name()) {
       return Status::NotSupported(
           "TTL is only supported in Block-Based Table format. ");
     }
@@ -1343,46 +1307,30 @@ Status ColumnFamilyData::ValidateOptions(
 
   if (cf_options.periodic_compaction_seconds > 0 &&
       cf_options.periodic_compaction_seconds != kDefaultPeriodicCompSecs) {
-    if (!cf_options.table_factory->IsInstanceOf(
-            TableFactory::kBlockBasedTableName())) {
+    if (cf_options.table_factory->Name() != BlockBasedTableFactory().Name()) {
       return Status::NotSupported(
           "Periodic Compaction is only supported in "
           "Block-Based Table format. ");
     }
   }
-
-  if (cf_options.enable_blob_garbage_collection &&
-      (cf_options.blob_garbage_collection_age_cutoff < 0.0 ||
-       cf_options.blob_garbage_collection_age_cutoff > 1.0)) {
-    return Status::InvalidArgument(
-        "The age cutoff for blob garbage collection should be in the range "
-        "[0.0, 1.0].");
-  }
-
-  if (cf_options.compaction_style == kCompactionStyleFIFO &&
-      db_options.max_open_files != -1 && cf_options.ttl > 0) {
-    return Status::NotSupported(
-        "FIFO compaction only supported with max_open_files = -1.");
-  }
-
   return s;
 }
 
 #ifndef ROCKSDB_LITE
 Status ColumnFamilyData::SetOptions(
-    const DBOptions& db_opts,
+    const DBOptions& db_options,
     const std::unordered_map<std::string, std::string>& options_map) {
-  ColumnFamilyOptions cf_opts =
-      BuildColumnFamilyOptions(initial_cf_options_, mutable_cf_options_);
-  ConfigOptions config_opts;
-  config_opts.mutable_options_only = true;
-  Status s = GetColumnFamilyOptionsFromMap(config_opts, cf_opts, options_map,
-                                           &cf_opts);
+  MutableCFOptions new_mutable_cf_options;
+  Status s =
+      GetMutableOptionsFromStrings(mutable_cf_options_, options_map,
+                                   ioptions_.info_log, &new_mutable_cf_options);
   if (s.ok()) {
-    s = ValidateOptions(db_opts, cf_opts);
+    ColumnFamilyOptions cf_options =
+        BuildColumnFamilyOptions(initial_cf_options_, new_mutable_cf_options);
+    s = ValidateOptions(db_options, cf_options);
   }
   if (s.ok()) {
-    mutable_cf_options_ = MutableCFOptions(cf_opts);
+    mutable_cf_options_ = new_mutable_cf_options;
     mutable_cf_options_.RefreshDerivedOptions(ioptions_);
   }
   return s;
@@ -1421,8 +1369,7 @@ Status ColumnFamilyData::AddDirectories(
 
     if (existing_dir == created_dirs->end()) {
       std::unique_ptr<FSDirectory> path_directory;
-      s = DBImpl::CreateAndNewDirectory(ioptions_.fs.get(), p.path,
-                                        &path_directory);
+      s = DBImpl::CreateAndNewDirectory(ioptions_.fs, p.path, &path_directory);
       if (!s.ok()) {
         return s;
       }
@@ -1450,26 +1397,22 @@ ColumnFamilySet::ColumnFamilySet(const std::string& dbname,
                                  const ImmutableDBOptions* db_options,
                                  const FileOptions& file_options,
                                  Cache* table_cache,
-                                 WriteBufferManager* _write_buffer_manager,
-                                 WriteController* _write_controller,
-                                 BlockCacheTracer* const block_cache_tracer,
-                                 const std::shared_ptr<IOTracer>& io_tracer,
-                                 const std::string& db_session_id)
+                                 WriteBufferManager* write_buffer_manager,
+                                 WriteController* write_controller,
+                                 BlockCacheTracer* const block_cache_tracer)
     : max_column_family_(0),
       dummy_cfd_(new ColumnFamilyData(
           ColumnFamilyData::kDummyColumnFamilyDataId, "", nullptr, nullptr,
           nullptr, ColumnFamilyOptions(), *db_options, file_options, nullptr,
-          block_cache_tracer, io_tracer, db_session_id)),
+          block_cache_tracer)),
       default_cfd_cache_(nullptr),
       db_name_(dbname),
       db_options_(db_options),
       file_options_(file_options),
       table_cache_(table_cache),
-      write_buffer_manager_(_write_buffer_manager),
-      write_controller_(_write_controller),
-      block_cache_tracer_(block_cache_tracer),
-      io_tracer_(io_tracer),
-      db_session_id_(db_session_id) {
+      write_buffer_manager_(write_buffer_manager),
+      write_controller_(write_controller),
+      block_cache_tracer_(block_cache_tracer) {
   // initialize linked list
   dummy_cfd_->prev_ = dummy_cfd_;
   dummy_cfd_->next_ = dummy_cfd_;
@@ -1535,8 +1478,7 @@ ColumnFamilyData* ColumnFamilySet::CreateColumnFamily(
   assert(column_families_.find(name) == column_families_.end());
   ColumnFamilyData* new_cfd = new ColumnFamilyData(
       id, name, dummy_versions, table_cache_, write_buffer_manager_, options,
-      *db_options_, file_options_, this, block_cache_tracer_, io_tracer_,
-      db_session_id_);
+      *db_options_, file_options_, this, block_cache_tracer_);
   column_families_.insert({name, id});
   column_family_data_.insert({id, new_cfd});
   max_column_family_ = std::max(max_column_family_, id);
@@ -1604,7 +1546,7 @@ ColumnFamilyHandle* ColumnFamilyMemTablesImpl::GetColumnFamilyHandle() {
 uint32_t GetColumnFamilyID(ColumnFamilyHandle* column_family) {
   uint32_t column_family_id = 0;
   if (column_family != nullptr) {
-    auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(column_family);
+    auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(column_family);
     column_family_id = cfh->GetID();
   }
   return column_family_id;

db/column_family.h

@@ -44,7 +44,6 @@ class LogBuffer;
 class InstrumentedMutex;
 class InstrumentedMutexLock;
 struct SuperVersionContext;
-class BlobFileCache;
 
 extern const double kIncSlowdownRatio;
 // This file contains a list of data structures for managing column family
@@ -208,6 +207,8 @@ struct SuperVersion {
   uint64_t version_number;
   WriteStallCondition write_stall_condition;
 
+  InstrumentedMutex* db_mutex;
+
   // should be called outside the mutex
   SuperVersion() = default;
   ~SuperVersion();
@@ -251,12 +252,13 @@ extern Status CheckCFPathsSupported(const DBOptions& db_options,
 
 extern ColumnFamilyOptions SanitizeOptions(const ImmutableDBOptions& db_options,
                                            const ColumnFamilyOptions& src);
-// Wrap user defined table properties collector factories `from cf_options`
+// Wrap user defined table proproties collector factories `from cf_options`
 // into internal ones in int_tbl_prop_collector_factories. Add a system internal
 // one too.
 extern void GetIntTblPropCollectorFactory(
     const ImmutableCFOptions& ioptions,
-    IntTblPropCollectorFactories* int_tbl_prop_collector_factories);
+    std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+        int_tbl_prop_collector_factories);
 
 class ColumnFamilySet;
 
@@ -276,6 +278,17 @@ class ColumnFamilyData {
   // holding a DB mutex, or as the leader in a write batch group).
   void Ref() { refs_.fetch_add(1); }
 
+  // Unref decreases the reference count, but does not handle deletion
+  // when the count goes to 0.  If this method returns true then the
+  // caller should delete the instance immediately, or later, by calling
+  // FreeDeadColumnFamilies().  Unref() can only be called while holding
+  // a DB mutex, or during single-threaded recovery.
+  bool Unref() {
+    int old_refs = refs_.fetch_sub(1);
+    assert(old_refs > 0);
+    return old_refs == 1;
+  }
+
   // UnrefAndTryDelete() decreases the reference count and do free if needed,
   // return true if this is freed else false, UnrefAndTryDelete() can only
   // be called while holding a DB mutex, or during single-threaded recovery.
@@ -312,7 +325,7 @@ class ColumnFamilyData {
   FlushReason GetFlushReason() const { return flush_reason_; }
   // thread-safe
   const FileOptions* soptions() const;
-  const ImmutableOptions* ioptions() const { return &ioptions_; }
+  const ImmutableCFOptions* ioptions() const { return &ioptions_; }
   // REQUIRES: DB mutex held
   // This returns the MutableCFOptions used by current SuperVersion
   // You should use this API to reference MutableCFOptions most of the time.
@@ -346,11 +359,6 @@ class ColumnFamilyData {
 
   MemTableList* imm() { return &imm_; }
   MemTable* mem() { return mem_; }
-
-  bool IsEmpty() {
-    return mem()->GetFirstSequenceNumber() == 0 && imm()->NumNotFlushed() == 0;
-  }
-
   Version* current() { return current_; }
   Version* dummy_versions() { return dummy_versions_; }
   void SetCurrent(Version* _current);
@@ -368,20 +376,17 @@ class ColumnFamilyData {
 
   // See Memtable constructor for explanation of earliest_seq param.
   MemTable* ConstructNewMemtable(const MutableCFOptions& mutable_cf_options,
-                                 SequenceNumber earliest_seq,
-                                 uint64_t log_number = 0);
+                                 SequenceNumber earliest_seq);
   void CreateNewMemtable(const MutableCFOptions& mutable_cf_options,
-                         SequenceNumber earliest_seq, uint64_t log_number = 0);
+                         SequenceNumber earliest_seq);
 
   TableCache* table_cache() const { return table_cache_.get(); }
-  BlobFileCache* blob_file_cache() const { return blob_file_cache_.get(); }
 
   // See documentation in compaction_picker.h
   // REQUIRES: DB mutex held
   bool NeedsCompaction() const;
   // REQUIRES: DB mutex held
   Compaction* PickCompaction(const MutableCFOptions& mutable_options,
-                             const MutableDBOptions& mutable_db_options,
                              LogBuffer* log_buffer);
 
   // Check if the passed range overlap with any running compactions.
@@ -398,8 +403,7 @@ class ColumnFamilyData {
   //
   // Thread-safe
   Status RangesOverlapWithMemtables(const autovector<Range>& ranges,
-                                    SuperVersion* super_version,
-                                    bool allow_data_in_errors, bool* overlap);
+                                    SuperVersion* super_version, bool* overlap);
 
   // A flag to tell a manual compaction is to compact all levels together
   // instead of a specific level.
@@ -408,7 +412,6 @@ class ColumnFamilyData {
   static const int kCompactToBaseLevel;
   // REQUIRES: DB mutex held
   Compaction* CompactRange(const MutableCFOptions& mutable_cf_options,
-                           const MutableDBOptions& mutable_db_options,
                            int input_level, int output_level,
                            const CompactRangeOptions& compact_range_options,
                            const InternalKey* begin, const InternalKey* end,
@@ -425,7 +428,8 @@ class ColumnFamilyData {
     return internal_comparator_;
   }
 
-  const IntTblPropCollectorFactories* int_tbl_prop_collector_factories() const {
+  const std::vector<std::unique_ptr<IntTblPropCollectorFactory>>*
+  int_tbl_prop_collector_factories() const {
     return &int_tbl_prop_collector_factories_;
   }
 
@@ -437,7 +441,7 @@ class ColumnFamilyData {
   // Get SuperVersion stored in thread local storage. If it does not exist,
   // get a reference from a current SuperVersion.
   SuperVersion* GetThreadLocalSuperVersion(DBImpl* db);
-  // Try to return SuperVersion back to thread local storage. Return true on
+  // Try to return SuperVersion back to thread local storage. Retrun true on
   // success and false on failure. It fails when the thread local storage
   // contains anything other than SuperVersion::kSVInUse flag.
   bool ReturnThreadLocalSuperVersion(SuperVersion* sv);
@@ -451,6 +455,7 @@ class ColumnFamilyData {
   // the clients to allocate SuperVersion outside of mutex.
   // IMPORTANT: Only call this from DBImpl::InstallSuperVersion()
   void InstallSuperVersion(SuperVersionContext* sv_context,
+                           InstrumentedMutex* db_mutex,
                            const MutableCFOptions& mutable_cf_options);
   void InstallSuperVersion(SuperVersionContext* sv_context,
                            InstrumentedMutex* db_mutex);
@@ -470,11 +475,9 @@ class ColumnFamilyData {
     kPendingCompactionBytes,
   };
   static std::pair<WriteStallCondition, WriteStallCause>
-  GetWriteStallConditionAndCause(
-      int num_unflushed_memtables, int num_l0_files,
-      uint64_t num_compaction_needed_bytes,
-      const MutableCFOptions& mutable_cf_options,
-      const ImmutableCFOptions& immutable_cf_options);
+  GetWriteStallConditionAndCause(int num_unflushed_memtables, int num_l0_files,
+                                 uint64_t num_compaction_needed_bytes,
+                                 const MutableCFOptions& mutable_cf_options);
 
   // Recalculate some small conditions, which are changed only during
   // compaction, adding new memtable and/or
@@ -501,23 +504,7 @@ class ColumnFamilyData {
 
   FSDirectory* GetDataDir(size_t path_id) const;
 
-  // full_history_ts_low_ can only increase.
-  void SetFullHistoryTsLow(std::string ts_low) {
-    assert(!ts_low.empty());
-    const Comparator* ucmp = user_comparator();
-    assert(ucmp);
-    if (full_history_ts_low_.empty() ||
-        ucmp->CompareTimestamp(ts_low, full_history_ts_low_) > 0) {
-      full_history_ts_low_ = std::move(ts_low);
-    }
-  }
-
-  const std::string& GetFullHistoryTsLow() const {
-    return full_history_ts_low_;
-  }
-
   ThreadLocalPtr* TEST_GetLocalSV() { return local_sv_.get(); }
-  WriteBufferManager* write_buffer_mgr() { return write_buffer_manager_; }
 
  private:
   friend class ColumnFamilySet;
@@ -529,9 +516,7 @@ class ColumnFamilyData {
                    const ImmutableDBOptions& db_options,
                    const FileOptions& file_options,
                    ColumnFamilySet* column_family_set,
-                   BlockCacheTracer* const block_cache_tracer,
-                   const std::shared_ptr<IOTracer>& io_tracer,
-                   const std::string& db_session_id);
+                   BlockCacheTracer* const block_cache_tracer);
 
   std::vector<std::string> GetDbPaths() const;
 
@@ -545,16 +530,16 @@ class ColumnFamilyData {
   std::atomic<bool> dropped_;  // true if client dropped it
 
   const InternalKeyComparator internal_comparator_;
-  IntTblPropCollectorFactories int_tbl_prop_collector_factories_;
+  std::vector<std::unique_ptr<IntTblPropCollectorFactory>>
+      int_tbl_prop_collector_factories_;
 
   const ColumnFamilyOptions initial_cf_options_;
-  const ImmutableOptions ioptions_;
+  const ImmutableCFOptions ioptions_;
   MutableCFOptions mutable_cf_options_;
 
   const bool is_delete_range_supported_;
 
   std::unique_ptr<TableCache> table_cache_;
-  std::unique_ptr<BlobFileCache> blob_file_cache_;
 
   std::unique_ptr<InternalStats> internal_stats_;
 
@@ -613,8 +598,6 @@ class ColumnFamilyData {
   std::vector<std::shared_ptr<FSDirectory>> data_dirs_;
 
   bool db_paths_registered_;
-
-  std::string full_history_ts_low_;
 };
 
 // ColumnFamilySet has interesting thread-safety requirements
@@ -664,11 +647,9 @@ class ColumnFamilySet {
   ColumnFamilySet(const std::string& dbname,
                   const ImmutableDBOptions* db_options,
                   const FileOptions& file_options, Cache* table_cache,
-                  WriteBufferManager* _write_buffer_manager,
-                  WriteController* _write_controller,
-                  BlockCacheTracer* const block_cache_tracer,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  const std::string& db_session_id);
+                  WriteBufferManager* write_buffer_manager,
+                  WriteController* write_controller,
+                  BlockCacheTracer* const block_cache_tracer);
   ~ColumnFamilySet();
 
   ColumnFamilyData* GetDefault() const;
@@ -697,10 +678,6 @@ class ColumnFamilySet {
 
   Cache* get_table_cache() { return table_cache_; }
 
-  WriteBufferManager* write_buffer_manager() { return write_buffer_manager_; }
-
-  WriteController* write_controller() { return write_controller_; }
-
  private:
   friend class ColumnFamilyData;
   // helper function that gets called from cfd destructor
@@ -732,8 +709,6 @@ class ColumnFamilySet {
   WriteBufferManager* write_buffer_manager_;
   WriteController* write_controller_;
   BlockCacheTracer* const block_cache_tracer_;
-  std::shared_ptr<IOTracer> io_tracer_;
-  std::string db_session_id_;
 };
 
 // We use ColumnFamilyMemTablesImpl to provide WriteBatch a way to access

db/column_family_test.cc

@@ -8,37 +8,45 @@
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
 #include <algorithm>
+#include <vector>
 #include <string>
 #include <thread>
-#include <vector>
 
 #include "db/db_impl/db_impl.h"
 #include "db/db_test_util.h"
+#include "memtable/hash_skiplist_rep.h"
 #include "options/options_parser.h"
 #include "port/port.h"
 #include "port/stack_trace.h"
-#include "rocksdb/convenience.h"
 #include "rocksdb/db.h"
 #include "rocksdb/env.h"
 #include "rocksdb/iterator.h"
 #include "rocksdb/utilities/object_registry.h"
+#include "test_util/fault_injection_test_env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
 #include "test_util/testutil.h"
 #include "util/coding.h"
 #include "util/string_util.h"
-#include "utilities/fault_injection_env.h"
 #include "utilities/merge_operators.h"
 
 namespace ROCKSDB_NAMESPACE {
 
 static const int kValueSize = 1000;
 
+namespace {
+std::string RandomString(Random* rnd, int len) {
+  std::string r;
+  test::RandomString(rnd, len, &r);
+  return r;
+}
+}  // anonymous namespace
+
 // counts how many operations were performed
-class EnvCounter : public SpecialEnv {
+class EnvCounter : public EnvWrapper {
  public:
   explicit EnvCounter(Env* base)
-      : SpecialEnv(base), num_new_writable_file_(0) {}
+      : EnvWrapper(base), num_new_writable_file_(0) {}
   int GetNumberOfNewWritableFileCalls() {
     return num_new_writable_file_;
   }
@@ -56,30 +64,33 @@ class ColumnFamilyTestBase : public testing::Test {
  public:
   explicit ColumnFamilyTestBase(uint32_t format) : rnd_(139), format_(format) {
     Env* base_env = Env::Default();
-    EXPECT_OK(
-        test::CreateEnvFromSystem(ConfigOptions(), &base_env, &env_guard_));
+#ifndef ROCKSDB_LITE
+    const char* test_env_uri = getenv("TEST_ENV_URI");
+    if (test_env_uri) {
+      Env* test_env = nullptr;
+      Status s = Env::LoadEnv(test_env_uri, &test_env, &env_guard_);
+      base_env = test_env;
+      EXPECT_OK(s);
+      EXPECT_NE(Env::Default(), base_env);
+    }
+#endif  // !ROCKSDB_LITE
     EXPECT_NE(nullptr, base_env);
     env_ = new EnvCounter(base_env);
-    env_->skip_fsync_ = true;
     dbname_ = test::PerThreadDBPath("column_family_test");
     db_options_.create_if_missing = true;
     db_options_.fail_if_options_file_error = true;
     db_options_.env = env_;
-    EXPECT_OK(DestroyDB(dbname_, Options(db_options_, column_family_options_)));
+    DestroyDB(dbname_, Options(db_options_, column_family_options_));
   }
 
   ~ColumnFamilyTestBase() override {
     std::vector<ColumnFamilyDescriptor> column_families;
     for (auto h : handles_) {
       ColumnFamilyDescriptor cfdescriptor;
-      Status s = h->GetDescriptor(&cfdescriptor);
-#ifdef ROCKSDB_LITE
-      EXPECT_TRUE(s.IsNotSupported());
-#else
-      EXPECT_OK(s);
-#endif  // ROCKSDB_LITE
+      h->GetDescriptor(&cfdescriptor);
       column_families.push_back(cfdescriptor);
     }
+    Close();
     ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->DisableProcessing();
     Destroy(column_families);
     delete env_;
@@ -98,11 +109,11 @@ class ColumnFamilyTestBase : public testing::Test {
       // preserves the implementation that was in place when all of the
       // magic values in this file were picked.
       *storage = std::string(kValueSize, ' ');
+      return Slice(*storage);
     } else {
       Random r(k);
-      *storage = r.RandomString(kValueSize);
+      return test::RandomString(&r, kValueSize, storage);
     }
-    return Slice(*storage);
   }
 
   void Build(int base, int n, int flush_every = 0) {
@@ -111,7 +122,7 @@ class ColumnFamilyTestBase : public testing::Test {
 
     for (int i = 0; i < n; i++) {
       if (flush_every != 0 && i != 0 && i % flush_every == 0) {
-        DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
+        DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
         dbi->TEST_FlushMemTable();
       }
 
@@ -165,7 +176,7 @@ class ColumnFamilyTestBase : public testing::Test {
   void Close() {
     for (auto h : handles_) {
       if (h) {
-        ASSERT_OK(db_->DestroyColumnFamilyHandle(h));
+        db_->DestroyColumnFamilyHandle(h);
       }
     }
     handles_.clear();
@@ -179,8 +190,8 @@ class ColumnFamilyTestBase : public testing::Test {
     std::vector<ColumnFamilyDescriptor> column_families;
     names_.clear();
     for (size_t i = 0; i < cf.size(); ++i) {
-      column_families.emplace_back(
-          cf[i], options.size() == 0 ? column_family_options_ : options[i]);
+      column_families.push_back(ColumnFamilyDescriptor(
+          cf[i], options.size() == 0 ? column_family_options_ : options[i]));
       names_.push_back(cf[i]);
     }
     return DB::Open(db_options_, dbname_, column_families, &handles_, &db_);
@@ -191,8 +202,8 @@ class ColumnFamilyTestBase : public testing::Test {
     std::vector<ColumnFamilyDescriptor> column_families;
     names_.clear();
     for (size_t i = 0; i < cf.size(); ++i) {
-      column_families.emplace_back(
-          cf[i], options.size() == 0 ? column_family_options_ : options[i]);
+      column_families.push_back(ColumnFamilyDescriptor(
+          cf[i], options.size() == 0 ? column_family_options_ : options[i]));
       names_.push_back(cf[i]);
     }
     return DB::OpenForReadOnly(db_options_, dbname_, column_families, &handles_,
@@ -216,7 +227,7 @@ class ColumnFamilyTestBase : public testing::Test {
     Open({"default"});
   }
 
-  DBImpl* dbfull() { return static_cast_with_check<DBImpl>(db_); }
+  DBImpl* dbfull() { return reinterpret_cast<DBImpl*>(db_); }
 
   int GetProperty(int cf, std::string property) {
     std::string value;
@@ -276,11 +287,7 @@ class ColumnFamilyTestBase : public testing::Test {
       // Verify the CF options of the returned CF handle.
       ColumnFamilyDescriptor desc;
       ASSERT_OK(handles_[cfi]->GetDescriptor(&desc));
-      // Need to sanitize the default column family options before comparing
-      // them.
-      ASSERT_OK(RocksDBOptionsParser::VerifyCFOptions(
-          ConfigOptions(), desc.options,
-          SanitizeOptions(dbfull()->immutable_db_options(), current_cf_opt)));
+      RocksDBOptionsParser::VerifyCFOptions(desc.options, current_cf_opt);
 #endif  // !ROCKSDB_LITE
       cfi++;
     }
@@ -306,7 +313,7 @@ class ColumnFamilyTestBase : public testing::Test {
   void DropColumnFamilies(const std::vector<int>& cfs) {
     for (auto cf : cfs) {
       ASSERT_OK(db_->DropColumnFamily(handles_[cf]));
-      ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[cf]));
+      db_->DestroyColumnFamilyHandle(handles_[cf]);
       handles_[cf] = nullptr;
       names_[cf] = "";
     }
@@ -320,14 +327,14 @@ class ColumnFamilyTestBase : public testing::Test {
       // 10 bytes for key, rest is value
       if (!save) {
         ASSERT_OK(Put(cf, test::RandomKey(&rnd_, 11),
-                      rnd_.RandomString(key_value_size - 10)));
+                      RandomString(&rnd_, key_value_size - 10)));
       } else {
         std::string key = test::RandomKey(&rnd_, 11);
         keys_[cf].insert(key);
-        ASSERT_OK(Put(cf, key, rnd_.RandomString(key_value_size - 10)));
+        ASSERT_OK(Put(cf, key, RandomString(&rnd_, key_value_size - 10)));
       }
     }
-    ASSERT_OK(db_->FlushWAL(/*sync=*/false));
+    db_->FlushWAL(false);
   }
 
 #ifndef ROCKSDB_LITE  // TEST functions in DB are not supported in lite
@@ -561,7 +568,7 @@ TEST_P(ColumnFamilyTest, DontReuseColumnFamilyID) {
     Open();
     CreateColumnFamilies({"one", "two", "three"});
     for (size_t i = 0; i < handles_.size(); ++i) {
-      auto cfh = static_cast_with_check<ColumnFamilyHandleImpl>(handles_[i]);
+      auto cfh = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[i]);
       ASSERT_EQ(i, cfh->GetID());
     }
     if (iter == 1) {
@@ -577,7 +584,7 @@ TEST_P(ColumnFamilyTest, DontReuseColumnFamilyID) {
     CreateColumnFamilies({"three2"});
     // ID 3 that was used for dropped column family "three" should not be
     // reused
-    auto cfh3 = static_cast_with_check<ColumnFamilyHandleImpl>(handles_[3]);
+    auto cfh3 = reinterpret_cast<ColumnFamilyHandleImpl*>(handles_[3]);
     ASSERT_EQ(4U, cfh3->GetID());
     Close();
     Destroy();
@@ -645,11 +652,11 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest) {
   // after flushing file B is deleted. At the same time, the min log number of
   // default CF is not written to manifest. Log file A still remains.
   // Flushed to SST file Y.
-  ASSERT_OK(Flush(1));
-  ASSERT_OK(Flush(0));
+  Flush(1);
+  Flush(0);
   ASSERT_OK(Put(1, "bar", "v3"));  // seqID 4
   ASSERT_OK(Put(1, "foo", "v4"));  // seqID 5
-  ASSERT_OK(db_->FlushWAL(/*sync=*/false));
+  db_->FlushWAL(false);
 
   // Preserve file system state up to here to simulate a crash condition.
   fault_env->SetFilesystemActive(false);
@@ -700,19 +707,19 @@ TEST_P(FlushEmptyCFTestWithParam, FlushEmptyCFTest2) {
   // and is set to current. Both CFs' min log number is set to file C so after
   // flushing file B is deleted. Log file A still remains.
   // Flushed to SST file Y.
-  ASSERT_OK(Flush(1));
+  Flush(1);
   ASSERT_OK(Put(0, "bar", "v2"));  // seqID 4
   ASSERT_OK(Put(2, "bar", "v2"));  // seqID 5
   ASSERT_OK(Put(1, "bar", "v3"));  // seqID 6
   // Flushing all column families. This forces all CFs' min log to current. This
   // is written to the manifest file. Log file C is cleared.
-  ASSERT_OK(Flush(0));
-  ASSERT_OK(Flush(1));
-  ASSERT_OK(Flush(2));
+  Flush(0);
+  Flush(1);
+  Flush(2);
   // Write to log file D
   ASSERT_OK(Put(1, "bar", "v4"));  // seqID 7
   ASSERT_OK(Put(1, "bar", "v5"));  // seqID 8
-  ASSERT_OK(db_->FlushWAL(/*sync=*/false));
+  db_->FlushWAL(false);
   // Preserve file system state up to here to simulate a crash condition.
   fault_env->SetFilesystemActive(false);
   std::vector<std::string> names;
@@ -841,15 +848,13 @@ TEST_P(ColumnFamilyTest, WriteBatchFailure) {
   Open();
   CreateColumnFamiliesAndReopen({"one", "two"});
   WriteBatch batch;
-  ASSERT_OK(batch.Put(handles_[0], Slice("existing"), Slice("column-family")));
-  ASSERT_OK(
-      batch.Put(handles_[1], Slice("non-existing"), Slice("column-family")));
+  batch.Put(handles_[0], Slice("existing"), Slice("column-family"));
+  batch.Put(handles_[1], Slice("non-existing"), Slice("column-family"));
   ASSERT_OK(db_->Write(WriteOptions(), &batch));
   DropColumnFamilies({1});
   WriteOptions woptions_ignore_missing_cf;
   woptions_ignore_missing_cf.ignore_missing_column_families = true;
-  ASSERT_OK(
-      batch.Put(handles_[0], Slice("still here"), Slice("column-family")));
+  batch.Put(handles_[0], Slice("still here"), Slice("column-family"));
   ASSERT_OK(db_->Write(woptions_ignore_missing_cf, &batch));
   ASSERT_EQ("column-family", Get(0, "still here"));
   Status s = db_->Write(WriteOptions(), &batch);
@@ -888,9 +893,11 @@ TEST_P(ColumnFamilyTest, IgnoreRecoveredLog) {
   ASSERT_OK(env_->CreateDirIfMissing(dbname_));
   ASSERT_OK(env_->CreateDirIfMissing(backup_logs));
   std::vector<std::string> old_files;
-  ASSERT_OK(env_->GetChildren(backup_logs, &old_files));
+  env_->GetChildren(backup_logs, &old_files);
   for (auto& file : old_files) {
-    ASSERT_OK(env_->DeleteFile(backup_logs + "/" + file));
+    if (file != "." && file != "..") {
+      env_->DeleteFile(backup_logs + "/" + file);
+    }
   }
 
   column_family_options_.merge_operator =
@@ -917,9 +924,11 @@ TEST_P(ColumnFamilyTest, IgnoreRecoveredLog) {
 
   // copy the logs to backup
   std::vector<std::string> logs;
-  ASSERT_OK(env_->GetChildren(db_options_.wal_dir, &logs));
+  env_->GetChildren(db_options_.wal_dir, &logs);
   for (auto& log : logs) {
-    CopyFile(db_options_.wal_dir + "/" + log, backup_logs + "/" + log);
+    if (log != ".." && log != ".") {
+      CopyFile(db_options_.wal_dir + "/" + log, backup_logs + "/" + log);
+    }
   }
 
   // recover the DB
@@ -944,7 +953,9 @@ TEST_P(ColumnFamilyTest, IgnoreRecoveredLog) {
     if (iter == 0) {
       // copy the logs from backup back to wal dir
       for (auto& log : logs) {
-        CopyFile(backup_logs + "/" + log, db_options_.wal_dir + "/" + log);
+        if (log != ".." && log != ".") {
+          CopyFile(backup_logs + "/" + log, db_options_.wal_dir + "/" + log);
+        }
       }
     }
   }
@@ -971,14 +982,13 @@ TEST_P(ColumnFamilyTest, FlushTest) {
     for (int i = 0; i < 3; ++i) {
       uint64_t max_total_in_memory_state =
           MaxTotalInMemoryState();
-      ASSERT_OK(Flush(i));
+      Flush(i);
       AssertMaxTotalInMemoryState(max_total_in_memory_state);
     }
     ASSERT_OK(Put(1, "foofoo", "bar"));
     ASSERT_OK(Put(0, "foofoo", "bar"));
 
     for (auto* it : iterators) {
-      ASSERT_OK(it->status());
       delete it;
     }
   }
@@ -1076,10 +1086,10 @@ TEST_P(ColumnFamilyTest, CrashAfterFlush) {
   CreateColumnFamilies({"one"});
 
   WriteBatch batch;
-  ASSERT_OK(batch.Put(handles_[0], Slice("foo"), Slice("bar")));
-  ASSERT_OK(batch.Put(handles_[1], Slice("foo"), Slice("bar")));
+  batch.Put(handles_[0], Slice("foo"), Slice("bar"));
+  batch.Put(handles_[1], Slice("foo"), Slice("bar"));
   ASSERT_OK(db_->Write(WriteOptions(), &batch));
-  ASSERT_OK(Flush(0));
+  Flush(0);
   fault_env->SetFilesystemActive(false);
 
   std::vector<std::string> names;
@@ -1089,7 +1099,7 @@ TEST_P(ColumnFamilyTest, CrashAfterFlush) {
     }
   }
   Close();
-  ASSERT_OK(fault_env->DropUnsyncedFileData());
+  fault_env->DropUnsyncedFileData();
   fault_env->ResetState();
   Open(names, {});
 
@@ -2063,7 +2073,6 @@ std::string IterStatus(Iterator* iter) {
   if (iter->Valid()) {
     result = iter->key().ToString() + "->" + iter->value().ToString();
   } else {
-    EXPECT_OK(iter->status());
     result = "(invalid)";
   }
   return result;
@@ -2222,7 +2231,7 @@ TEST_P(ColumnFamilyTest, FlushStaleColumnFamilies) {
   // files for column family [one], because it's empty
   AssertCountLiveFiles(4);
 
-  ASSERT_OK(Flush(0));
+  Flush(0);
   ASSERT_EQ(0, dbfull()->TEST_total_log_size());
   Close();
 }
@@ -2278,8 +2287,6 @@ TEST_P(ColumnFamilyTest, SanitizeOptions) {
               // not a multiple of 4k, round up 4k
               expected_arena_block_size += 4 * 1024;
             }
-            expected_arena_block_size =
-                std::min(size_t{1024 * 1024}, expected_arena_block_size);
             ASSERT_EQ(expected_arena_block_size, result.arena_block_size);
           }
         }
@@ -2320,7 +2327,7 @@ TEST_P(ColumnFamilyTest, ReadDroppedColumnFamily) {
         ASSERT_OK(db_->DropColumnFamily(handles_[2]));
       } else {
         // delete CF two
-        ASSERT_OK(db_->DestroyColumnFamilyHandle(handles_[2]));
+        db_->DestroyColumnFamilyHandle(handles_[2]);
         handles_[2] = nullptr;
       }
       // Make sure iterator created can still be used.
@@ -2376,6 +2383,7 @@ TEST_P(ColumnFamilyTest, LiveIteratorWithDroppedColumnFamily) {
   // 1MB should create ~10 files for each CF
   int kKeysNum = 10000;
   PutRandomData(1, kKeysNum, 100);
+
   {
     std::unique_ptr<Iterator> iterator(
         db_->NewIterator(ReadOptions(), handles_[1]));
@@ -2422,9 +2430,6 @@ TEST_P(ColumnFamilyTest, FlushAndDropRaceCondition) {
 
   env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
                  Env::Priority::HIGH);
-  // Make sure the task is sleeping. Otherwise, it might start to execute
-  // after sleeping_task.WaitUntilDone() and cause TSAN warning.
-  sleeping_task.WaitUntilSleeping();
 
   // 1MB should create ~10 files for each CF
   int kKeysNum = 10000;
@@ -2439,9 +2444,6 @@ TEST_P(ColumnFamilyTest, FlushAndDropRaceCondition) {
   // now we sleep again. this is just so we're certain that flush job finished
   env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
                  Env::Priority::HIGH);
-  // Make sure the task is sleeping. Otherwise, it might start to execute
-  // after sleeping_task.WaitUntilDone() and cause TSAN warning.
-  sleeping_task.WaitUntilSleeping();
   sleeping_task.WakeUp();
   sleeping_task.WaitUntilDone();
 
@@ -2991,9 +2993,6 @@ TEST_P(ColumnFamilyTest, FlushCloseWALFiles) {
   test::SleepingBackgroundTask sleeping_task;
   env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
                  Env::Priority::HIGH);
-  // Make sure the task is sleeping. Otherwise, it might start to execute
-  // after sleeping_task.WaitUntilDone() and cause TSAN warning.
-  sleeping_task.WaitUntilSleeping();
 
   WriteOptions wo;
   wo.sync = true;
@@ -3026,9 +3025,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) {
   ASSERT_OK(Put(1, "fodor", "mirko"));
   // Create an iterator holding the current super version.
   Iterator* it = db_->NewIterator(ReadOptions(), handles_[1]);
-  ASSERT_OK(it->status());
   // A flush will make `it` hold the last reference of its super version.
-  ASSERT_OK(Flush(1));
+  Flush(1);
 
   ASSERT_OK(Put(1, "fodor", "mirko"));
   ASSERT_OK(Put(0, "fodor", "mirko"));
@@ -3040,9 +3038,6 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile1) {
   test::SleepingBackgroundTask sleeping_task;
   env_->Schedule(&test::SleepingBackgroundTask::DoSleepTask, &sleeping_task,
                  Env::Priority::HIGH);
-  // Make sure the task is sleeping. Otherwise, it might start to execute
-  // after sleeping_task.WaitUntilDone() and cause TSAN warning.
-  sleeping_task.WaitUntilSleeping();
 
   WriteOptions wo;
   wo.sync = true;
@@ -3079,9 +3074,8 @@ TEST_P(ColumnFamilyTest, IteratorCloseWALFile2) {
   ReadOptions ro;
   ro.background_purge_on_iterator_cleanup = true;
   Iterator* it = db_->NewIterator(ro, handles_[1]);
-  ASSERT_OK(it->status());
   // A flush will make `it` hold the last reference of its super version.
-  ASSERT_OK(Flush(1));
+  Flush(1);
 
   ASSERT_OK(Put(1, "fodor", "mirko"));
   ASSERT_OK(Put(0, "fodor", "mirko"));
@@ -3135,7 +3129,7 @@ TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
   CreateColumnFamilies({"one"});
   ASSERT_OK(Put(1, "fodor", "mirko"));
   ASSERT_OK(Put(1, "fodar2", "mirko"));
-  ASSERT_OK(Flush(1));
+  Flush(1);
 
   // Create an iterator holding the current super version, as well as
   // the SST file just flushed.
@@ -3147,7 +3141,7 @@ TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
 
   ASSERT_OK(Put(1, "fodor", "mirko"));
   ASSERT_OK(Put(1, "fodar2", "mirko"));
-  ASSERT_OK(Flush(1));
+  Flush(1);
 
   WaitForCompaction();
 
@@ -3174,8 +3168,6 @@ TEST_P(ColumnFamilyTest, ForwardIteratorCloseWALFile) {
   // Deleting the iterator will clear its super version, triggering
   // closing all files
   it->Seek("");
-  ASSERT_OK(it->status());
-
   ASSERT_EQ(2, env.num_open_wal_file_.load());
   ASSERT_EQ(0, env.delete_count_.load());
 
@@ -3206,8 +3198,8 @@ TEST_P(ColumnFamilyTest, LogSyncConflictFlush) {
   Open();
   CreateColumnFamiliesAndReopen({"one", "two"});
 
-  ASSERT_OK(Put(0, "", ""));
-  ASSERT_OK(Put(1, "foo", "bar"));
+  Put(0, "", "");
+  Put(1, "foo", "bar");
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->LoadDependency(
       {{"DBImpl::SyncWAL:BeforeMarkLogsSynced:1",
@@ -3217,12 +3209,12 @@ TEST_P(ColumnFamilyTest, LogSyncConflictFlush) {
 
   ROCKSDB_NAMESPACE::SyncPoint::GetInstance()->EnableProcessing();
 
-  ROCKSDB_NAMESPACE::port::Thread thread([&] { ASSERT_OK(db_->SyncWAL()); });
+  ROCKSDB_NAMESPACE::port::Thread thread([&] { db_->SyncWAL(); });
 
   TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:1");
-  ASSERT_OK(Flush(1));
-  ASSERT_OK(Put(1, "foo", "bar"));
-  ASSERT_OK(Flush(1));
+  Flush(1);
+  Put(1, "foo", "bar");
+  Flush(1);
 
   TEST_SYNC_POINT("ColumnFamilyTest::LogSyncConflictFlush:2");
 
@@ -3244,7 +3236,7 @@ TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) {
   Build(0, 100);
 
   // Flush the 0th column family to force a roll of the wal log
-  ASSERT_OK(Flush(0));
+  Flush(0);
 
   // Add some more entries
   Build(100, 100);
@@ -3259,7 +3251,7 @@ TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) {
     FileType type;
     if (!(ParseFileName(filenames[i], &number, &type))) continue;
 
-    if (type != kWalFile) continue;
+    if (type != kLogFile) continue;
 
     logfs.push_back(filenames[i]);
   }
@@ -3304,7 +3296,7 @@ TEST_P(ColumnFamilyTest, DISABLED_LogTruncationTest) {
   Close();
 
   // cleanup
-  ASSERT_OK(env_->DeleteDir(backup_logs));
+  env_->DeleteDir(backup_logs);
 }
 
 TEST_P(ColumnFamilyTest, DefaultCfPathsTest) {
@@ -3320,14 +3312,14 @@ TEST_P(ColumnFamilyTest, DefaultCfPathsTest) {
 
   // Fill Column family 1.
   PutRandomData(1, 100, 100);
-  ASSERT_OK(Flush(1));
+  Flush(1);
 
   ASSERT_EQ(1, GetSstFileCount(cf_opt1.cf_paths[0].path));
   ASSERT_EQ(0, GetSstFileCount(dbname_));
 
   // Fill column family 2
   PutRandomData(2, 100, 100);
-  ASSERT_OK(Flush(2));
+  Flush(2);
 
   // SST from Column family 2 should be generated in
   // db_paths which is dbname_ in this case.
@@ -3346,31 +3338,29 @@ TEST_P(ColumnFamilyTest, MultipleCFPathsTest) {
   Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2});
 
   PutRandomData(1, 100, 100, true /* save */);
-  ASSERT_OK(Flush(1));
+  Flush(1);
 
   // Check that files are generated in appropriate paths.
   ASSERT_EQ(1, GetSstFileCount(cf_opt1.cf_paths[0].path));
   ASSERT_EQ(0, GetSstFileCount(dbname_));
 
   PutRandomData(2, 100, 100, true /* save */);
-  ASSERT_OK(Flush(2));
+  Flush(2);
 
   ASSERT_EQ(1, GetSstFileCount(cf_opt2.cf_paths[0].path));
   ASSERT_EQ(0, GetSstFileCount(dbname_));
 
   // Re-open and verify the keys.
   Reopen({ColumnFamilyOptions(), cf_opt1, cf_opt2});
-  DBImpl* dbi = static_cast_with_check<DBImpl>(db_);
+  DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
   for (int cf = 1; cf != 3; ++cf) {
     ReadOptions read_options;
     read_options.readahead_size = 0;
     auto it = dbi->NewIterator(read_options, handles_[cf]);
     for (it->SeekToFirst(); it->Valid(); it->Next()) {
-      ASSERT_OK(it->status());
       Slice key(it->key());
       ASSERT_NE(keys_[cf].end(), keys_[cf].find(key.ToString()));
     }
-    ASSERT_OK(it->status());
     delete it;
 
     for (const auto& key : keys_[cf]) {
@@ -3379,30 +3369,6 @@ TEST_P(ColumnFamilyTest, MultipleCFPathsTest) {
   }
 }
 
-TEST(ColumnFamilyTest, ValidateBlobGCCutoff) {
-  DBOptions db_options;
-
-  ColumnFamilyOptions cf_options;
-  cf_options.enable_blob_garbage_collection = true;
-
-  cf_options.blob_garbage_collection_age_cutoff = -0.5;
-  ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options)
-                  .IsInvalidArgument());
-
-  cf_options.blob_garbage_collection_age_cutoff = 0.0;
-  ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options));
-
-  cf_options.blob_garbage_collection_age_cutoff = 0.5;
-  ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options));
-
-  cf_options.blob_garbage_collection_age_cutoff = 1.0;
-  ASSERT_OK(ColumnFamilyData::ValidateOptions(db_options, cf_options));
-
-  cf_options.blob_garbage_collection_age_cutoff = 1.5;
-  ASSERT_TRUE(ColumnFamilyData::ValidateOptions(db_options, cf_options)
-                  .IsInvalidArgument());
-}
-
 }  // namespace ROCKSDB_NAMESPACE
 
 #ifdef ROCKSDB_UNITTESTS_WITH_CUSTOM_OBJECTS_FROM_STATIC_LIBS

db/compact_files_test.cc

@@ -16,7 +16,6 @@
 #include "rocksdb/env.h"
 #include "test_util/sync_point.h"
 #include "test_util/testharness.h"
-#include "util/cast_util.h"
 #include "util/string_util.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -91,9 +90,9 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
   // create couple files
   // Background compaction starts and waits in BackgroundCallCompaction:0
   for (int i = 0; i < kLevel0Trigger * 4; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), ToString(i), ""));
-    ASSERT_OK(db->Put(WriteOptions(), ToString(100 - i), ""));
-    ASSERT_OK(db->Flush(FlushOptions()));
+    db->Put(WriteOptions(), ToString(i), "");
+    db->Put(WriteOptions(), ToString(100 - i), "");
+    db->Flush(FlushOptions());
   }
 
   ROCKSDB_NAMESPACE::ColumnFamilyMetaData meta;
@@ -118,78 +117,6 @@ TEST_F(CompactFilesTest, L0ConflictsFiles) {
   delete db;
 }
 
-TEST_F(CompactFilesTest, MultipleLevel) {
-  Options options;
-  options.create_if_missing = true;
-  options.level_compaction_dynamic_level_bytes = true;
-  options.num_levels = 6;
-  // Add listener
-  FlushedFileCollector* collector = new FlushedFileCollector();
-  options.listeners.emplace_back(collector);
-
-  DB* db = nullptr;
-  DestroyDB(db_name_, options);
-  Status s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
-  ASSERT_NE(db, nullptr);
-
-  // create couple files in L0, L3, L4 and L5
-  for (int i = 5; i > 2; --i) {
-    collector->ClearFlushedFiles();
-    ASSERT_OK(db->Put(WriteOptions(), ToString(i), ""));
-    ASSERT_OK(db->Flush(FlushOptions()));
-    auto l0_files = collector->GetFlushedFiles();
-    ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, i));
-
-    std::string prop;
-    ASSERT_TRUE(
-        db->GetProperty("rocksdb.num-files-at-level" + ToString(i), &prop));
-    ASSERT_EQ("1", prop);
-  }
-  ASSERT_OK(db->Put(WriteOptions(), ToString(0), ""));
-  ASSERT_OK(db->Flush(FlushOptions()));
-
-  ColumnFamilyMetaData meta;
-  db->GetColumnFamilyMetaData(&meta);
-  // Compact files except the file in L3
-  std::vector<std::string> files;
-  for (int i = 0; i < 6; ++i) {
-    if (i == 3) continue;
-    for (auto& file : meta.levels[i].files) {
-      files.push_back(file.db_path + "/" + file.name);
-    }
-  }
-
-  SyncPoint::GetInstance()->LoadDependency({
-      {"CompactionJob::Run():Start", "CompactFilesTest.MultipleLevel:0"},
-      {"CompactFilesTest.MultipleLevel:1", "CompactFilesImpl:3"},
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  std::thread thread([&] {
-    TEST_SYNC_POINT("CompactFilesTest.MultipleLevel:0");
-    ASSERT_OK(db->Put(WriteOptions(), "bar", "v2"));
-    ASSERT_OK(db->Put(WriteOptions(), "foo", "v2"));
-    ASSERT_OK(db->Flush(FlushOptions()));
-    TEST_SYNC_POINT("CompactFilesTest.MultipleLevel:1");
-  });
-
-  // Compaction cannot move up the data to higher level
-  // here we have input file from level 5, so the output level has to be >= 5
-  for (int invalid_output_level = 0; invalid_output_level < 5;
-       invalid_output_level++) {
-    s = db->CompactFiles(CompactionOptions(), files, invalid_output_level);
-    std::cout << s.ToString() << std::endl;
-    ASSERT_TRUE(s.IsInvalidArgument());
-  }
-
-  ASSERT_OK(db->CompactFiles(CompactionOptions(), files, 5));
-  SyncPoint::GetInstance()->DisableProcessing();
-  thread.join();
-
-  delete db;
-}
-
 TEST_F(CompactFilesTest, ObsoleteFiles) {
   Options options;
   // to trigger compaction more easily
@@ -210,18 +137,18 @@ TEST_F(CompactFilesTest, ObsoleteFiles) {
   DB* db = nullptr;
   DestroyDB(db_name_, options);
   Status s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
-  ASSERT_NE(db, nullptr);
+  assert(s.ok());
+  assert(db);
 
   // create couple files
   for (int i = 1000; i < 2000; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), ToString(i),
-                      std::string(kWriteBufferSize / 10, 'a' + (i % 26))));
+    db->Put(WriteOptions(), ToString(i),
+            std::string(kWriteBufferSize / 10, 'a' + (i % 26)));
   }
 
   auto l0_files = collector->GetFlushedFiles();
   ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files, 1));
-  ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForCompact());
+  reinterpret_cast<DBImpl*>(db)->TEST_WaitForCompact();
 
   // verify all compaction input files are deleted
   for (auto fname : l0_files) {
@@ -254,17 +181,15 @@ TEST_F(CompactFilesTest, NotCutOutputOnLevel0) {
 
   // create couple files
   for (int i = 0; i < 500; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), ToString(i),
-                      std::string(1000, 'a' + (i % 26))));
+    db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
   }
-  ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
+  reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
   auto l0_files_1 = collector->GetFlushedFiles();
   collector->ClearFlushedFiles();
   for (int i = 0; i < 500; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), ToString(i),
-                      std::string(1000, 'a' + (i % 26))));
+    db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
   }
-  ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
+  reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
   auto l0_files_2 = collector->GetFlushedFiles();
   ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_1, 0));
   ASSERT_OK(db->CompactFiles(CompactionOptions(), l0_files_2, 0));
@@ -287,13 +212,13 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
   DB* db = nullptr;
   DestroyDB(db_name_, options);
   Status s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
+  assert(s.ok());
   assert(db);
 
   // Create 5 files.
   for (int i = 0; i < 5; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), "key" + ToString(i), "value"));
-    ASSERT_OK(db->Flush(FlushOptions()));
+    db->Put(WriteOptions(), "key" + ToString(i), "value");
+    db->Flush(FlushOptions());
   }
 
   auto l0_files = collector->GetFlushedFiles();
@@ -311,8 +236,8 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
 
   // In the meantime flush another file.
   TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:0");
-  ASSERT_OK(db->Put(WriteOptions(), "key5", "value"));
-  ASSERT_OK(db->Flush(FlushOptions()));
+  db->Put(WriteOptions(), "key5", "value");
+  db->Flush(FlushOptions());
   TEST_SYNC_POINT("CompactFilesTest.CapturingPendingFiles:1");
 
   compaction_thread.join();
@@ -323,7 +248,7 @@ TEST_F(CompactFilesTest, CapturingPendingFiles) {
 
   // Make sure we can reopen the DB.
   s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
+  ASSERT_TRUE(s.ok());
   assert(db);
   delete db;
 }
@@ -367,8 +292,8 @@ TEST_F(CompactFilesTest, CompactionFilterWithGetSv) {
   cf->SetDB(db);
 
   // Write one L0 file
-  ASSERT_OK(db->Put(WriteOptions(), "K1", "V1"));
-  ASSERT_OK(db->Flush(FlushOptions()));
+  db->Put(WriteOptions(), "K1", "V1");
+  db->Flush(FlushOptions());
 
   // Compact all L0 files using CompactFiles
   ROCKSDB_NAMESPACE::ColumnFamilyMetaData meta;
@@ -411,8 +336,8 @@ TEST_F(CompactFilesTest, SentinelCompressionType) {
     DB* db = nullptr;
     ASSERT_OK(DB::Open(options, db_name_, &db));
 
-    ASSERT_OK(db->Put(WriteOptions(), "key", "val"));
-    ASSERT_OK(db->Flush(FlushOptions()));
+    db->Put(WriteOptions(), "key", "val");
+    db->Flush(FlushOptions());
 
     auto l0_files = collector->GetFlushedFiles();
     ASSERT_EQ(1, l0_files.size());
@@ -451,15 +376,14 @@ TEST_F(CompactFilesTest, GetCompactionJobInfo) {
   DB* db = nullptr;
   DestroyDB(db_name_, options);
   Status s = DB::Open(options, db_name_, &db);
-  ASSERT_OK(s);
+  assert(s.ok());
   assert(db);
 
   // create couple files
   for (int i = 0; i < 500; ++i) {
-    ASSERT_OK(db->Put(WriteOptions(), ToString(i),
-                      std::string(1000, 'a' + (i % 26))));
+    db->Put(WriteOptions(), ToString(i), std::string(1000, 'a' + (i % 26)));
   }
-  ASSERT_OK(static_cast_with_check<DBImpl>(db)->TEST_WaitForFlushMemTable());
+  reinterpret_cast<DBImpl*>(db)->TEST_WaitForFlushMemTable();
   auto l0_files_1 = collector->GetFlushedFiles();
   CompactionOptions co;
   co.compression = CompressionType::kLZ4Compression;

db/compacted_db_impl.cc

@@ -4,12 +4,10 @@
 //  (found in the LICENSE.Apache file in the root directory).
 
 #ifndef ROCKSDB_LITE
-#include "db/db_impl/compacted_db_impl.h"
-
+#include "db/compacted_db_impl.h"
 #include "db/db_impl/db_impl.h"
 #include "db/version_set.h"
 #include "table/get_context.h"
-#include "util/cast_util.h"
 
 namespace ROCKSDB_NAMESPACE {
 
@@ -17,13 +15,11 @@ extern void MarkKeyMayExist(void* arg);
 extern bool SaveValue(void* arg, const ParsedInternalKey& parsed_key,
                       const Slice& v, bool hit_and_return);
 
-CompactedDBImpl::CompactedDBImpl(const DBOptions& options,
-                                 const std::string& dbname)
-    : DBImpl(options, dbname, /*seq_per_batch*/ false, +/*batch_per_txn*/ true,
-             /*read_only*/ true),
-      cfd_(nullptr),
-      version_(nullptr),
-      user_comparator_(nullptr) {}
+CompactedDBImpl::CompactedDBImpl(
+  const DBOptions& options, const std::string& dbname)
+  : DBImpl(options, dbname), cfd_(nullptr), version_(nullptr),
+    user_comparator_(nullptr) {
+}
 
 CompactedDBImpl::~CompactedDBImpl() {
 }
@@ -43,11 +39,8 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
                          GetContext::kNotFound, key, value, nullptr, nullptr,
                          nullptr, true, nullptr, nullptr);
   LookupKey lkey(key, kMaxSequenceNumber);
-  Status s = files_.files[FindFile(key)].fd.table_reader->Get(
-      options, lkey.internal_key(), &get_context, nullptr);
-  if (!s.ok() && !s.IsNotFound()) {
-    return s;
-  }
+  files_.files[FindFile(key)].fd.table_reader->Get(options, lkey.internal_key(),
+                                                   &get_context, nullptr);
   if (get_context.State() == GetContext::kFound) {
     return Status::OK();
   }
@@ -79,15 +72,10 @@ std::vector<Status> CompactedDBImpl::MultiGet(const ReadOptions& options,
                              GetContext::kNotFound, keys[idx], &pinnable_val,
                              nullptr, nullptr, nullptr, true, nullptr, nullptr);
       LookupKey lkey(keys[idx], kMaxSequenceNumber);
-      Status s = r->Get(options, lkey.internal_key(), &get_context, nullptr);
-      assert(static_cast<size_t>(idx) < statuses.size());
-      if (!s.ok() && !s.IsNotFound()) {
-        statuses[idx] = s;
-      } else {
-        value.assign(pinnable_val.data(), pinnable_val.size());
-        if (get_context.State() == GetContext::kFound) {
-          statuses[idx] = Status::OK();
-        }
+      r->Get(options, lkey.internal_key(), &get_context, nullptr);
+      value.assign(pinnable_val.data(), pinnable_val.size());
+      if (get_context.State() == GetContext::kFound) {
+        statuses[idx] = Status::OK();
       }
     }
     ++idx;
@@ -102,8 +90,8 @@ Status CompactedDBImpl::Init(const Options& options) {
                             ColumnFamilyOptions(options));
   Status s = Recover({cf}, true /* read only */, false, true);
   if (s.ok()) {
-    cfd_ = static_cast_with_check<ColumnFamilyHandleImpl>(DefaultColumnFamily())
-               ->cfd();
+    cfd_ = reinterpret_cast<ColumnFamilyHandleImpl*>(
+              DefaultColumnFamily())->cfd();
     cfd_->InstallSuperVersion(&sv_context, &mutex_);
   }
   mutex_.Unlock();
@@ -159,7 +147,7 @@ Status CompactedDBImpl::Open(const Options& options,
   std::unique_ptr<CompactedDBImpl> db(new CompactedDBImpl(db_options, dbname));
   Status s = db->Init(options);
   if (s.ok()) {
-    db->StartPeriodicWorkScheduler();
+    db->StartTimedTasks();
     ROCKS_LOG_INFO(db->immutable_db_options_.info_log,
                    "Opened the db as fully compacted mode");
     LogFlush(db->immutable_db_options_.info_log);

db/compacted_db_impl.h

@@ -18,7 +18,7 @@ class CompactedDBImpl : public DBImpl {
   CompactedDBImpl(const CompactedDBImpl&) = delete;
   void operator=(const CompactedDBImpl&) = delete;
 
-  ~CompactedDBImpl() override;
+  virtual ~CompactedDBImpl();
 
   static Status Open(const Options& options, const std::string& dbname,
                      DB** dbptr);
@@ -82,11 +82,6 @@ class CompactedDBImpl : public DBImpl {
                        ColumnFamilyHandle* /*column_family*/) override {
     return Status::NotSupported("Not supported in compacted db mode.");
   }
-
-  virtual Status SyncWAL() override {
-    return Status::NotSupported("Not supported in compacted db mode.");
-  }
-
   using DB::IngestExternalFile;
   virtual Status IngestExternalFile(
       ColumnFamilyHandle* /*column_family*/,

db/compaction/clipping_iterator.h

@@ -1,275 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-
-#include "table/internal_iterator.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// An internal iterator that wraps another one and ensures that any keys
-// returned are strictly within a range [start, end). If the underlying
-// iterator has already performed the bounds checking, it relies on that result;
-// otherwise, it performs the necessary key comparisons itself. Both bounds
-// are optional.
-class ClippingIterator : public InternalIterator {
- public:
-  ClippingIterator(InternalIterator* iter, const Slice* start, const Slice* end,
-                   const Comparator* cmp)
-      : iter_(iter), start_(start), end_(end), cmp_(cmp), valid_(false) {
-    assert(iter_);
-    assert(cmp_);
-    assert(!start_ || !end_ || cmp_->Compare(*start_, *end_) <= 0);
-
-    UpdateAndEnforceBounds();
-  }
-
-  bool Valid() const override { return valid_; }
-
-  void SeekToFirst() override {
-    if (start_) {
-      iter_->Seek(*start_);
-    } else {
-      iter_->SeekToFirst();
-    }
-
-    UpdateAndEnforceUpperBound();
-  }
-
-  void SeekToLast() override {
-    if (end_) {
-      iter_->SeekForPrev(*end_);
-
-      // Upper bound is exclusive, so we need a key which is strictly smaller
-      if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) {
-        iter_->Prev();
-      }
-    } else {
-      iter_->SeekToLast();
-    }
-
-    UpdateAndEnforceLowerBound();
-  }
-
-  void Seek(const Slice& target) override {
-    if (start_ && cmp_->Compare(target, *start_) < 0) {
-      iter_->Seek(*start_);
-      UpdateAndEnforceUpperBound();
-      return;
-    }
-
-    if (end_ && cmp_->Compare(target, *end_) >= 0) {
-      valid_ = false;
-      return;
-    }
-
-    iter_->Seek(target);
-    UpdateAndEnforceUpperBound();
-  }
-
-  void SeekForPrev(const Slice& target) override {
-    if (start_ && cmp_->Compare(target, *start_) < 0) {
-      valid_ = false;
-      return;
-    }
-
-    if (end_ && cmp_->Compare(target, *end_) >= 0) {
-      iter_->SeekForPrev(*end_);
-
-      // Upper bound is exclusive, so we need a key which is strictly smaller
-      if (iter_->Valid() && cmp_->Compare(iter_->key(), *end_) == 0) {
-        iter_->Prev();
-      }
-
-      UpdateAndEnforceLowerBound();
-      return;
-    }
-
-    iter_->SeekForPrev(target);
-    UpdateAndEnforceLowerBound();
-  }
-
-  void Next() override {
-    assert(valid_);
-    iter_->Next();
-    UpdateAndEnforceUpperBound();
-  }
-
-  bool NextAndGetResult(IterateResult* result) override {
-    assert(valid_);
-    assert(result);
-
-    IterateResult res;
-    valid_ = iter_->NextAndGetResult(&res);
-
-    if (!valid_) {
-      return false;
-    }
-
-    if (end_) {
-      EnforceUpperBoundImpl(res.bound_check_result);
-
-      if (!valid_) {
-        return false;
-      }
-    }
-
-    res.bound_check_result = IterBoundCheck::kInbound;
-    *result = res;
-
-    return true;
-  }
-
-  void Prev() override {
-    assert(valid_);
-    iter_->Prev();
-    UpdateAndEnforceLowerBound();
-  }
-
-  Slice key() const override {
-    assert(valid_);
-    return iter_->key();
-  }
-
-  Slice user_key() const override {
-    assert(valid_);
-    return iter_->user_key();
-  }
-
-  Slice value() const override {
-    assert(valid_);
-    return iter_->value();
-  }
-
-  Status status() const override { return iter_->status(); }
-
-  bool PrepareValue() override {
-    assert(valid_);
-
-    if (iter_->PrepareValue()) {
-      return true;
-    }
-
-    assert(!iter_->Valid());
-    valid_ = false;
-    return false;
-  }
-
-  bool MayBeOutOfLowerBound() override {
-    assert(valid_);
-    return false;
-  }
-
-  IterBoundCheck UpperBoundCheckResult() override {
-    assert(valid_);
-    return IterBoundCheck::kInbound;
-  }
-
-  void SetPinnedItersMgr(PinnedIteratorsManager* pinned_iters_mgr) override {
-    iter_->SetPinnedItersMgr(pinned_iters_mgr);
-  }
-
-  bool IsKeyPinned() const override {
-    assert(valid_);
-    return iter_->IsKeyPinned();
-  }
-
-  bool IsValuePinned() const override {
-    assert(valid_);
-    return iter_->IsValuePinned();
-  }
-
-  Status GetProperty(std::string prop_name, std::string* prop) override {
-    return iter_->GetProperty(prop_name, prop);
-  }
-
- private:
-  void UpdateValid() {
-    assert(!iter_->Valid() || iter_->status().ok());
-
-    valid_ = iter_->Valid();
-  }
-
-  void EnforceUpperBoundImpl(IterBoundCheck bound_check_result) {
-    if (bound_check_result == IterBoundCheck::kInbound) {
-      return;
-    }
-
-    if (bound_check_result == IterBoundCheck::kOutOfBound) {
-      valid_ = false;
-      return;
-    }
-
-    assert(bound_check_result == IterBoundCheck::kUnknown);
-
-    if (cmp_->Compare(key(), *end_) >= 0) {
-      valid_ = false;
-    }
-  }
-
-  void EnforceUpperBound() {
-    if (!valid_) {
-      return;
-    }
-
-    if (!end_) {
-      return;
-    }
-
-    EnforceUpperBoundImpl(iter_->UpperBoundCheckResult());
-  }
-
-  void EnforceLowerBound() {
-    if (!valid_) {
-      return;
-    }
-
-    if (!start_) {
-      return;
-    }
-
-    if (!iter_->MayBeOutOfLowerBound()) {
-      return;
-    }
-
-    if (cmp_->Compare(key(), *start_) < 0) {
-      valid_ = false;
-    }
-  }
-
-  void AssertBounds() {
-    assert(!valid_ || !start_ || cmp_->Compare(key(), *start_) >= 0);
-    assert(!valid_ || !end_ || cmp_->Compare(key(), *end_) < 0);
-  }
-
-  void UpdateAndEnforceBounds() {
-    UpdateValid();
-    EnforceUpperBound();
-    EnforceLowerBound();
-    AssertBounds();
-  }
-
-  void UpdateAndEnforceUpperBound() {
-    UpdateValid();
-    EnforceUpperBound();
-    AssertBounds();
-  }
-
-  void UpdateAndEnforceLowerBound() {
-    UpdateValid();
-    EnforceLowerBound();
-    AssertBounds();
-  }
-
-  InternalIterator* iter_;
-  const Slice* start_;
-  const Slice* end_;
-  const Comparator* cmp_;
-  bool valid_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/compaction/clipping_iterator_test.cc

@@ -1,256 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/compaction/clipping_iterator.h"
-
-#include <algorithm>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "rocksdb/comparator.h"
-#include "test_util/testharness.h"
-#include "test_util/testutil.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// A vector iterator which does its own bounds checking. This is for testing the
-// optimizations in the clipping iterator where we bypass the bounds checking if
-// the input iterator has already performed it.
-class BoundsCheckingVectorIterator : public test::VectorIterator {
- public:
-  BoundsCheckingVectorIterator(const std::vector<std::string>& keys,
-                               const std::vector<std::string>& values,
-                               const Slice* start, const Slice* end,
-                               const Comparator* cmp)
-      : VectorIterator(keys, values), start_(start), end_(end), cmp_(cmp) {
-    assert(cmp_);
-  }
-
-  bool NextAndGetResult(IterateResult* result) override {
-    assert(Valid());
-    assert(result);
-
-    Next();
-
-    if (!Valid()) {
-      return false;
-    }
-
-    result->key = key();
-    result->bound_check_result = UpperBoundCheckResult();
-    result->value_prepared = true;
-
-    return true;
-  }
-
-  bool MayBeOutOfLowerBound() override {
-    assert(Valid());
-
-    if (!start_) {
-      return false;
-    }
-
-    return cmp_->Compare(key(), *start_) < 0;
-  }
-
-  IterBoundCheck UpperBoundCheckResult() override {
-    assert(Valid());
-
-    if (!end_) {
-      return IterBoundCheck::kInbound;
-    }
-
-    return cmp_->Compare(key(), *end_) >= 0 ? IterBoundCheck::kOutOfBound
-                                            : IterBoundCheck::kInbound;
-  }
-
- private:
-  const Slice* start_;
-  const Slice* end_;
-  const Comparator* cmp_;
-};
-
-class ClippingIteratorTest
-    : public ::testing::Test,
-      public ::testing::WithParamInterface<std::tuple<bool, size_t, size_t>> {};
-
-TEST_P(ClippingIteratorTest, Clip) {
-  const std::vector<std::string> keys{"key0", "key1", "key2", "key3", "key4",
-                                      "key5", "key6", "key7", "key8", "key9"};
-  const std::vector<std::string> values{
-      "unused0", "value1",  "value2",  "value3",  "unused4",
-      "unused5", "unused6", "unused7", "unused8", "unused9"};
-
-  assert(keys.size() == values.size());
-
-  // Note: the input always contains key1, key2, and key3; however, the clipping
-  // window is based on the test parameters: its left edge is a value in the
-  // range [0, 4], and its size is a value in the range [0, 5]
-  const std::vector<std::string> input_keys{keys[1], keys[2], keys[3]};
-  const std::vector<std::string> input_values{values[1], values[2], values[3]};
-
-  const bool use_bounds_checking_vec_it = std::get<0>(GetParam());
-
-  const size_t clip_start_idx = std::get<1>(GetParam());
-  const size_t clip_window_size = std::get<2>(GetParam());
-  const size_t clip_end_idx = clip_start_idx + clip_window_size;
-
-  const Slice start(keys[clip_start_idx]);
-  const Slice end(keys[clip_end_idx]);
-
-  std::unique_ptr<InternalIterator> input(
-      use_bounds_checking_vec_it
-          ? new BoundsCheckingVectorIterator(input_keys, input_values, &start,
-                                             &end, BytewiseComparator())
-          : new test::VectorIterator(input_keys, input_values));
-
-  ClippingIterator clip(input.get(), &start, &end, BytewiseComparator());
-
-  // The range the clipping iterator should return values from. This is
-  // essentially the intersection of the input range [1, 4) and the clipping
-  // window [clip_start_idx, clip_end_idx)
-  const size_t data_start_idx =
-      std::max(clip_start_idx, static_cast<size_t>(1));
-  const size_t data_end_idx = std::min(clip_end_idx, static_cast<size_t>(4));
-
-  // Range is empty; all Seeks should fail
-  if (data_start_idx >= data_end_idx) {
-    clip.SeekToFirst();
-    ASSERT_FALSE(clip.Valid());
-
-    clip.SeekToLast();
-    ASSERT_FALSE(clip.Valid());
-
-    for (size_t i = 0; i < keys.size(); ++i) {
-      clip.Seek(keys[i]);
-      ASSERT_FALSE(clip.Valid());
-
-      clip.SeekForPrev(keys[i]);
-      ASSERT_FALSE(clip.Valid());
-    }
-
-    return;
-  }
-
-  // Range is non-empty; call SeekToFirst and iterate forward
-  clip.SeekToFirst();
-  ASSERT_TRUE(clip.Valid());
-  ASSERT_EQ(clip.key(), keys[data_start_idx]);
-  ASSERT_EQ(clip.value(), values[data_start_idx]);
-  ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-  ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-
-  for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) {
-    clip.Next();
-    ASSERT_TRUE(clip.Valid());
-    ASSERT_EQ(clip.key(), keys[i]);
-    ASSERT_EQ(clip.value(), values[i]);
-    ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-    ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-  }
-
-  clip.Next();
-  ASSERT_FALSE(clip.Valid());
-
-  // Do it again using NextAndGetResult
-  clip.SeekToFirst();
-  ASSERT_TRUE(clip.Valid());
-  ASSERT_EQ(clip.key(), keys[data_start_idx]);
-  ASSERT_EQ(clip.value(), values[data_start_idx]);
-  ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-  ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-
-  for (size_t i = data_start_idx + 1; i < data_end_idx; ++i) {
-    IterateResult result;
-    ASSERT_TRUE(clip.NextAndGetResult(&result));
-    ASSERT_EQ(result.key, keys[i]);
-    ASSERT_EQ(result.bound_check_result, IterBoundCheck::kInbound);
-    ASSERT_TRUE(clip.Valid());
-    ASSERT_EQ(clip.key(), keys[i]);
-    ASSERT_EQ(clip.value(), values[i]);
-    ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-    ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-  }
-
-  IterateResult result;
-  ASSERT_FALSE(clip.NextAndGetResult(&result));
-  ASSERT_FALSE(clip.Valid());
-
-  // Call SeekToLast and iterate backward
-  clip.SeekToLast();
-  ASSERT_TRUE(clip.Valid());
-  ASSERT_EQ(clip.key(), keys[data_end_idx - 1]);
-  ASSERT_EQ(clip.value(), values[data_end_idx - 1]);
-  ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-  ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-
-  for (size_t i = data_end_idx - 2; i >= data_start_idx; --i) {
-    clip.Prev();
-    ASSERT_TRUE(clip.Valid());
-    ASSERT_EQ(clip.key(), keys[i]);
-    ASSERT_EQ(clip.value(), values[i]);
-    ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-    ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-  }
-
-  clip.Prev();
-  ASSERT_FALSE(clip.Valid());
-
-  // Call Seek/SeekForPrev for all keys; Seek should return the smallest key
-  // which is >= the target; SeekForPrev should return the largest key which is
-  // <= the target
-  for (size_t i = 0; i < keys.size(); ++i) {
-    clip.Seek(keys[i]);
-
-    if (i < data_start_idx) {
-      ASSERT_TRUE(clip.Valid());
-      ASSERT_EQ(clip.key(), keys[data_start_idx]);
-      ASSERT_EQ(clip.value(), values[data_start_idx]);
-      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-    } else if (i < data_end_idx) {
-      ASSERT_TRUE(clip.Valid());
-      ASSERT_EQ(clip.key(), keys[i]);
-      ASSERT_EQ(clip.value(), values[i]);
-      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-    } else {
-      ASSERT_FALSE(clip.Valid());
-    }
-
-    clip.SeekForPrev(keys[i]);
-
-    if (i < data_start_idx) {
-      ASSERT_FALSE(clip.Valid());
-    } else if (i < data_end_idx) {
-      ASSERT_TRUE(clip.Valid());
-      ASSERT_EQ(clip.key(), keys[i]);
-      ASSERT_EQ(clip.value(), values[i]);
-      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-    } else {
-      ASSERT_TRUE(clip.Valid());
-      ASSERT_EQ(clip.key(), keys[data_end_idx - 1]);
-      ASSERT_EQ(clip.value(), values[data_end_idx - 1]);
-      ASSERT_FALSE(clip.MayBeOutOfLowerBound());
-      ASSERT_EQ(clip.UpperBoundCheckResult(), IterBoundCheck::kInbound);
-    }
-  }
-}
-
-INSTANTIATE_TEST_CASE_P(
-    ClippingIteratorTest, ClippingIteratorTest,
-    ::testing::Combine(
-        ::testing::Bool(),
-        ::testing::Range(static_cast<size_t>(0), static_cast<size_t>(5)),
-        ::testing::Range(static_cast<size_t>(0), static_cast<size_t>(6))));
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/compaction/compaction.cc

@@ -7,14 +7,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
-#include "db/compaction/compaction.h"
-
 #include <cinttypes>
 #include <vector>
 
 #include "db/column_family.h"
+#include "db/compaction/compaction.h"
 #include "rocksdb/compaction_filter.h"
-#include "rocksdb/sst_partitioner.h"
 #include "test_util/sync_point.h"
 #include "util/string_util.h"
 
@@ -25,7 +23,7 @@ const uint64_t kRangeTombstoneSentinel =
 
 int sstableKeyCompare(const Comparator* user_cmp, const InternalKey& a,
                       const InternalKey& b) {
-  auto c = user_cmp->CompareWithoutTimestamp(a.user_key(), b.user_key());
+  auto c = user_cmp->Compare(a.user_key(), b.user_key());
   if (c != 0) {
     return c;
   }
@@ -204,24 +202,26 @@ bool Compaction::IsFullCompaction(
   return num_files_in_compaction == total_num_files;
 }
 
-Compaction::Compaction(
-    VersionStorageInfo* vstorage, const ImmutableOptions& _immutable_options,
-    const MutableCFOptions& _mutable_cf_options,
-    const MutableDBOptions& _mutable_db_options,
-    std::vector<CompactionInputFiles> _inputs, int _output_level,
-    uint64_t _target_file_size, uint64_t _max_compaction_bytes,
-    uint32_t _output_path_id, CompressionType _compression,
-    CompressionOptions _compression_opts, uint32_t _max_subcompactions,
-    std::vector<FileMetaData*> _grandparents, bool _manual_compaction,
-    double _score, bool _deletion_compaction,
-    CompactionReason _compaction_reason)
+Compaction::Compaction(VersionStorageInfo* vstorage,
+                       const ImmutableCFOptions& _immutable_cf_options,
+                       const MutableCFOptions& _mutable_cf_options,
+                       std::vector<CompactionInputFiles> _inputs,
+                       int _output_level, uint64_t _target_file_size,
+                       uint64_t _max_compaction_bytes, uint32_t _output_path_id,
+                       CompressionType _compression,
+                       CompressionOptions _compression_opts,
+                       uint32_t _max_subcompactions,
+                       std::vector<FileMetaData*> _grandparents,
+                       bool _manual_compaction, double _score,
+                       bool _deletion_compaction,
+                       CompactionReason _compaction_reason)
     : input_vstorage_(vstorage),
       start_level_(_inputs[0].level),
       output_level_(_output_level),
       max_output_file_size_(_target_file_size),
       max_compaction_bytes_(_max_compaction_bytes),
       max_subcompactions_(_max_subcompactions),
-      immutable_options_(_immutable_options),
+      immutable_cf_options_(_immutable_cf_options),
       mutable_cf_options_(_mutable_cf_options),
       input_version_(nullptr),
       number_levels_(vstorage->num_levels()),
@@ -237,14 +237,19 @@ Compaction::Compaction(
       is_full_compaction_(IsFullCompaction(vstorage, inputs_)),
       is_manual_compaction_(_manual_compaction),
       is_trivial_move_(false),
-      compaction_reason_(_compaction_reason),
-      notify_on_compaction_completion_(false) {
+      compaction_reason_(_compaction_reason) {
   MarkFilesBeingCompacted(true);
   if (is_manual_compaction_) {
     compaction_reason_ = CompactionReason::kManualCompaction;
   }
   if (max_subcompactions_ == 0) {
-    max_subcompactions_ = _mutable_db_options.max_subcompactions;
+    max_subcompactions_ = immutable_cf_options_.max_subcompactions;
+  }
+  if (!bottommost_level_) {
+    // Currently we only enable dictionary compression during compaction to the
+    // bottommost level.
+    output_compression_opts_.max_dict_bytes = 0;
+    output_compression_opts_.zstd_max_train_bytes = 0;
   }
 
 #ifndef NDEBUG
@@ -276,7 +281,7 @@ Compaction::~Compaction() {
 
 bool Compaction::InputCompressionMatchesOutput() const {
   int base_level = input_vstorage_->base_level();
-  bool matches = (GetCompressionType(immutable_options_, input_vstorage_,
+  bool matches = (GetCompressionType(immutable_cf_options_, input_vstorage_,
                                      mutable_cf_options_, start_level_,
                                      base_level) == output_compression_);
   if (matches) {
@@ -301,8 +306,8 @@ bool Compaction::IsTrivialMove() const {
   }
 
   if (is_manual_compaction_ &&
-      (immutable_options_.compaction_filter != nullptr ||
-       immutable_options_.compaction_filter_factory != nullptr)) {
+      (immutable_cf_options_.compaction_filter != nullptr ||
+       immutable_cf_options_.compaction_filter_factory != nullptr)) {
     // This is a manual compaction and we have a compaction filter that should
     // be executed, we cannot do a trivial move
     return false;
@@ -323,8 +328,6 @@ bool Compaction::IsTrivialMove() const {
 
   // assert inputs_.size() == 1
 
-  std::unique_ptr<SstPartitioner> partitioner = CreateSstPartitioner();
-
   for (const auto& file : inputs_.front().files) {
     std::vector<FileMetaData*> file_grand_parents;
     if (output_level_ + 1 >= number_levels_) {
@@ -337,13 +340,6 @@ bool Compaction::IsTrivialMove() const {
     if (compaction_size > max_compaction_bytes_) {
       return false;
     }
-
-    if (partitioner.get() != nullptr) {
-      if (!partitioner->CanDoTrivialMove(file->smallest.user_key(),
-                                         file->largest.user_key())) {
-        return false;
-      }
-    }
   }
 
   return true;
@@ -375,13 +371,7 @@ bool Compaction::KeyNotExistsBeyondOutputLevel(
         auto* f = files[level_ptrs->at(lvl)];
         if (user_cmp->Compare(user_key, f->largest.user_key()) <= 0) {
           // We've advanced far enough
-          // In the presence of user-defined timestamp, we may need to handle
-          // the case in which f->smallest.user_key() (including ts) has the
-          // same user key, but the ts part is smaller. If so,
-          // Compare(user_key, f->smallest.user_key()) returns -1.
-          // That's why we need CompareWithoutTimestamp().
-          if (user_cmp->CompareWithoutTimestamp(user_key,
-                                                f->smallest.user_key()) >= 0) {
+          if (user_cmp->Compare(user_key, f->smallest.user_key()) >= 0) {
             // Key falls in this file's range, so it may
             // exist beyond output level
             return false;
@@ -510,14 +500,14 @@ uint64_t Compaction::OutputFilePreallocationSize() const {
   }
 
   if (max_output_file_size_ != port::kMaxUint64 &&
-      (immutable_options_.compaction_style == kCompactionStyleLevel ||
+      (immutable_cf_options_.compaction_style == kCompactionStyleLevel ||
        output_level() > 0)) {
     preallocation_size = std::min(max_output_file_size_, preallocation_size);
   }
 
   // Over-estimate slightly so we don't end up just barely crossing
   // the threshold
-  // No point to preallocate more than 1GB.
+  // No point to prellocate more than 1GB.
   return std::min(uint64_t{1073741824},
                   preallocation_size + (preallocation_size / 10));
 }
@@ -527,35 +517,14 @@ std::unique_ptr<CompactionFilter> Compaction::CreateCompactionFilter() const {
     return nullptr;
   }
 
-  if (!cfd_->ioptions()
-           ->compaction_filter_factory->ShouldFilterTableFileCreation(
-               TableFileCreationReason::kCompaction)) {
-    return nullptr;
-  }
-
   CompactionFilter::Context context;
   context.is_full_compaction = is_full_compaction_;
   context.is_manual_compaction = is_manual_compaction_;
   context.column_family_id = cfd_->GetID();
-  context.reason = TableFileCreationReason::kCompaction;
   return cfd_->ioptions()->compaction_filter_factory->CreateCompactionFilter(
       context);
 }
 
-std::unique_ptr<SstPartitioner> Compaction::CreateSstPartitioner() const {
-  if (!immutable_options_.sst_partitioner_factory) {
-    return nullptr;
-  }
-
-  SstPartitioner::Context context;
-  context.is_full_compaction = is_full_compaction_;
-  context.is_manual_compaction = is_manual_compaction_;
-  context.output_level = output_level_;
-  context.smallest_user_key = smallest_user_key_;
-  context.largest_user_key = largest_user_key_;
-  return immutable_options_.sst_partitioner_factory->CreatePartitioner(context);
-}
-
 bool Compaction::IsOutputLevelEmpty() const {
   return inputs_.back().level != output_level_ || inputs_.back().empty();
 }
@@ -564,14 +533,6 @@ bool Compaction::ShouldFormSubcompactions() const {
   if (max_subcompactions_ <= 1 || cfd_ == nullptr) {
     return false;
   }
-
-  // Note: the subcompaction boundary picking logic does not currently guarantee
-  // that all user keys that differ only by timestamp get processed by the same
-  // subcompaction.
-  if (cfd_->user_comparator()->timestamp_size() > 0) {
-    return false;
-  }
-
   if (cfd_->ioptions()->compaction_style == kCompactionStyleLevel) {
     return (start_level_ == 0 || is_manual_compaction_) && output_level_ > 0 &&
            !IsOutputLevelEmpty();
@@ -582,29 +543,6 @@ bool Compaction::ShouldFormSubcompactions() const {
   }
 }
 
-bool Compaction::DoesInputReferenceBlobFiles() const {
-  assert(input_version_);
-
-  const VersionStorageInfo* storage_info = input_version_->storage_info();
-  assert(storage_info);
-
-  if (storage_info->GetBlobFiles().empty()) {
-    return false;
-  }
-
-  for (size_t i = 0; i < inputs_.size(); ++i) {
-    for (const FileMetaData* meta : inputs_[i].files) {
-      assert(meta);
-
-      if (meta->oldest_blob_file_number != kInvalidBlobFileNumber) {
-        return true;
-      }
-    }
-  }
-
-  return false;
-}
-
 uint64_t Compaction::MinInputFileOldestAncesterTime() const {
   uint64_t min_oldest_ancester_time = port::kMaxUint64;
   for (const auto& level_files : inputs_) {

db/compaction/compaction.h

@@ -11,7 +11,6 @@
 #include "db/version_set.h"
 #include "memory/arena.h"
 #include "options/cf_options.h"
-#include "rocksdb/sst_partitioner.h"
 #include "util/autovector.h"
 
 namespace ROCKSDB_NAMESPACE {
@@ -70,9 +69,8 @@ class CompactionFilter;
 class Compaction {
  public:
   Compaction(VersionStorageInfo* input_version,
-             const ImmutableOptions& immutable_options,
+             const ImmutableCFOptions& immutable_cf_options,
              const MutableCFOptions& mutable_cf_options,
-             const MutableDBOptions& mutable_db_options,
              std::vector<CompactionInputFiles> inputs, int output_level,
              uint64_t target_file_size, uint64_t max_compaction_bytes,
              uint32_t output_path_id, CompressionType compression,
@@ -162,7 +160,7 @@ class Compaction {
   CompressionType output_compression() const { return output_compression_; }
 
   // What compression options for output
-  const CompressionOptions& output_compression_opts() const {
+  CompressionOptions output_compression_opts() const {
     return output_compression_opts_;
   }
 
@@ -223,10 +221,10 @@ class Compaction {
   // How many total levels are there?
   int number_levels() const { return number_levels_; }
 
-  // Return the ImmutableOptions that should be used throughout the compaction
+  // Return the ImmutableCFOptions that should be used throughout the compaction
   // procedure
-  const ImmutableOptions* immutable_options() const {
-    return &immutable_options_;
+  const ImmutableCFOptions* immutable_cf_options() const {
+    return &immutable_cf_options_;
   }
 
   // Return the MutableCFOptions that should be used throughout the compaction
@@ -257,20 +255,12 @@ class Compaction {
   // Create a CompactionFilter from compaction_filter_factory
   std::unique_ptr<CompactionFilter> CreateCompactionFilter() const;
 
-  // Create a SstPartitioner from sst_partitioner_factory
-  std::unique_ptr<SstPartitioner> CreateSstPartitioner() const;
-
   // Is the input level corresponding to output_level_ empty?
   bool IsOutputLevelEmpty() const;
 
   // Should this compaction be broken up into smaller ones run in parallel?
   bool ShouldFormSubcompactions() const;
 
-  // Returns true iff at least one input file references a blob file.
-  //
-  // PRE: input version has been set.
-  bool DoesInputReferenceBlobFiles() const;
-
   // test function to validate the functionality of IsBottommostLevel()
   // function -- determines if compaction with inputs and storage is bottommost
   static bool TEST_IsBottommostLevel(
@@ -303,16 +293,6 @@ class Compaction {
 
   uint64_t MinInputFileOldestAncesterTime() const;
 
-  // Called by DBImpl::NotifyOnCompactionCompleted to make sure number of
-  // compaction begin and compaction completion callbacks match.
-  void SetNotifyOnCompactionCompleted() {
-    notify_on_compaction_completion_ = true;
-  }
-
-  bool ShouldNotifyOnCompactionCompleted() const {
-    return notify_on_compaction_completion_;
-  }
-
  private:
   // mark (or clear) all files that are being compacted
   void MarkFilesBeingCompacted(bool mark_as_compacted);
@@ -345,7 +325,7 @@ class Compaction {
   uint64_t max_output_file_size_;
   uint64_t max_compaction_bytes_;
   uint32_t max_subcompactions_;
-  const ImmutableOptions immutable_options_;
+  const ImmutableCFOptions immutable_cf_options_;
   const MutableCFOptions mutable_cf_options_;
   Version* input_version_;
   VersionEdit edit_;
@@ -356,7 +336,7 @@ class Compaction {
   const uint32_t output_path_id_;
   CompressionType output_compression_;
   CompressionOptions output_compression_opts_;
-  // If true, then the compaction can be done by simply deleting input files.
+  // If true, then the comaction can be done by simply deleting input files.
   const bool deletion_compaction_;
 
   // Compaction input files organized by level. Constant after construction
@@ -396,10 +376,6 @@ class Compaction {
 
   // Reason for compaction
   CompactionReason compaction_reason_;
-
-  // Notify on compaction completion only if listener was notified on compaction
-  // begin.
-  bool notify_on_compaction_completion_;
 };
 
 // Return sum of sizes of all files in `files`.

db/compaction/compaction_iteration_stats.h

@@ -34,8 +34,4 @@ struct CompactionIterationStats {
   // Single-Delete diagnostics for exceptional situations
   uint64_t num_single_del_fallthru = 0;
   uint64_t num_single_del_mismatch = 0;
-
-  // Blob related statistics
-  uint64_t num_blobs_read = 0;
-  uint64_t total_blob_bytes_read = 0;
 };

db/compaction/compaction_iterator.cc

@@ -3,13 +3,9 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#include <cinttypes>
+
 #include "db/compaction/compaction_iterator.h"
-
-#include <iterator>
-#include <limits>
-
-#include "db/blob/blob_file_builder.h"
-#include "db/blob/blob_index.h"
 #include "db/snapshot_checker.h"
 #include "port/likely.h"
 #include "rocksdb/listener.h"
@@ -33,31 +29,27 @@
    (snapshot_checker_ == nullptr || LIKELY(IsInEarliestSnapshot(seq))))
 
 namespace ROCKSDB_NAMESPACE {
+
 CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
     SequenceNumber last_sequence, std::vector<SequenceNumber>* snapshots,
     SequenceNumber earliest_write_conflict_snapshot,
     const SnapshotChecker* snapshot_checker, Env* env,
     bool report_detailed_time, bool expect_valid_internal_key,
-    CompactionRangeDelAggregator* range_del_agg,
-    BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
-    const Compaction* compaction, const CompactionFilter* compaction_filter,
+    CompactionRangeDelAggregator* range_del_agg, const Compaction* compaction,
+    const CompactionFilter* compaction_filter,
     const std::atomic<bool>* shutting_down,
     const SequenceNumber preserve_deletes_seqnum,
-    const std::atomic<int>* manual_compaction_paused,
-    const std::atomic<bool>* manual_compaction_canceled,
-    const std::shared_ptr<Logger> info_log,
-    const std::string* full_history_ts_low)
+    const std::atomic<bool>* manual_compaction_paused,
+    const std::shared_ptr<Logger> info_log)
     : CompactionIterator(
           input, cmp, merge_helper, last_sequence, snapshots,
           earliest_write_conflict_snapshot, snapshot_checker, env,
           report_detailed_time, expect_valid_internal_key, range_del_agg,
-          blob_file_builder, allow_data_in_errors,
           std::unique_ptr<CompactionProxy>(
-              compaction ? new RealCompaction(compaction) : nullptr),
+              compaction ? new CompactionProxy(compaction) : nullptr),
           compaction_filter, shutting_down, preserve_deletes_seqnum,
-          manual_compaction_paused, manual_compaction_canceled, info_log,
-          full_history_ts_low) {}
+          manual_compaction_paused, info_log) {}
 
 CompactionIterator::CompactionIterator(
     InternalIterator* input, const Comparator* cmp, MergeHelper* merge_helper,
@@ -66,51 +58,36 @@ CompactionIterator::CompactionIterator(
     const SnapshotChecker* snapshot_checker, Env* env,
     bool report_detailed_time, bool expect_valid_internal_key,
     CompactionRangeDelAggregator* range_del_agg,
-    BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
     std::unique_ptr<CompactionProxy> compaction,
     const CompactionFilter* compaction_filter,
     const std::atomic<bool>* shutting_down,
     const SequenceNumber preserve_deletes_seqnum,
-    const std::atomic<int>* manual_compaction_paused,
-    const std::atomic<bool>* manual_compaction_canceled,
-    const std::shared_ptr<Logger> info_log,
-    const std::string* full_history_ts_low)
-    : input_(input, cmp,
-             !compaction || compaction->DoesInputReferenceBlobFiles()),
+    const std::atomic<bool>* manual_compaction_paused,
+    const std::shared_ptr<Logger> info_log)
+    : input_(input),
       cmp_(cmp),
       merge_helper_(merge_helper),
       snapshots_(snapshots),
       earliest_write_conflict_snapshot_(earliest_write_conflict_snapshot),
       snapshot_checker_(snapshot_checker),
       env_(env),
-      clock_(env_->GetSystemClock().get()),
       report_detailed_time_(report_detailed_time),
       expect_valid_internal_key_(expect_valid_internal_key),
       range_del_agg_(range_del_agg),
-      blob_file_builder_(blob_file_builder),
       compaction_(std::move(compaction)),
       compaction_filter_(compaction_filter),
       shutting_down_(shutting_down),
       manual_compaction_paused_(manual_compaction_paused),
-      manual_compaction_canceled_(manual_compaction_canceled),
       preserve_deletes_seqnum_(preserve_deletes_seqnum),
-      info_log_(info_log),
-      allow_data_in_errors_(allow_data_in_errors),
-      timestamp_size_(cmp_ ? cmp_->timestamp_size() : 0),
-      full_history_ts_low_(full_history_ts_low),
       current_user_key_sequence_(0),
       current_user_key_snapshot_(0),
       merge_out_iter_(merge_helper_),
-      blob_garbage_collection_cutoff_file_number_(
-          ComputeBlobGarbageCollectionCutoffFileNumber(compaction_.get())),
       current_key_committed_(false),
-      cmp_with_history_ts_low_(0),
-      level_(compaction_ == nullptr ? 0 : compaction_->level()) {
+      info_log_(info_log) {
+  assert(compaction_filter_ == nullptr || compaction_ != nullptr);
   assert(snapshots_ != nullptr);
-  bottommost_level_ = compaction_ == nullptr
-                          ? false
-                          : compaction_->bottommost_level() &&
-                                !compaction_->allow_ingest_behind();
+  bottommost_level_ =
+      compaction_ == nullptr ? false : compaction_->bottommost_level();
   if (compaction_ != nullptr) {
     level_ptrs_ = std::vector<size_t>(compaction_->number_levels(), 0);
   }
@@ -131,16 +108,14 @@ CompactionIterator::CompactionIterator(
   for (size_t i = 1; i < snapshots_->size(); ++i) {
     assert(snapshots_->at(i - 1) < snapshots_->at(i));
   }
-  assert(timestamp_size_ == 0 || !full_history_ts_low_ ||
-         timestamp_size_ == full_history_ts_low_->size());
 #endif
-  input_.SetPinnedItersMgr(&pinned_iters_mgr_);
+  input_->SetPinnedItersMgr(&pinned_iters_mgr_);
   TEST_SYNC_POINT_CALLBACK("CompactionIterator:AfterInit", compaction_.get());
 }
 
 CompactionIterator::~CompactionIterator() {
-  // input_ Iterator lifetime is longer than pinned_iters_mgr_ lifetime
-  input_.SetPinnedItersMgr(nullptr);
+  // input_ Iteartor lifetime is longer than pinned_iters_mgr_ lifetime
+  input_->SetPinnedItersMgr(nullptr);
 }
 
 void CompactionIterator::ResetRecordCounts() {
@@ -167,13 +142,14 @@ void CompactionIterator::Next() {
     if (merge_out_iter_.Valid()) {
       key_ = merge_out_iter_.key();
       value_ = merge_out_iter_.value();
-      Status s = ParseInternalKey(key_, &ikey_, allow_data_in_errors_);
+      bool valid_key __attribute__((__unused__));
+      valid_key =  ParseInternalKey(key_, &ikey_);
       // MergeUntil stops when it encounters a corrupt key and does not
       // include them in the result, so we expect the keys here to be valid.
-      assert(s.ok());
-      if (!s.ok()) {
-        ROCKS_LOG_FATAL(info_log_, "Invalid key in compaction. %s",
-                        s.getState());
+      assert(valid_key);
+      if (!valid_key) {
+        ROCKS_LOG_FATAL(info_log_, "Invalid key (%s) in compaction",
+                        key_.ToString(true).c_str());
       }
 
       // Keep current_key_ in sync.
@@ -193,7 +169,7 @@ void CompactionIterator::Next() {
     // Only advance the input iterator if there is no merge output and the
     // iterator is not already at the next record.
     if (!at_next_) {
-      AdvanceInputIter();
+      input_->Next();
     }
     NextFromInput();
   }
@@ -206,188 +182,90 @@ void CompactionIterator::Next() {
   PrepareOutput();
 }
 
-bool CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
+void CompactionIterator::InvokeFilterIfNeeded(bool* need_skip,
                                               Slice* skip_until) {
-  if (!compaction_filter_ ||
-      (ikey_.type != kTypeValue && ikey_.type != kTypeBlobIndex)) {
-    return true;
-  }
-  bool error = false;
-  // If the user has specified a compaction filter and the sequence
-  // number is greater than any external snapshot, then invoke the
-  // filter. If the return value of the compaction filter is true,
-  // replace the entry with a deletion marker.
-  CompactionFilter::Decision filter = CompactionFilter::Decision::kUndetermined;
-  compaction_filter_value_.clear();
-  compaction_filter_skip_until_.Clear();
-  CompactionFilter::ValueType value_type =
-      ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
-                               : CompactionFilter::ValueType::kBlobIndex;
-  // Hack: pass internal key to BlobIndexCompactionFilter since it needs
-  // to get sequence number.
-  assert(compaction_filter_);
-  Slice& filter_key =
-      (ikey_.type == kTypeValue ||
-       !compaction_filter_->IsStackedBlobDbInternalCompactionFilter())
-          ? ikey_.user_key
-          : key_;
-  {
-    StopWatchNano timer(clock_, report_detailed_time_);
-    if (kTypeBlobIndex == ikey_.type) {
-      blob_value_.Reset();
-      filter = compaction_filter_->FilterBlobByKey(
-          level_, filter_key, &compaction_filter_value_,
-          compaction_filter_skip_until_.rep());
-      if (CompactionFilter::Decision::kUndetermined == filter &&
-          !compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
-        // For integrated BlobDB impl, CompactionIterator reads blob value.
-        // For Stacked BlobDB impl, the corresponding CompactionFilter's
-        // FilterV2 method should read the blob value.
-        BlobIndex blob_index;
-        Status s = blob_index.DecodeFrom(value_);
-        if (!s.ok()) {
-          status_ = s;
-          valid_ = false;
-          return false;
-        }
-        if (blob_index.HasTTL() || blob_index.IsInlined()) {
-          status_ = Status::Corruption("Unexpected TTL/inlined blob index");
-          valid_ = false;
-          return false;
-        }
-        if (compaction_ == nullptr) {
-          status_ =
-              Status::Corruption("Unexpected blob index outside of compaction");
-          valid_ = false;
-          return false;
-        }
-        const Version* const version = compaction_->input_version();
-        assert(version);
-
-        uint64_t bytes_read = 0;
-        s = version->GetBlob(ReadOptions(), ikey_.user_key, blob_index,
-                             &blob_value_, &bytes_read);
-        if (!s.ok()) {
-          status_ = s;
-          valid_ = false;
-          return false;
-        }
-
-        ++iter_stats_.num_blobs_read;
-        iter_stats_.total_blob_bytes_read += bytes_read;
-
-        value_type = CompactionFilter::ValueType::kValue;
-      }
-    }
-    if (CompactionFilter::Decision::kUndetermined == filter) {
+  if (compaction_filter_ != nullptr &&
+      (ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex)) {
+    // If the user has specified a compaction filter and the sequence
+    // number is greater than any external snapshot, then invoke the
+    // filter. If the return value of the compaction filter is true,
+    // replace the entry with a deletion marker.
+    CompactionFilter::Decision filter;
+    compaction_filter_value_.clear();
+    compaction_filter_skip_until_.Clear();
+    CompactionFilter::ValueType value_type =
+        ikey_.type == kTypeValue ? CompactionFilter::ValueType::kValue
+                                 : CompactionFilter::ValueType::kBlobIndex;
+    // Hack: pass internal key to BlobIndexCompactionFilter since it needs
+    // to get sequence number.
+    Slice& filter_key = ikey_.type == kTypeValue ? ikey_.user_key : key_;
+    {
+      StopWatchNano timer(env_, report_detailed_time_);
       filter = compaction_filter_->FilterV2(
-          level_, filter_key, value_type,
-          blob_value_.empty() ? value_ : blob_value_, &compaction_filter_value_,
-          compaction_filter_skip_until_.rep());
+          compaction_->level(), filter_key, value_type, value_,
+          &compaction_filter_value_, compaction_filter_skip_until_.rep());
+      iter_stats_.total_filter_time +=
+          env_ != nullptr && report_detailed_time_ ? timer.ElapsedNanos() : 0;
     }
-    iter_stats_.total_filter_time +=
-        env_ != nullptr && report_detailed_time_ ? timer.ElapsedNanos() : 0;
-  }
 
-  if (CompactionFilter::Decision::kUndetermined == filter) {
-    // Should not reach here, since FilterV2 should never return kUndetermined.
-    status_ =
-        Status::NotSupported("FilterV2() should never return kUndetermined");
-    valid_ = false;
-    return false;
-  }
+    if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
+        cmp_->Compare(*compaction_filter_skip_until_.rep(), ikey_.user_key) <=
+            0) {
+      // Can't skip to a key smaller than the current one.
+      // Keep the key as per FilterV2 documentation.
+      filter = CompactionFilter::Decision::kKeep;
+    }
 
-  if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil &&
-      cmp_->Compare(*compaction_filter_skip_until_.rep(), ikey_.user_key) <=
-          0) {
-    // Can't skip to a key smaller than the current one.
-    // Keep the key as per FilterV2 documentation.
-    filter = CompactionFilter::Decision::kKeep;
+    if (filter == CompactionFilter::Decision::kRemove) {
+      // convert the current key to a delete; key_ is pointing into
+      // current_key_ at this point, so updating current_key_ updates key()
+      ikey_.type = kTypeDeletion;
+      current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
+      // no value associated with delete
+      value_.clear();
+      iter_stats_.num_record_drop_user++;
+    } else if (filter == CompactionFilter::Decision::kChangeValue) {
+      value_ = compaction_filter_value_;
+    } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
+      *need_skip = true;
+      compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
+                                                       kValueTypeForSeek);
+      *skip_until = compaction_filter_skip_until_.Encode();
+    }
   }
-
-  if (filter == CompactionFilter::Decision::kRemove) {
-    // convert the current key to a delete; key_ is pointing into
-    // current_key_ at this point, so updating current_key_ updates key()
-    ikey_.type = kTypeDeletion;
-    current_key_.UpdateInternalKey(ikey_.sequence, kTypeDeletion);
-    // no value associated with delete
-    value_.clear();
-    iter_stats_.num_record_drop_user++;
-  } else if (filter == CompactionFilter::Decision::kChangeValue) {
-    if (ikey_.type == kTypeBlobIndex) {
-      // value transfer from blob file to inlined data
-      ikey_.type = kTypeValue;
-      current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-    }
-    value_ = compaction_filter_value_;
-  } else if (filter == CompactionFilter::Decision::kRemoveAndSkipUntil) {
-    *need_skip = true;
-    compaction_filter_skip_until_.ConvertFromUserKey(kMaxSequenceNumber,
-                                                     kValueTypeForSeek);
-    *skip_until = compaction_filter_skip_until_.Encode();
-  } else if (filter == CompactionFilter::Decision::kChangeBlobIndex) {
-    // Only the StackableDB-based BlobDB impl's compaction filter should return
-    // kChangeBlobIndex. Decision about rewriting blob and changing blob index
-    // in the integrated BlobDB impl is made in subsequent call to
-    // PrepareOutput() and its callees.
-    if (!compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
-      status_ = Status::NotSupported(
-          "Only stacked BlobDB's internal compaction filter can return "
-          "kChangeBlobIndex.");
-      valid_ = false;
-      return false;
-    }
-    if (ikey_.type == kTypeValue) {
-      // value transfer from inlined data to blob file
-      ikey_.type = kTypeBlobIndex;
-      current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-    }
-    value_ = compaction_filter_value_;
-  } else if (filter == CompactionFilter::Decision::kIOError) {
-    if (!compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
-      status_ = Status::NotSupported(
-          "CompactionFilter for integrated BlobDB should not return kIOError");
-      valid_ = false;
-      return false;
-    }
-    status_ = Status::IOError("Failed to access blob during compaction filter");
-    error = true;
-  }
-  return !error;
 }
 
 void CompactionIterator::NextFromInput() {
   at_next_ = false;
   valid_ = false;
 
-  while (!valid_ && input_.Valid() && !IsPausingManualCompaction() &&
+  while (!valid_ && input_->Valid() && !IsPausingManualCompaction() &&
          !IsShuttingDown()) {
-    key_ = input_.key();
-    value_ = input_.value();
+    key_ = input_->key();
+    value_ = input_->value();
     iter_stats_.num_input_records++;
 
-    Status pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_);
-    if (!pik_status.ok()) {
-      iter_stats_.num_input_corrupt_records++;
-
+    if (!ParseInternalKey(key_, &ikey_)) {
       // If `expect_valid_internal_key_` is false, return the corrupted key
       // and let the caller decide what to do with it.
+      // TODO(noetzli): We should have a more elegant solution for this.
       if (expect_valid_internal_key_) {
-        status_ = pik_status;
-        return;
+        assert(!"Corrupted internal key not expected.");
+        status_ = Status::Corruption("Corrupted internal key not expected.");
+        break;
       }
       key_ = current_key_.SetInternalKey(key_);
       has_current_user_key_ = false;
       current_user_key_sequence_ = kMaxSequenceNumber;
       current_user_key_snapshot_ = 0;
+      iter_stats_.num_input_corrupt_records++;
       valid_ = true;
       break;
     }
     TEST_SYNC_POINT_CALLBACK("CompactionIterator:ProcessKV", &ikey_);
 
     // Update input statistics
-    if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion ||
-        ikey_.type == kTypeDeletionWithTimestamp) {
+    if (ikey_.type == kTypeDeletion || ikey_.type == kTypeSingleDeletion) {
       iter_stats_.num_input_deletion_records++;
     }
     iter_stats_.total_input_raw_key_bytes += key_.size();
@@ -400,61 +278,25 @@ void CompactionIterator::NextFromInput() {
     // merge_helper_->compaction_filter_skip_until_.
     Slice skip_until;
 
-    bool user_key_equal_without_ts = false;
-    int cmp_ts = 0;
-    if (has_current_user_key_) {
-      user_key_equal_without_ts =
-          cmp_->EqualWithoutTimestamp(ikey_.user_key, current_user_key_);
-      // if timestamp_size_ > 0, then curr_ts_ has been initialized by a
-      // previous key.
-      cmp_ts = timestamp_size_ ? cmp_->CompareTimestamp(
-                                     ExtractTimestampFromUserKey(
-                                         ikey_.user_key, timestamp_size_),
-                                     curr_ts_)
-                               : 0;
-    }
-
     // Check whether the user key changed. After this if statement current_key_
     // is a copy of the current input key (maybe converted to a delete by the
     // compaction filter). ikey_.user_key is pointing to the copy.
-    if (!has_current_user_key_ || !user_key_equal_without_ts || cmp_ts != 0) {
+    if (!has_current_user_key_ ||
+        !cmp_->Equal(ikey_.user_key, current_user_key_)) {
       // First occurrence of this user key
       // Copy key for output
       key_ = current_key_.SetInternalKey(key_, &ikey_);
-
-      // If timestamp_size_ > 0, then copy from ikey_ to curr_ts_ for the use
-      // in next iteration to compare with the timestamp of next key.
-      UpdateTimestampAndCompareWithFullHistoryLow();
-
-      // If
-      // (1) !has_current_user_key_, OR
-      // (2) timestamp is disabled, OR
-      // (3) all history will be preserved, OR
-      // (4) user key (excluding timestamp) is different from previous key, OR
-      // (5) timestamp is NO older than *full_history_ts_low_
-      // then current_user_key_ must be treated as a different user key.
-      // This means, if a user key (excluding ts) is the same as the previous
-      // user key, and its ts is older than *full_history_ts_low_, then we
-      // consider this key for GC, e.g. it may be dropped if certain conditions
-      // match.
-      if (!has_current_user_key_ || !timestamp_size_ || !full_history_ts_low_ ||
-          !user_key_equal_without_ts || cmp_with_history_ts_low_ >= 0) {
-        // Initialize for future comparison for rule (A) and etc.
-        current_user_key_sequence_ = kMaxSequenceNumber;
-        current_user_key_snapshot_ = 0;
-        has_current_user_key_ = true;
-      }
       current_user_key_ = ikey_.user_key;
-
+      has_current_user_key_ = true;
       has_outputted_key_ = false;
-
+      current_user_key_sequence_ = kMaxSequenceNumber;
+      current_user_key_snapshot_ = 0;
       current_key_committed_ = KeyCommitted(ikey_.sequence);
 
       // Apply the compaction filter to the first committed version of the user
       // key.
-      if (current_key_committed_ &&
-          !InvokeFilterIfNeeded(&need_skip, &skip_until)) {
-        break;
+      if (current_key_committed_) {
+        InvokeFilterIfNeeded(&need_skip, &skip_until);
       }
     } else {
       // Update the current key to reflect the new sequence number/type without
@@ -474,9 +316,8 @@ void CompactionIterator::NextFromInput() {
         current_key_committed_ = KeyCommitted(ikey_.sequence);
         // Apply the compaction filter to the first committed version of the
         // user key.
-        if (current_key_committed_ &&
-            !InvokeFilterIfNeeded(&need_skip, &skip_until)) {
-          break;
+        if (current_key_committed_) {
+          InvokeFilterIfNeeded(&need_skip, &skip_until);
         }
       }
     }
@@ -490,7 +331,8 @@ void CompactionIterator::NextFromInput() {
     // If there are no snapshots, then this kv affect visibility at tip.
     // Otherwise, search though all existing snapshots to find the earliest
     // snapshot that is affected by this kv.
-    SequenceNumber last_sequence = current_user_key_sequence_;
+    SequenceNumber last_sequence __attribute__((__unused__));
+    last_sequence = current_user_key_sequence_;
     current_user_key_sequence_ = ikey_.sequence;
     SequenceNumber last_snapshot = current_user_key_snapshot_;
     SequenceNumber prev_snapshot = 0;  // 0 means no previous snapshot
@@ -505,8 +347,8 @@ void CompactionIterator::NextFromInput() {
       // In the previous iteration we encountered a single delete that we could
       // not compact out.  We will keep this Put, but can drop it's data.
       // (See Optimization 3, below.)
-      assert(ikey_.type == kTypeValue || ikey_.type == kTypeBlobIndex);
-      if (ikey_.type != kTypeValue && ikey_.type != kTypeBlobIndex) {
+      assert(ikey_.type == kTypeValue);
+      if (ikey_.type != kTypeValue) {
         ROCKS_LOG_FATAL(info_log_,
                         "Unexpected key type %d for compaction output",
                         ikey_.type);
@@ -519,11 +361,6 @@ void CompactionIterator::NextFromInput() {
                         current_user_key_snapshot_, last_snapshot);
       }
 
-      if (ikey_.type == kTypeBlobIndex) {
-        ikey_.type = kTypeValue;
-        current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-      }
-
       value_.clear();
       valid_ = true;
       clear_and_output_next_key_ = false;
@@ -563,13 +400,11 @@ void CompactionIterator::NextFromInput() {
       // The easiest way to process a SingleDelete during iteration is to peek
       // ahead at the next key.
       ParsedInternalKey next_ikey;
-      AdvanceInputIter();
+      input_->Next();
 
       // Check whether the next key exists, is not corrupt, and is the same key
       // as the single delete.
-      if (input_.Valid() &&
-          ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
-              .ok() &&
+      if (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
           cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
         // Check whether the next key belongs to the same snapshot as the
         // SingleDelete.
@@ -582,7 +417,7 @@ void CompactionIterator::NextFromInput() {
             // to handle the second SingleDelete
 
             // First SingleDelete has been skipped since we already called
-            // input_.Next().
+            // input_->Next().
             ++iter_stats_.num_record_drop_obsolete;
             ++iter_stats_.num_single_del_mismatch;
           } else if (has_outputted_key_ ||
@@ -604,9 +439,9 @@ void CompactionIterator::NextFromInput() {
 
             ++iter_stats_.num_record_drop_hidden;
             ++iter_stats_.num_record_drop_obsolete;
-            // Already called input_.Next() once.  Call it a second time to
+            // Already called input_->Next() once.  Call it a second time to
             // skip past the second key.
-            AdvanceInputIter();
+            input_->Next();
           } else {
             // Found a matching value, but we cannot drop both keys since
             // there is an earlier snapshot and we need to leave behind a record
@@ -673,12 +508,9 @@ void CompactionIterator::NextFromInput() {
                         last_sequence, current_user_key_sequence_);
       }
 
-      ++iter_stats_.num_record_drop_hidden;  // rule (A)
-      AdvanceInputIter();
-    } else if (compaction_ != nullptr &&
-               (ikey_.type == kTypeDeletion ||
-                (ikey_.type == kTypeDeletionWithTimestamp &&
-                 cmp_with_history_ts_low_ < 0)) &&
+      ++iter_stats_.num_record_drop_hidden;  // (A)
+      input_->Next();
+    } else if (compaction_ != nullptr && ikey_.type == kTypeDeletion &&
                IN_EARLIEST_SNAPSHOT(ikey_.sequence) &&
                ikeyNotNeededForIncrementalSnapshot() &&
                compaction_->KeyNotExistsBeyondOutputLevel(ikey_.user_key,
@@ -702,47 +534,30 @@ void CompactionIterator::NextFromInput() {
       // given that:
       // (1) The deletion is earlier than earliest_write_conflict_snapshot, and
       // (2) No value exist earlier than the deletion.
-      //
-      // Note also that a deletion marker of type kTypeDeletionWithTimestamp
-      // will be treated as a different user key unless the timestamp is older
-      // than *full_history_ts_low_.
       ++iter_stats_.num_record_drop_obsolete;
       if (!bottommost_level_) {
         ++iter_stats_.num_optimized_del_drop_obsolete;
       }
-      AdvanceInputIter();
-    } else if ((ikey_.type == kTypeDeletion ||
-                (ikey_.type == kTypeDeletionWithTimestamp &&
-                 cmp_with_history_ts_low_ < 0)) &&
-               bottommost_level_ && ikeyNotNeededForIncrementalSnapshot()) {
+      input_->Next();
+    } else if ((ikey_.type == kTypeDeletion) && bottommost_level_ &&
+               ikeyNotNeededForIncrementalSnapshot()) {
       // Handle the case where we have a delete key at the bottom most level
       // We can skip outputting the key iff there are no subsequent puts for this
       // key
-      assert(!compaction_ || compaction_->KeyNotExistsBeyondOutputLevel(
-                                 ikey_.user_key, &level_ptrs_));
       ParsedInternalKey next_ikey;
-      AdvanceInputIter();
-      // Skip over all versions of this key that happen to occur in the same
-      // snapshot range as the delete.
-      //
-      // Note that a deletion marker of type kTypeDeletionWithTimestamp will be
-      // considered to have a different user key unless the timestamp is older
-      // than *full_history_ts_low_.
-      while (!IsPausingManualCompaction() && !IsShuttingDown() &&
-             input_.Valid() &&
-             (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
-                  .ok()) &&
-             cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key) &&
+      input_->Next();
+      // Skip over all versions of this key that happen to occur in the same snapshot
+      // range as the delete
+      while (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
+             cmp_->Equal(ikey_.user_key, next_ikey.user_key) &&
              (prev_snapshot == 0 ||
               DEFINITELY_NOT_IN_SNAPSHOT(next_ikey.sequence, prev_snapshot))) {
-        AdvanceInputIter();
+        input_->Next();
       }
       // If you find you still need to output a row with this key, we need to output the
       // delete too
-      if (input_.Valid() &&
-          (ParseInternalKey(input_.key(), &next_ikey, allow_data_in_errors_)
-               .ok()) &&
-          cmp_->EqualWithoutTimestamp(ikey_.user_key, next_ikey.user_key)) {
+      if (input_->Valid() && ParseInternalKey(input_->key(), &next_ikey) &&
+          cmp_->Equal(ikey_.user_key, next_ikey.user_key)) {
         valid_ = true;
         at_next_ = true;
       }
@@ -754,15 +569,12 @@ void CompactionIterator::NextFromInput() {
       }
 
       pinned_iters_mgr_.StartPinning();
-      Version* version = compaction_ ? compaction_->input_version() : nullptr;
-
       // We know the merge type entry is not hidden, otherwise we would
       // have hit (A)
       // We encapsulate the merge related state machine in a different
       // object to minimize change to the existing flow.
-      Status s = merge_helper_->MergeUntil(&input_, range_del_agg_,
-                                           prev_snapshot, bottommost_level_,
-                                           allow_data_in_errors_, version);
+      Status s = merge_helper_->MergeUntil(input_, range_del_agg_,
+                                           prev_snapshot, bottommost_level_);
       merge_out_iter_.SeekToFirst();
 
       if (!s.ok() && !s.IsMergeInProgress()) {
@@ -773,13 +585,14 @@ void CompactionIterator::NextFromInput() {
         //       These will be correctly set below.
         key_ = merge_out_iter_.key();
         value_ = merge_out_iter_.value();
-        pik_status = ParseInternalKey(key_, &ikey_, allow_data_in_errors_);
+        bool valid_key __attribute__((__unused__));
+        valid_key = ParseInternalKey(key_, &ikey_);
         // MergeUntil stops when it encounters a corrupt key and does not
         // include them in the result, so we expect the keys here to valid.
-        assert(pik_status.ok());
-        if (!pik_status.ok()) {
-          ROCKS_LOG_FATAL(info_log_, "Invalid key in compaction. %s",
-                          pik_status.getState());
+        assert(valid_key);
+        if (!valid_key) {
+          ROCKS_LOG_FATAL(info_log_, "Invalid key (%s) in compaction",
+                          key_.ToString(true).c_str());
         }
         // Keep current_key_ in sync.
         current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
@@ -805,14 +618,14 @@ void CompactionIterator::NextFromInput() {
       if (should_delete) {
         ++iter_stats_.num_record_drop_hidden;
         ++iter_stats_.num_record_drop_range_del;
-        AdvanceInputIter();
+        input_->Next();
       } else {
         valid_ = true;
       }
     }
 
     if (need_skip) {
-      SkipUntil(skip_until);
+      input_->Seek(skip_until);
     }
   }
 
@@ -825,142 +638,23 @@ void CompactionIterator::NextFromInput() {
   }
 }
 
-bool CompactionIterator::ExtractLargeValueIfNeededImpl() {
-  if (!blob_file_builder_) {
-    return false;
-  }
-
-  blob_index_.clear();
-  const Status s = blob_file_builder_->Add(user_key(), value_, &blob_index_);
-
-  if (!s.ok()) {
-    status_ = s;
-    valid_ = false;
-
-    return false;
-  }
-
-  if (blob_index_.empty()) {
-    return false;
-  }
-
-  value_ = blob_index_;
-
-  return true;
-}
-
-void CompactionIterator::ExtractLargeValueIfNeeded() {
-  assert(ikey_.type == kTypeValue);
-
-  if (!ExtractLargeValueIfNeededImpl()) {
-    return;
-  }
-
-  ikey_.type = kTypeBlobIndex;
-  current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-}
-
-void CompactionIterator::GarbageCollectBlobIfNeeded() {
-  assert(ikey_.type == kTypeBlobIndex);
-
-  if (!compaction_) {
-    return;
-  }
-
-  // GC for integrated BlobDB
-  if (compaction_->enable_blob_garbage_collection()) {
-    BlobIndex blob_index;
-
-    {
-      const Status s = blob_index.DecodeFrom(value_);
-
-      if (!s.ok()) {
-        status_ = s;
-        valid_ = false;
-
-        return;
-      }
-    }
-
-    if (blob_index.IsInlined() || blob_index.HasTTL()) {
-      status_ = Status::Corruption("Unexpected TTL/inlined blob index");
-      valid_ = false;
-
-      return;
-    }
-
-    if (blob_index.file_number() >=
-        blob_garbage_collection_cutoff_file_number_) {
-      return;
-    }
-
-    const Version* const version = compaction_->input_version();
-    assert(version);
-
-    uint64_t bytes_read = 0;
-
-    {
-      const Status s = version->GetBlob(ReadOptions(), user_key(), blob_index,
-                                        &blob_value_, &bytes_read);
-
-      if (!s.ok()) {
-        status_ = s;
-        valid_ = false;
-
-        return;
-      }
-    }
-
-    ++iter_stats_.num_blobs_read;
-    iter_stats_.total_blob_bytes_read += bytes_read;
-
-    value_ = blob_value_;
-
-    if (ExtractLargeValueIfNeededImpl()) {
-      return;
-    }
-
-    ikey_.type = kTypeValue;
-    current_key_.UpdateInternalKey(ikey_.sequence, ikey_.type);
-
-    return;
-  }
-
-  // GC for stacked BlobDB
-  if (compaction_filter_ &&
-      compaction_filter_->IsStackedBlobDbInternalCompactionFilter()) {
-    const auto blob_decision = compaction_filter_->PrepareBlobOutput(
-        user_key(), value_, &compaction_filter_value_);
-
-    if (blob_decision == CompactionFilter::BlobDecision::kCorruption) {
-      status_ =
-          Status::Corruption("Corrupted blob reference encountered during GC");
-      valid_ = false;
-
-      return;
-    }
-
-    if (blob_decision == CompactionFilter::BlobDecision::kIOError) {
-      status_ = Status::IOError("Could not relocate blob during GC");
-      valid_ = false;
-
-      return;
-    }
-
-    if (blob_decision == CompactionFilter::BlobDecision::kChangeValue) {
-      value_ = compaction_filter_value_;
-
-      return;
-    }
-  }
-}
-
 void CompactionIterator::PrepareOutput() {
   if (valid_) {
-    if (ikey_.type == kTypeValue) {
-      ExtractLargeValueIfNeeded();
-    } else if (ikey_.type == kTypeBlobIndex) {
-      GarbageCollectBlobIfNeeded();
+    if (compaction_filter_ && ikey_.type == kTypeBlobIndex) {
+      const auto blob_decision = compaction_filter_->PrepareBlobOutput(
+          user_key(), value_, &compaction_filter_value_);
+
+      if (blob_decision == CompactionFilter::BlobDecision::kCorruption) {
+        status_ = Status::Corruption(
+            "Corrupted blob reference encountered during GC");
+        valid_ = false;
+      } else if (blob_decision == CompactionFilter::BlobDecision::kIOError) {
+        status_ = Status::IOError("Could not relocate blob during GC");
+        valid_ = false;
+      } else if (blob_decision ==
+                 CompactionFilter::BlobDecision::kChangeValue) {
+        value_ = compaction_filter_value_;
+      }
     }
 
     // Zeroing out the sequence number leads to better compression.
@@ -985,18 +679,7 @@ void CompactionIterator::PrepareOutput() {
                         ikey_.type);
       }
       ikey_.sequence = 0;
-      if (!timestamp_size_) {
-        current_key_.UpdateInternalKey(0, ikey_.type);
-      } else if (full_history_ts_low_ && cmp_with_history_ts_low_ < 0) {
-        // We can also zero out timestamp for better compression.
-        // For the same user key (excluding timestamp), the timestamp-based
-        // history can be collapsed to save some space if the timestamp is
-        // older than *full_history_ts_low_.
-        const std::string kTsMin(timestamp_size_, static_cast<char>(0));
-        const Slice ts_slice = kTsMin;
-        ikey_.SetTimestamp(ts_slice);
-        current_key_.UpdateInternalKey(0, ikey_.type, &ts_slice);
-      }
+      current_key_.UpdateInternalKey(0, ikey_.type);
     }
   }
 }
@@ -1088,30 +771,4 @@ bool CompactionIterator::IsInEarliestSnapshot(SequenceNumber sequence) {
   return in_snapshot == SnapshotCheckerResult::kInSnapshot;
 }
 
-uint64_t CompactionIterator::ComputeBlobGarbageCollectionCutoffFileNumber(
-    const CompactionProxy* compaction) {
-  if (!compaction) {
-    return 0;
-  }
-
-  if (!compaction->enable_blob_garbage_collection()) {
-    return 0;
-  }
-
-  Version* const version = compaction->input_version();
-  assert(version);
-
-  const VersionStorageInfo* const storage_info = version->storage_info();
-  assert(storage_info);
-
-  const auto& blob_files = storage_info->GetBlobFiles();
-
-  auto it = blob_files.begin();
-  std::advance(
-      it, compaction->blob_garbage_collection_age_cutoff() * blob_files.size());
-
-  return it != blob_files.end() ? it->first
-                                : std::numeric_limits<uint64_t>::max();
-}
-
 }  // namespace ROCKSDB_NAMESPACE

db/compaction/compaction_iterator.h

@@ -5,7 +5,6 @@
 #pragma once
 
 #include <algorithm>
-#include <cinttypes>
 #include <deque>
 #include <string>
 #include <unordered_set>
@@ -22,135 +21,39 @@
 
 namespace ROCKSDB_NAMESPACE {
 
-class BlobFileBuilder;
-
-// A wrapper of internal iterator whose purpose is to count how
-// many entries there are in the iterator.
-class SequenceIterWrapper : public InternalIterator {
- public:
-  SequenceIterWrapper(InternalIterator* iter, const Comparator* cmp,
-                      bool need_count_entries)
-      : icmp_(cmp, /*named=*/false),
-        inner_iter_(iter),
-        need_count_entries_(need_count_entries) {}
-  bool Valid() const override { return inner_iter_->Valid(); }
-  Status status() const override { return inner_iter_->status(); }
-  void Next() override {
-    num_itered_++;
-    inner_iter_->Next();
-  }
-  void Seek(const Slice& target) override {
-    if (!need_count_entries_) {
-      inner_iter_->Seek(target);
-    } else {
-      // For flush cases, we need to count total number of entries, so we
-      // do Next() rather than Seek().
-      while (inner_iter_->Valid() &&
-             icmp_.Compare(inner_iter_->key(), target) < 0) {
-        Next();
-      }
-    }
-  }
-  Slice key() const override { return inner_iter_->key(); }
-  Slice value() const override { return inner_iter_->value(); }
-
-  // Unused InternalIterator methods
-  void SeekToFirst() override { assert(false); }
-  void Prev() override { assert(false); }
-  void SeekForPrev(const Slice& /* target */) override { assert(false); }
-  void SeekToLast() override { assert(false); }
-
-  uint64_t num_itered() const { return num_itered_; }
-
- private:
-  InternalKeyComparator icmp_;
-  InternalIterator* inner_iter_;  // not owned
-  uint64_t num_itered_ = 0;
-  bool need_count_entries_;
-};
-
 class CompactionIterator {
  public:
   // A wrapper around Compaction. Has a much smaller interface, only what
   // CompactionIterator uses. Tests can override it.
   class CompactionProxy {
    public:
+    explicit CompactionProxy(const Compaction* compaction)
+        : compaction_(compaction) {}
+
     virtual ~CompactionProxy() = default;
-
-    virtual int level() const = 0;
-
-    virtual bool KeyNotExistsBeyondOutputLevel(
-        const Slice& user_key, std::vector<size_t>* level_ptrs) const = 0;
-
-    virtual bool bottommost_level() const = 0;
-
-    virtual int number_levels() const = 0;
-
-    virtual Slice GetLargestUserKey() const = 0;
-
-    virtual bool allow_ingest_behind() const = 0;
-
-    virtual bool preserve_deletes() const = 0;
-
-    virtual bool enable_blob_garbage_collection() const = 0;
-
-    virtual double blob_garbage_collection_age_cutoff() const = 0;
-
-    virtual Version* input_version() const = 0;
-
-    virtual bool DoesInputReferenceBlobFiles() const = 0;
-  };
-
-  class RealCompaction : public CompactionProxy {
-   public:
-    explicit RealCompaction(const Compaction* compaction)
-        : compaction_(compaction) {
-      assert(compaction_);
-      assert(compaction_->immutable_options());
-      assert(compaction_->mutable_cf_options());
+    virtual int level(size_t /*compaction_input_level*/ = 0) const {
+      return compaction_->level();
     }
-
-    int level() const override { return compaction_->level(); }
-
-    bool KeyNotExistsBeyondOutputLevel(
-        const Slice& user_key, std::vector<size_t>* level_ptrs) const override {
+    virtual bool KeyNotExistsBeyondOutputLevel(
+        const Slice& user_key, std::vector<size_t>* level_ptrs) const {
       return compaction_->KeyNotExistsBeyondOutputLevel(user_key, level_ptrs);
     }
-
-    bool bottommost_level() const override {
+    virtual bool bottommost_level() const {
       return compaction_->bottommost_level();
     }
-
-    int number_levels() const override { return compaction_->number_levels(); }
-
-    Slice GetLargestUserKey() const override {
+    virtual int number_levels() const { return compaction_->number_levels(); }
+    virtual Slice GetLargestUserKey() const {
       return compaction_->GetLargestUserKey();
     }
-
-    bool allow_ingest_behind() const override {
-      return compaction_->immutable_options()->allow_ingest_behind;
+    virtual bool allow_ingest_behind() const {
+      return compaction_->immutable_cf_options()->allow_ingest_behind;
+    }
+    virtual bool preserve_deletes() const {
+      return compaction_->immutable_cf_options()->preserve_deletes;
     }
 
-    bool preserve_deletes() const override {
-      return compaction_->immutable_options()->preserve_deletes;
-    }
-
-    bool enable_blob_garbage_collection() const override {
-      return compaction_->mutable_cf_options()->enable_blob_garbage_collection;
-    }
-
-    double blob_garbage_collection_age_cutoff() const override {
-      return compaction_->mutable_cf_options()
-          ->blob_garbage_collection_age_cutoff;
-    }
-
-    Version* input_version() const override {
-      return compaction_->input_version();
-    }
-
-    bool DoesInputReferenceBlobFiles() const override {
-      return compaction_->DoesInputReferenceBlobFiles();
-    }
+   protected:
+    CompactionProxy() = default;
 
    private:
     const Compaction* compaction_;
@@ -163,15 +66,12 @@ class CompactionIterator {
       const SnapshotChecker* snapshot_checker, Env* env,
       bool report_detailed_time, bool expect_valid_internal_key,
       CompactionRangeDelAggregator* range_del_agg,
-      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
       const Compaction* compaction = nullptr,
       const CompactionFilter* compaction_filter = nullptr,
       const std::atomic<bool>* shutting_down = nullptr,
       const SequenceNumber preserve_deletes_seqnum = 0,
-      const std::atomic<int>* manual_compaction_paused = nullptr,
-      const std::atomic<bool>* manual_compaction_canceled = nullptr,
-      const std::shared_ptr<Logger> info_log = nullptr,
-      const std::string* full_history_ts_low = nullptr);
+      const std::atomic<bool>* manual_compaction_paused = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr);
 
   // Constructor with custom CompactionProxy, used for tests.
   CompactionIterator(
@@ -181,15 +81,12 @@ class CompactionIterator {
       const SnapshotChecker* snapshot_checker, Env* env,
       bool report_detailed_time, bool expect_valid_internal_key,
       CompactionRangeDelAggregator* range_del_agg,
-      BlobFileBuilder* blob_file_builder, bool allow_data_in_errors,
       std::unique_ptr<CompactionProxy> compaction,
       const CompactionFilter* compaction_filter = nullptr,
       const std::atomic<bool>* shutting_down = nullptr,
       const SequenceNumber preserve_deletes_seqnum = 0,
-      const std::atomic<int>* manual_compaction_paused = nullptr,
-      const std::atomic<bool>* manual_compaction_canceled = nullptr,
-      const std::shared_ptr<Logger> info_log = nullptr,
-      const std::string* full_history_ts_low = nullptr);
+      const std::atomic<bool>* manual_compaction_paused = nullptr,
+      const std::shared_ptr<Logger> info_log = nullptr);
 
   ~CompactionIterator();
 
@@ -213,39 +110,18 @@ class CompactionIterator {
   bool Valid() const { return valid_; }
   const Slice& user_key() const { return current_user_key_; }
   const CompactionIterationStats& iter_stats() const { return iter_stats_; }
-  uint64_t num_input_entry_scanned() const { return input_.num_itered(); }
 
  private:
   // Processes the input stream to find the next output
   void NextFromInput();
 
-  // Do final preparations before presenting the output to the callee.
+  // Do last preparations before presenting the output to the callee. At this
+  // point this only zeroes out the sequence number if possible for better
+  // compression.
   void PrepareOutput();
 
-  // Passes the output value to the blob file builder (if any), and replaces it
-  // with the corresponding blob reference if it has been actually written to a
-  // blob file (i.e. if it passed the value size check). Returns true if the
-  // value got extracted to a blob file, false otherwise.
-  bool ExtractLargeValueIfNeededImpl();
-
-  // Extracts large values as described above, and updates the internal key's
-  // type to kTypeBlobIndex if the value got extracted. Should only be called
-  // for regular values (kTypeValue).
-  void ExtractLargeValueIfNeeded();
-
-  // Relocates valid blobs residing in the oldest blob files if garbage
-  // collection is enabled. Relocated blobs are written to new blob files or
-  // inlined in the LSM tree depending on the current settings (i.e.
-  // enable_blob_files and min_blob_size). Should only be called for blob
-  // references (kTypeBlobIndex).
-  //
-  // Note: the stacked BlobDB implementation's compaction filter based GC
-  // algorithm is also called from here.
-  void GarbageCollectBlobIfNeeded();
-
   // Invoke compaction filter if needed.
-  // Return true on success, false on failures (e.g.: kIOError).
-  bool InvokeFilterIfNeeded(bool* need_skip, Slice* skip_until);
+  void InvokeFilterIfNeeded(bool* need_skip, Slice* skip_until);
 
   // Given a sequence number, return the sequence number of the
   // earliest snapshot that this sequence number is visible in.
@@ -269,24 +145,7 @@ class CompactionIterator {
 
   bool IsInEarliestSnapshot(SequenceNumber sequence);
 
-  // Extract user-defined timestamp from user key if possible and compare it
-  // with *full_history_ts_low_ if applicable.
-  inline void UpdateTimestampAndCompareWithFullHistoryLow() {
-    if (!timestamp_size_) {
-      return;
-    }
-    Slice ts = ExtractTimestampFromUserKey(ikey_.user_key, timestamp_size_);
-    curr_ts_.assign(ts.data(), ts.size());
-    if (full_history_ts_low_) {
-      cmp_with_history_ts_low_ =
-          cmp_->CompareTimestamp(ts, *full_history_ts_low_);
-    }
-  }
-
-  static uint64_t ComputeBlobGarbageCollectionCutoffFileNumber(
-      const CompactionProxy* compaction);
-
-  SequenceIterWrapper input_;
+  InternalIterator* input_;
   const Comparator* cmp_;
   MergeHelper* merge_helper_;
   const std::vector<SequenceNumber>* snapshots_;
@@ -300,16 +159,13 @@ class CompactionIterator {
   const SequenceNumber earliest_write_conflict_snapshot_;
   const SnapshotChecker* const snapshot_checker_;
   Env* env_;
-  SystemClock* clock_;
   bool report_detailed_time_;
   bool expect_valid_internal_key_;
   CompactionRangeDelAggregator* range_del_agg_;
-  BlobFileBuilder* blob_file_builder_;
   std::unique_ptr<CompactionProxy> compaction_;
   const CompactionFilter* compaction_filter_;
   const std::atomic<bool>* shutting_down_;
-  const std::atomic<int>* manual_compaction_paused_;
-  const std::atomic<bool>* manual_compaction_canceled_;
+  const std::atomic<bool>* manual_compaction_paused_;
   const SequenceNumber preserve_deletes_seqnum_;
   bool bottommost_level_;
   bool valid_ = false;
@@ -317,20 +173,6 @@ class CompactionIterator {
   SequenceNumber earliest_snapshot_;
   SequenceNumber latest_snapshot_;
 
-  std::shared_ptr<Logger> info_log_;
-
-  bool allow_data_in_errors_;
-
-  // Comes from comparator.
-  const size_t timestamp_size_;
-
-  // Lower bound timestamp to retain full history in terms of user-defined
-  // timestamp. If a key's timestamp is older than full_history_ts_low_, then
-  // the key *may* be eligible for garbage collection (GC). The skipping logic
-  // is in `NextFromInput()` and `PrepareOutput()`.
-  // If nullptr, NO GC will be performed and all history will be preserved.
-  const std::string* const full_history_ts_low_;
-
   // State
   //
   // Points to a copy of the current compaction iterator output (current_key_)
@@ -349,13 +191,11 @@ class CompactionIterator {
   // Stores whether ikey_.user_key is valid. If set to false, the user key is
   // not compared against the current key in the underlying iterator.
   bool has_current_user_key_ = false;
-  // If false, the iterator holds a copy of the current compaction iterator
-  // output (or current key in the underlying iterator during NextFromInput()).
-  bool at_next_ = false;
-
+  bool at_next_ = false;  // If false, the iterator
+  // Holds a copy of the current compaction iterator output (or current key in
+  // the underlying iterator during NextFromInput()).
   IterKey current_key_;
   Slice current_user_key_;
-  std::string curr_ts_;
   SequenceNumber current_user_key_sequence_;
   SequenceNumber current_user_key_snapshot_;
 
@@ -370,11 +210,6 @@ class CompactionIterator {
   // PinnedIteratorsManager used to pin input_ Iterator blocks while reading
   // merge operands and then releasing them after consuming them.
   PinnedIteratorsManager pinned_iters_mgr_;
-
-  uint64_t blob_garbage_collection_cutoff_file_number_;
-
-  std::string blob_index_;
-  PinnableSlice blob_value_;
   std::string compaction_filter_value_;
   InternalKey compaction_filter_skip_until_;
   // "level_ptrs" holds indices that remember which file of an associated
@@ -389,15 +224,7 @@ class CompactionIterator {
   // Used to avoid purging uncommitted values. The application can specify
   // uncommitted values by providing a SnapshotChecker object.
   bool current_key_committed_;
-
-  // Saved result of ucmp->CompareTimestamp(current_ts_, *full_history_ts_low_)
-  int cmp_with_history_ts_low_;
-
-  const int level_;
-
-  void AdvanceInputIter() { input_.Next(); }
-
-  void SkipUntil(const Slice& skip_until) { input_.Seek(skip_until); }
+  std::shared_ptr<Logger> info_log_;
 
   bool IsShuttingDown() {
     // This is a best-effort facility, so memory_order_relaxed is sufficient.
@@ -406,10 +233,8 @@ class CompactionIterator {
 
   bool IsPausingManualCompaction() {
     // This is a best-effort facility, so memory_order_relaxed is sufficient.
-    return (manual_compaction_paused_ &&
-            manual_compaction_paused_->load(std::memory_order_relaxed) > 0) ||
-           (manual_compaction_canceled_ &&
-            manual_compaction_canceled_->load(std::memory_order_relaxed));
+    return manual_compaction_paused_ &&
+           manual_compaction_paused_->load(std::memory_order_relaxed);
   }
 };
 }  // namespace ROCKSDB_NAMESPACE