add <sys/sysmacros.h> to avoid warning with glibc 2.25

Summary:
https://github.com/facebook/rocksdb/issues/2152
Closes https://github.com/facebook/rocksdb/pull/2208

Differential Revision: D4945577

Pulled By: lightmark

fbshipit-source-id: 4e679150f2c9443d3be0b6008b26b65fabbda75a

.circleci/cat_ignore_eagain

@@ -1,54 +0,0 @@
-#! /bin/bash
-
-# Work around issue with parallel make output causing random error, as in
-# make[1]: write error: stdout
-# Probably due to a kernel bug:
-#   https://bugs.launchpad.net/ubuntu/+source/linux-signed/+bug/1814393
-# Seems to affect image ubuntu-1604:201903-01 and ubuntu-1604:202004-01
-
-cd "$(dirname $0)"
-
-if [ ! -x cat_ignore_eagain.out ]; then
-    cc -x c -o cat_ignore_eagain.out - << EOF
-#include <unistd.h>
-#include <errno.h>
-#include <stdio.h>
-int main() {
-    int n, m, p;
-    char buf[1024];
-    for (;;) {
-        n = read(STDIN_FILENO, buf, 1024);
-        if (n > 0 && n <= 1024) {
-            for (m = 0; m < n;) {
-                p = write(STDOUT_FILENO, buf + m, n - m);
-                if (p < 0) {
-                    if (errno == EAGAIN) {
-                        // ignore but pause a bit
-                        usleep(100);
-                    } else {
-                        perror("write failed");
-                        return 42;
-                    }
-                } else {
-                    m += p;
-                }
-            }
-        } else if (n < 0) {
-            if (errno == EAGAIN) {
-                // ignore but pause a bit
-                usleep(100);
-            } else {
-                // Some non-ignorable error
-                perror("read failed");
-                return 43;
-            }
-        } else {
-            // EOF
-            return 0;
-        }
-    }
-}
-EOF
-fi
-
-exec ./cat_ignore_eagain.out

.circleci/config.yml

@@ -1,703 +0,0 @@
-version: 2.1
-
-orbs:
-  win: circleci/windows@2.4.0
-  slack: circleci/slack@3.4.2
-
-aliases:
-  - &notify-on-master-failure
-    fail_only: true
-    only_for_branches: master
-
-commands:
-  install-pyenv-on-macos:
-    steps:
-      - run:
-          name: Install pyenv on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install pyenv
-
-  install-cmake-on-macos:
-    steps:
-      - run:
-          name: Install cmake on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install cmake
-
-  increase-max-open-files-on-macos:
-    steps:
-      - run:
-          name: Increase max open files
-          command: |
-            sudo sysctl -w kern.maxfiles=1048576
-            sudo sysctl -w kern.maxfilesperproc=1048576
-            sudo launchctl limit maxfiles 1048576
-
-  pre-steps:
-    parameters:
-      python-version:
-        default: "3.5.9"
-        type: string
-    steps:
-      - checkout
-      - run: pyenv install --skip-existing <<parameters.python-version>>
-      - run: pyenv global <<parameters.python-version>>
-      - run:
-          name: Setup Environment Variables
-          command: |
-            echo "export GTEST_THROW_ON_FAILURE=0" >> $BASH_ENV
-            echo "export GTEST_OUTPUT=\"xml:/tmp/test-results/\"" >> $BASH_ENV
-            echo "export SKIP_FORMAT_BUCK_CHECKS=1" >> $BASH_ENV
-            echo "export PRINT_PARALLEL_OUTPUTS=1" >> $BASH_ENV
-
-  pre-steps-macos:
-      steps:
-        - pre-steps:
-            python-version: "3.6.0"
-
-  post-steps:
-    steps:
-      - slack/status: *notify-on-master-failure
-      - store_test_results: # store test result if there's any
-          path: /tmp/test-results
-      - store_artifacts: # store LOG for debugging if there's any
-          path: LOG
-      - run: # on fail, compress Test Logs for diagnosing the issue
-         name: Compress Test Logs
-         command: tar -cvzf t.tar.gz t
-         when: on_fail
-      - store_artifacts: # on fail, store Test Logs for diagnosing the issue
-          path: t.tar.gz
-          destination: test_logs
-          when: on_fail
-
-  install-clang-10:
-    steps:
-      - run:
-          name: Install Clang 10
-          command: |
-            echo "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "deb-src http://apt.llvm.org/xenial/ llvm-toolchain-xenial-10 main" | sudo tee -a /etc/apt/sources.list
-            echo "APT::Acquire::Retries \"10\";" | sudo tee -a /etc/apt/apt.conf.d/80-retries # llvm.org unreliable
-            sudo apt-get update -y && sudo apt-get install -y clang-10
-
-  install-gflags:
-    steps:
-      - run:
-          name: Install gflags
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-
-  install-gflags-on-macos:
-    steps:
-      - run:
-          name: Install gflags on macos
-          command: |
-            HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
-
-  install-gtest-parallel:
-    steps:
-      - run:
-          name: Install gtest-parallel
-          command: |
-            git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
-            echo 'export PATH=$HOME/gtest-parallel:$PATH' >> $BASH_ENV
-
-  install-compression-libs:
-    steps:
-      - run:
-          name: Install compression libs
-          command: |
-            sudo apt-get update -y && sudo apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-
-executors:
-  windows-2xlarge:
-    machine:
-      image: 'windows-server-2019-vs2019:stable'
-      resource_class: windows.2xlarge
-      shell: bash.exe
-
-jobs:
-  build-macos:
-    macos:
-      xcode: 9.4.1
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && OPT=-DCIRCLECI make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-cmake:
-    macos:
-      xcode: 9.4.1
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-cmake-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run: ulimit -S -n 1048576 && (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j32) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-mem-env:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: MEM_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-encrypted-env:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ENCRYPTED_ENV=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-shared_lib-alt_namespace-status_checked:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=shared OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-release:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-      - post-steps
-
-  build-linux-release-rtti:
-    machine:
-      image: ubuntu-1604:201903-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-      - run: make clean
-      - run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j8 static_lib tools db_bench | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-
-  build-linux-lite:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: LITE=1 make V=1 J=32 -j32 check | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-lite-release:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: if ./db_stress --version; then false; else true; fi # ensure without gflags
-      - install-gflags
-      - run: LITE=1 make V=1 -j8 release | .circleci/cat_ignore_eagain
-      - run: ./db_stress --version # ensure with gflags
-      - post-steps
-
-  build-linux-clang-no_test_run:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: xlarge
-    steps:
-      - checkout # check out the code in the project directory
-      - run: sudo apt-get update -y && sudo apt-get install -y clang libgflags-dev
-      - run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-clang10-asan:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-mini-tsan:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_TSAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out.
-      - post-steps
-
-  build-linux-clang10-ubsan:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out
-      - post-steps
-
-  build-linux-clang10-clang-analyze:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-clang-10
-      - run: sudo apt-get update -y && sudo apt-get install -y clang-tools-10
-      - run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze | .circleci/cat_ignore_eagain # aligned new doesn't work for reason we haven't figured out. For unknown, reason passing "clang++-10" as CLANG_ANALYZER doesn't work, and we need a full path.
-      - post-steps
-
-  build-linux-cmake-no_test_run:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: (mkdir build && cd build && cmake -DWITH_GFLAGS=0 .. && make V=1 -j8) | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-unity:
-    docker: # executor type
-      - image: gcc:latest
-    resource_class: large
-    steps:
-      - checkout # check out the code in the project directory
-      - run: apt-get update -y && apt-get install -y libgflags-dev
-      - run: TEST_TMPDIR=/dev/shm && make V=1 -j8 unity_test | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-gcc-4_8-no_test_run:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-4.8 g++-4.8 libgflags-dev
-      - run: CC=gcc-4.8 CXX=g++-4.8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-8-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-8 g++-8 libgflags-dev
-      - run: CC=gcc-8 CXX=g++-8 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-9-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-9 g++-9 libgflags-dev
-      - run: CC=gcc-9 CXX=g++-9 V=1 SKIP_LINK=1 make -j8 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-linux-gcc-10-cxx20-no_test_run:
-    machine:
-      image: ubuntu-2004:202010-01
-    resource_class: xlarge
-    steps:
-      - pre-steps
-      - run: sudo apt-get update -y && sudo apt-get install gcc-10 g++-10 libgflags-dev
-      - run: CC=gcc-10 CXX=g++-10 V=1 SKIP_LINK=1 ROCKSDB_CXX_STANDARD=c++20 make -j16 all | .circleci/cat_ignore_eagain # Linking broken because libgflags compiled with newer ABI
-      - post-steps
-
-  build-windows:
-    executor: windows-2xlarge
-    parameters:
-      extra_cmake_opt:
-        default: ""
-        type: string
-      vs_year:
-        default: "2019"
-        type: string
-      cmake_generator:
-        default: "Visual Studio 16 2019"
-        type: string
-    environment:
-      THIRDPARTY_HOME: C:/Users/circleci/thirdparty
-      CMAKE_HOME: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64
-      CMAKE_BIN: C:/Users/circleci/thirdparty/cmake-3.16.4-win64-x64/bin/cmake.exe
-      SNAPPY_HOME: C:/Users/circleci/thirdparty/snappy-1.1.7
-      SNAPPY_INCLUDE: C:/Users/circleci/thirdparty/snappy-1.1.7;C:/Users/circleci/thirdparty/snappy-1.1.7/build
-      SNAPPY_LIB_DEBUG: C:/Users/circleci/thirdparty/snappy-1.1.7/build/Debug/snappy.lib
-      VS_YEAR: <<parameters.vs_year>>
-      CMAKE_GENERATOR: <<parameters.cmake_generator>>
-    steps:
-      - checkout
-      - run:
-          name: "Setup VS"
-          command: |
-            if [[ "${VS_YEAR}" == "2019" ]]; then
-              echo "VS2019 already present."
-            elif [[ "${VS_YEAR}" == "2017" ]]; then
-              echo "Installing VS2017..."
-              powershell .circleci/vs2017_install.ps1
-            elif [[ "${VS_YEAR}" == "2015" ]]; then
-              echo "Installing VS2015..."
-              powershell .circleci/vs2015_install.ps1
-            fi
-      - run:
-          name: "Install thirdparty dependencies"
-          command: |
-            mkdir ${THIRDPARTY_HOME}
-            cd ${THIRDPARTY_HOME}
-            echo "Installing CMake..."
-            curl --fail --silent --show-error --output cmake-3.16.4-win64-x64.zip --location https://github.com/Kitware/CMake/releases/download/v3.16.4/cmake-3.16.4-win64-x64.zip
-            unzip -q cmake-3.16.4-win64-x64.zip
-            echo "Building Snappy dependency..."
-            curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
-            unzip -q snappy-1.1.7.zip
-            cd snappy-1.1.7
-            mkdir build
-            cd build
-            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" ..
-            msbuild.exe Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      - run:
-          name: "Build RocksDB"
-          command: |
-            mkdir build
-            cd build
-            ${CMAKE_BIN} -G "${CMAKE_GENERATOR}" -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DJNI=1 << parameters.extra_cmake_opt >> ..
-            cd ..
-            echo "Building with VS version: ${CMAKE_GENERATOR}"
-            msbuild.exe build/rocksdb.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      - run:
-          name: "Test RocksDB"
-          shell: powershell.exe
-          command: |
-            build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test -Concurrency 16
-
-  build-linux-java:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-linux-java-static:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    environment:
-      JAVA_HOME: /usr/lib/jvm/java-1.8.0-openjdk-amd64
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-java:
-    macos:
-      xcode: 9.4.1
-    resource_class: medium
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Shared Library"
-          command: make V=1 J=8 -j8 rocksdbjava | .circleci/cat_ignore_eagain
-      - run:
-          name: "Test RocksDBJava"
-          command: make V=1 J=8 -j8 jtest | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-macos-java-static:
-    macos:
-      xcode: 9.4.1
-    resource_class: medium
-    environment:
-      JAVA_HOME: /Library/Java/JavaVirtualMachines/jdk1.8.0_172.jdk/Contents/Home
-    steps:
-      - increase-max-open-files-on-macos
-      - install-pyenv-on-macos
-      - install-gflags-on-macos
-      - install-cmake-on-macos
-      - pre-steps-macos
-      - run:
-          name: "Set Java Environment"
-          command: |
-            echo "JAVA_HOME=${JAVA_HOME}"
-            echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-            which java && java -version
-            which javac && javac -version
-      - run:
-          name: "Build RocksDBJava Static Library"
-          command: make V=1 J=8 -j8 rocksdbjavastatic | .circleci/cat_ignore_eagain
-      - post-steps
-
-  build-examples:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: large
-    steps:
-      - pre-steps
-      - install-gflags
-      - run:
-          name: "Build examples"
-          command: |
-            OPT=-DTRAVIS V=1 make -j4 static_lib && cd examples && make -j4 | ../.circleci/cat_ignore_eagain
-      - post-steps
-
-  build-cmake-mingw:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - run: sudo apt-get update -y && sudo apt-get install -y mingw-w64
-      - run: sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
-      - run:
-          name: "Build cmake-mingw"
-          command: |
-            sudo apt-get install snapd && sudo snap install cmake --beta --classic
-            export PATH=/snap/bin:$PATH
-            sudo apt-get install -y openjdk-8-jdk
-            export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64
-            export PATH=$JAVA_HOME/bin:$PATH
-            echo "JAVA_HOME=${JAVA_HOME}"
-            which java && java -version
-            which javac && javac -version
-            mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      - post-steps
-
-  build-linux-non-shm:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    parameters:
-      start_test:
-        default: ""
-        type: string
-      end_test:
-        default: ""
-        type: string
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-gtest-parallel
-      - run:
-          name: "Build unit tests"
-          command: |
-            echo "env: $(env)"
-            echo "** done env"
-            ROCKSDBTESTS_START=<<parameters.start_test>> ROCKSDBTESTS_END=<<parameters.end_test>> ROCKSDBTESTS_SUBSET_TESTS_TO_FILE=/tmp/test_list make V=1 -j32 --output-sync=target build_subset_tests
-      - run:
-          name: "Run unit tests in parallel"
-          command: |
-            sed -i 's/[[:space:]]*$//; s/ / \.\//g; s/.*/.\/&/' /tmp/test_list
-            cat /tmp/test_list
-            export TEST_TMPDIR=/tmp/rocksdb_test_tmp
-            gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
-      - post-steps
-
-  build-format-compatible:
-    machine:
-      image: ubuntu-1604:202007-01
-    resource_class: 2xlarge
-    steps:
-      - pre-steps
-      - install-gflags
-      - install-compression-libs
-      - run:
-          name: "test"
-          command: |
-            export TEST_TMPDIR=/dev/shm/rocksdb
-            rm -rf /dev/shm/rocksdb
-            mkdir /dev/shm/rocksdb
-            tools/check_format_compatible.sh
-      - post-steps
-
-workflows:
-  version: 2
-  build-linux:
-    jobs:
-      - build-linux
-  build-linux-mem-env:
-    jobs:
-      - build-linux-mem-env
-  build-linux-encrypted-env:
-    jobs:
-      - build-linux-encrypted-env
-  build-linux-shared_lib-alt_namespace-status_checked:
-    jobs:
-      - build-linux-shared_lib-alt_namespace-status_checked
-  build-linux-lite:
-    jobs:
-      - build-linux-lite
-  build-linux-release:
-    jobs:
-      - build-linux-release
-  build-linux-release-rtti:
-    jobs:
-      - build-linux-release-rtti
-  build-linux-lite-release:
-    jobs:
-      - build-linux-lite-release
-  build-linux-clang10-asan:
-    jobs:
-      - build-linux-clang10-asan
-  build-linux-clang10-mini-tsan:
-    jobs:
-      - build-linux-clang10-mini-tsan
-  build-linux-clang10-ubsan:
-    jobs:
-      - build-linux-clang10-ubsan
-  build-linux-clang10-clang-analyze:
-    jobs:
-      - build-linux-clang10-clang-analyze
-  build-linux-unity:
-    jobs:
-      - build-linux-unity
-  build-windows-vs2019:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2019"
-  build-windows-vs2019-cxx20:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2019-cxx20"
-          extra_cmake_opt: -DCMAKE_CXX_STANDARD=20
-  build-windows-vs2017:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2017"
-          vs_year: "2017"
-          cmake_generator: "Visual Studio 15 Win64"
-  build-windows-vs2015:
-    jobs:
-      - build-windows:
-          name: "build-windows-vs2015"
-          vs_year: "2015"
-          cmake_generator: "Visual Studio 14 Win64"
-  build-java:
-    jobs:
-      - build-linux-java
-      - build-linux-java-static
-      - build-macos-java
-      - build-macos-java-static
-  build-examples:
-    jobs:
-      - build-examples
-  build-linux-non-shm:
-    jobs:
-      - build-linux-non-shm:
-          start_test: ""
-          end_test: "db_options_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "db_options_test" # make sure unique in src.mk
-          end_test: "filename_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "filename_test" # make sure unique in src.mk
-          end_test: "statistics_test" # make sure unique in src.mk
-      - build-linux-non-shm:
-          start_test: "statistics_test" # make sure unique in src.mk
-          end_test: ""
-  build-linux-compilers-no_test_run:
-    jobs:
-      - build-linux-clang-no_test_run
-      - build-linux-cmake-no_test_run
-      - build-linux-gcc-4_8-no_test_run
-      - build-linux-gcc-8-no_test_run
-      - build-linux-gcc-9-no_test_run
-      - build-linux-gcc-10-cxx20-no_test_run
-  build-macos:
-    jobs:
-      - build-macos
-  build-macos-cmake:
-    jobs:
-      - build-macos-cmake
-  build-cmake-mingw:
-    jobs:
-      - build-cmake-mingw
-  nightly:
-    triggers:
-      - schedule:
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-    jobs:
-      - build-format-compatible

.circleci/ubsan_suppression_list.txt

@@ -1,6 +0,0 @@
-# Supress UBSAN warnings related to stl_tree.h, e.g.
-# UndefinedBehaviorSanitizer: undefined-behavior /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43 in 
-# /usr/bin/../lib/gcc/x86_64-linux-gnu/5.4.0/../../../../include/c++/5.4.0/bits/stl_tree.h:1505:43:
-# runtime error: upcast of address 0x000001fa8820 with insufficient space for an object of type
-# 'std::_Rb_tree_node<std::pair<const std::__cxx11::basic_string<char>, rocksdb::(anonymous namespace)::LockHoldingInfo> >'
-src:*bits/stl_tree.h

.circleci/vs2015_install.ps1

@@ -1,24 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://go.microsoft.com/fwlink/?LinkId=691126"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2015 installer failed"
-    exit 1
-}
-$VS_INSTALL_ARGS = @("/Quiet", "/NoRestart")
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2015 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}
-echo "VS 2015 installed."

.circleci/vs2017_install.ps1

@@ -1,35 +0,0 @@
-$VS_DOWNLOAD_LINK = "https://aka.ms/vs/15/release/vs_buildtools.exe"
-$COLLECT_DOWNLOAD_LINK = "https://aka.ms/vscollect.exe"
-$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.14.13",
-                                                     "--add Microsoft.Component.MSBuild",
-                                                     "--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
-                                                     "--add Microsoft.VisualStudio.Component.TextTemplating",
-                                                     "--add Microsoft.VisualStudio.Component.VC.CoreIde",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Core",
-                                                     "--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64",
-                                                     "--add Microsoft.VisualStudio.ComponentGroup.NativeDesktop.Win81")
-
-curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
-if ($LASTEXITCODE -ne 0) {
-    echo "Download of the VS 2017 installer failed"
-    exit 1
-}
-
-$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
-Remove-Item -Path vs_installer.exe -Force
-$exitCode = $process.ExitCode
-if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
-    echo "VS 2017 installer exited with code $exitCode, which should be one of [0, 3010]."
-    curl.exe --retry 3 -kL $COLLECT_DOWNLOAD_LINK --output Collect.exe
-    if ($LASTEXITCODE -ne 0) {
-        echo "Download of the VS Collect tool failed."
-        exit 1
-    }
-    Start-Process "${PWD}\Collect.exe" -NoNewWindow -Wait -PassThru
-    New-Item -Path "C:\w\build-results" -ItemType "directory" -Force
-    Copy-Item -Path "C:\Users\circleci\AppData\Local\Temp\vslogs.zip" -Destination "C:\w\build-results\"
-    exit 1
-}
-echo "VS 2017 installed."

.deprecated_arcconfig

@@ -0,0 +1,17 @@
+{
+  "project_id" : "rocksdb",
+  "conduit_uri" : "https://phabricator.fb.com/api/",
+  "copyright_holder" : "Facebook",
+  "load" : [
+    "arcanist_util"
+  ],
+  "lint.engine" : "FacebookFbcodeLintEngine",
+  "lint.engine.single.linter" : "FbcodeCppLinter",
+  "unit.engine" : "FacebookFbcodeUnitTestEngine",
+  "arcanist_configuration" : "FacebookArcanistConfiguration",
+  "base" : "git:HEAD^, hg:.^",
+  "git.default-relative-commit" : "HEAD^",
+  "git:arc.feature.start.default" : "origin/master",
+  "arc.feature.start.default" : "master",
+  "history.immutable" : false
+}

.github/workflows/sanity_check.yml

@@ -1,41 +0,0 @@
-name: Check buck targets and code format
-on: [push, pull_request]
-jobs:
-  check:
-    name: Check TARGETS file and code format
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout feature branch
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-
-    - name: Fetch from upstream
-      run: |
-        git remote add upstream https://github.com/facebook/rocksdb.git && git fetch upstream
-
-    - name: Where am I
-      run: |
-        echo git status && git status
-        echo "git remote -v" && git remote -v
-        echo git branch && git branch
-
-    - name: Setup Python
-      uses: actions/setup-python@v1
-
-    - name: Install Dependencies
-      run: python -m pip install --upgrade pip
-
-    - name: Install argparse
-      run: pip install argparse
-
-    - name: Download clang-format-diff.py
-      uses: wei/wget@v1
-      with:
-        args: https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py
-
-    - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
-
-    - name: Compare buckify output
-      run: make check-buck-targets

.gitignore

@@ -1,5 +1,4 @@
 make_config.mk
-rocksdb.pc
 
 *.a
 *.arc
@@ -8,7 +7,6 @@ rocksdb.pc
 *.gcda
 *.gcno
 *.o
-*.o.tmp
 *.so
 *.so.*
 *_test
@@ -26,7 +24,6 @@ rocksdb.pc
 *.vcxproj.filters
 *.sln
 *.cmake
-.watchmanconfig
 CMakeCache.txt
 CMakeFiles/
 build/
@@ -34,10 +31,6 @@ build/
 ldb
 manifest_dump
 sst_dump
-blob_dump
-block_cache_trace_analyzer
-db_with_timestamp_basic_test
-tools/block_cache_analyzer/*.pyc
 column_aware_encoding_exp
 util/build_version.cc
 build_tools/VALGRIND_LOGS/
@@ -45,19 +38,12 @@ coverage/COVERAGE_REPORT
 .gdbhistory
 .gdb_history
 package/
+.phutil_module_cache
 unity.a
 tags
-etags
 rocksdb_dump
 rocksdb_undump
 db_test2
-trace_analyzer
-trace_analyzer_test
-block_cache_trace_analyzer
-io_tracer_parser
-.DS_Store
-.vs
-.vscode
 
 java/out
 java/target
@@ -73,7 +59,7 @@ rocksdb.h
 unity.cc
 java/crossbuild/.vagrant
 .vagrant/
-java/**/*.asc
+java/**.asc
 java/javadoc
 
 scan_build_report/
@@ -85,13 +71,3 @@ tp2/
 fbcode/
 fbcode
 buckifier/*.pyc
-buckifier/__pycache__
-
-compile_commands.json
-clang-format-diff.py
-.py3/
-
-fuzz/proto/gen/
-fuzz/crash-*
-
-cmake-build-*

.lgtm.yml

@@ -1,4 +0,0 @@
-extraction:
-  cpp:
-    index:
-      build_command: make static_lib

.travis.yml

@@ -1,246 +1,63 @@
-dist: xenial
+sudo: false
+dist: trusty
 language: cpp
 os:
   - linux
-arch:
-  - arm64
-  - ppc64le
+  - osx
 compiler:
   - clang
   - gcc
+jdk:
+  - oraclejdk7
 cache:
   - ccache
+  - apt
 
 addons:
-  apt:
-    update: true
-    sources:
-      - ubuntu-toolchain-r-test
-    packages:
-      - libgflags-dev
-      - libbz2-dev
-      - liblz4-dev
-      - libsnappy-dev
-      - liblzma-dev  # xv
-      - libzstd-dev
-      - zlib1g-dev
-
+   apt:
+      sources: ['ubuntu-toolchain-r-test', 'llvm-toolchain-precise-3.6']
+      packages: ['clang-3.6' , 'g++-6', 'zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev']
 env:
-  - TEST_GROUP=platform_dependent # 16-18 minutes
-  - TEST_GROUP=1 # 33-35 minutes
-  - TEST_GROUP=2 # 18-20 minutes
-  - TEST_GROUP=3 # 20-22 minutes
-  - TEST_GROUP=4 # 12-14 minutes
+  - TEST_GROUP=platform_dependent1
+  - TEST_GROUP=platform_dependent2
+  - TEST_GROUP=platform_dependent3
+  - TEST_GROUP=1
+  - TEST_GROUP=2
   # Run java tests
-  - JOB_NAME=java_test # 4-11 minutes
+  - JOB_NAME=java_test
   # Build ROCKSDB_LITE
-  - JOB_NAME=lite_build # 3-4 minutes
+  - JOB_NAME=lite_build
   # Build examples
-  - JOB_NAME=examples # 5-7 minutes
-  - JOB_NAME=cmake # 3-5 minutes
-  - JOB_NAME=cmake-gcc8 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9 # 3-5 minutes
-  - JOB_NAME=cmake-gcc9-c++20 # 3-5 minutes
-  - JOB_NAME=cmake-mingw # 3 minutes
-  - JOB_NAME=make-gcc4.8
-  - JOB_NAME=status_checked
+  - JOB_NAME=examples
 
 matrix:
   exclude:
-  - os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-mingw
-  - os : linux
-    arch: arm64
-    env: JOB_NAME=make-gcc4.8
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-mingw
-  - os: linux
-    arch: ppc64le
-    env: JOB_NAME=make-gcc4.8
-  - os: linux
-    compiler: clang
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
+  - os: osx
+    compiler: gcc
+  - os: osx
     env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=1
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
+  - os: osx
     env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=2
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=3
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: TEST_GROUP=4
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/ AND commit_message !~ /java/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=java_test
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=lite_build
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=examples
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc8
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=cmake-gcc9-c++20
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os : linux
-    arch: arm64
-    env: JOB_NAME=status_checked
-  - if: type = pull_request AND commit_message !~ /FULL_CI/
-    os: linux
-    arch: ppc64le
-    env: JOB_NAME=status_checked
-
-install:
-  - if [ "${JOB_NAME}" == cmake-gcc8 ]; then
-      sudo apt-get install -y g++-8 || exit $?;
-      CC=gcc-8 && CXX=g++-8;
-    fi
-  - if [ "${JOB_NAME}" == cmake-gcc9 ] || [ "${JOB_NAME}" == cmake-gcc9-c++20 ]; then
-      sudo apt-get install -y g++-9 || exit $?;
-      CC=gcc-9 && CXX=g++-9;
-    fi
-  - if [ "${JOB_NAME}" == cmake-mingw ]; then
-      sudo apt-get install -y mingw-w64 || exit $?;
-    fi
-  - if [ "${JOB_NAME}" == make-gcc4.8 ]; then
-      sudo apt-get install -y g++-4.8 || exit $?;
-      CC=gcc-4.8 && CXX=g++-4.8;
-    fi
-  - |
-    if [[ "${JOB_NAME}" == cmake* ]]; then
-      sudo apt-get remove -y cmake cmake-data
-      export CMAKE_DEB="cmake-3.14.5-Linux-$(uname -m).deb"
-      export CMAKE_DEB_URL="https://rocksdb-deps.s3-us-west-2.amazonaws.com/cmake/${CMAKE_DEB}"
-      curl --silent --fail --show-error --location --output "${CMAKE_DEB}" "${CMAKE_DEB_URL}" || exit $?
-      sudo dpkg -i "${CMAKE_DEB}" || exit $?
-      which cmake && cmake --version
-    fi
-  - |
-    if [[ "${JOB_NAME}" == java_test || "${JOB_NAME}" == cmake* ]]; then
-      # Ensure JDK 8
-      sudo apt-get install -y openjdk-8-jdk || exit $?
-      export PATH=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin:$PATH
-      export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)
-      echo "JAVA_HOME=${JAVA_HOME}"
-      which java && java -version
-      which javac && javac -version
-    fi
 
 before_script:
-  # Increase the maximum number of open file descriptors, since some tests use
-  # more FDs than the default limit.
-  - ulimit -n 8192
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'clang++' ]]; then CXX=clang++-3.6; fi
+  # test one linux g++ build with g++-6
+  - if [[ "${TRAVIS_OS_NAME}" == 'linux' && "${CXX}" == 'g++' && "${JOB_NAME}" == 'unittests' ]]; then CXX=g++-6; fi
+  # Limit the maximum number of open file descriptors to 8192
+  - ulimit -n 8192 || true
 
 script:
-  - date; ${CXX} --version
-  - if [ `command -v ccache` ]; then ccache -C; fi
-  - case $TEST_GROUP in
-    platform_dependent)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=only make -j4 all_but_some_tests check_some
-      ;;
-    1)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_END=backupable_db_test make -j4 check_some
-      ;;
-    2)
-      OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 make -j4 tools && OPT="-DTRAVIS -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=backupable_db_test ROCKSDBTESTS_END=db_universal_compaction_test make -j4 check_some
-      ;;
-    3)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=db_universal_compaction_test ROCKSDBTESTS_END=table_properties_collector_test make -j4 check_some
-      ;;
-    4)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ROCKSDBTESTS_PLATFORM_DEPENDENT=exclude ROCKSDBTESTS_START=table_properties_collector_test make -j4 check_some
-      ;;
-    esac
-  - case $JOB_NAME in
-    java_test)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make rocksdbjava jtest
-      ;;
-    lite_build)
-      OPT='-DTRAVIS -DROCKSDB_LITE' LIB_MODE=shared V=1 make -j4 all
-      ;;
-    examples)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 make -j4 static_lib && cd examples && make -j4
-      ;;
-    cmake-mingw)
-      sudo update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix;
-      mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-      ;;
-    cmake*)
-      case $JOB_NAME in
-        *-c++20)
-          OPT=-DCMAKE_CXX_STANDARD=20
-          ;;
-      esac
-
-      mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=0 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 .. && make -j4 && cd  .. && rm -rf build && mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_BUILD_TYPE=Release $OPT && make -j4 rocksdb rocksdbjni
-      ;;
-    make-gcc4.8)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 SKIP_LINK=1 make -j4 all && [ "Linking broken because libgflags compiled with newer ABI" ]
-      ;;
-    status_checked)
-      OPT=-DTRAVIS LIB_MODE=shared V=1 ASSERT_STATUS_CHECKED=1 make -j4 check_some
-      ;;
-    esac
+  - ${CXX} --version
+  - if [[ "${TEST_GROUP}" == 'platform_dependent1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=external_sst_file_basic_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=external_sst_file_basic_test ROCKSDBTESTS_END=checkpoint_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == 'platform_dependent3' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=checkpoint_test ROCKSDBTESTS_END=db_block_cache_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == '1' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=comparator_db_test make -j4 check_some; fi
+  - if [[ "${TEST_GROUP}" == '2' ]]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=comparator_db_test make -j4 check_some; fi
+  - if [[ "${JOB_NAME}" == 'java_test' ]]; then OPT=-DTRAVIS V=1 make clean jclean rocksdbjava jtest; fi
+  - if [[ "${JOB_NAME}" == 'lite_build' ]]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib; fi
+  - if [[ "${JOB_NAME}" == 'examples' ]]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
 notifications:
     email:
       - leveldb@fb.com
+    webhooks:
+      - https://buildtimetrend.herokuapp.com/travis

.watchmanconfig

@@ -1,6 +0,0 @@
-{
-  "content_hash_warming": true,
-  "content_hash_max_items": 333333,
-  "hint_num_files_per_dir": 8,
-  "fsevents_latency": 0.05
-}

AUTHORS

@@ -9,4 +9,3 @@ Sanjay Ghemawat <sanjay@google.com>
 # Partial list of contributors:
 Kevin Regan <kevin.d.regan@gmail.com>
 Johan Bilien <jobi@litl.com>
-Matthew Von-Maszewski <https://github.com/matthewvon> (Basho Technologies)

CODE_OF_CONDUCT.md

@@ -1,77 +0,0 @@
-# Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at <opensource-conduct@fb.com>. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
-

CONTRIBUTING.md

@@ -1,8 +1,5 @@
 # Contributing to RocksDB
 
-## Code of Conduct
-The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md)
-
 ## Contributor License Agreement ("CLA")
 
 In order to accept your pull request, we need you to submit a CLA. You

DEFAULT_OPTIONS_HISTORY.md

@@ -1,8 +1,5 @@
 # RocksDB default options change log
 ## Unreleased
-* delayed_write_rate takes the rate given by rate_limiter if not specified.
-
-## 5.2
 * Change the default of delayed slowdown value to 16MB/s and further increase the L0 stop condition to 36 files.
 
 ## 5.0 (11/17/2016)

INSTALL.md

@@ -21,18 +21,15 @@ depend on gflags. You will need to have gflags installed to run `make all`. This
 use binaries compiled by `make all` in production.
 
 * By default the binary we produce is optimized for the platform you're compiling on
-(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
-CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
-`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
-to build a portable binary, add `PORTABLE=1` before your make commands, like this:
-`PORTABLE=1 make static_lib`.
+(-march=native or the equivalent). If you want to build a portable binary, add 'PORTABLE=1' before
+your make commands, like this: `PORTABLE=1 make static_lib`. If you want to build a binary that
+makes use of SSE4, add 'USE_SSE=1' before your make commands, like this: `USE_SSE=1 make static_lib`.
 
 ## Dependencies
 
 * You can link RocksDB with following compression libraries:
   - [zlib](http://www.zlib.net/) - a library for data compression.
   - [bzip2](http://www.bzip.org/) - a library for data compression.
-  - [lz4](https://github.com/lz4/lz4) - a library for extremely fast data compression.
   - [snappy](http://google.github.io/snappy/) - a library for fast
       data compression.
   - [zstandard](http://www.zstd.net) - Fast real-time compression
@@ -43,10 +40,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       command line flags processing. You can compile rocksdb library even
       if you don't have gflags installed.
 
-* `make check` will also check code formatting, which requires [clang-format](https://clang.llvm.org/docs/ClangFormat.html)
-
-* If you wish to build the RocksJava static target, then cmake is required for building Snappy.
-
 ## Supported platforms
 
 * **Linux - Ubuntu**
@@ -58,7 +51,6 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
       `sudo apt-get install libsnappy-dev`.
     * Install zlib. Try: `sudo apt-get install zlib1g-dev`.
     * Install bzip2: `sudo apt-get install libbz2-dev`.
-    * Install lz4: `sudo apt-get install liblz4-dev`.
     * Install zstandard: `sudo apt-get install libzstd-dev`.
 
 * **Linux - CentOS / RHEL**
@@ -71,46 +63,34 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
               git checkout v2.0
               ./configure && make && sudo make install
 
-      **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
-      lib path to `LIBRARY_PATH`. If installed with default settings, the include path will be `/usr/local/include`
-      and the lib path will be `/usr/local/lib`.
+      **Notice**: Once installed, please add the include path for gflags to your CPATH env var and the
+      lib path to LIBRARY_PATH. If installed with default settings, the lib will be /usr/local/lib
+      and the include path will be /usr/local/include.
 
     * Install snappy:
 
-              sudo yum install snappy snappy-devel
+              wget https://github.com/google/snappy/releases/download/1.1.4/snappy-1.1.4.tar.gz
+              tar -xzvf snappy-1.1.4.tar.gz
+              cd snappy-1.1.4
+              ./configure && make && sudo make install
 
     * Install zlib:
 
-              sudo yum install zlib zlib-devel
+              sudo yum install zlib
+              sudo yum install zlib-devel
 
     * Install bzip2:
 
-              sudo yum install bzip2 bzip2-devel
-
-    * Install lz4:
-
-              sudo yum install lz4-devel
-
-    * Install ASAN (optional for debugging):
-
-              sudo yum install libasan
+              sudo yum install bzip2
+              sudo yum install bzip2-devel
 
     * Install zstandard:
-        * With [EPEL](https://fedoraproject.org/wiki/EPEL):
 
-              sudo yum install libzstd-devel
-
-        * With CentOS 8:
-
-              sudo dnf install libzstd-devel
-
-        * From source:
-
-              wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
-              mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
-              tar zxvf zstd-1.1.3.tar.gz
-              cd zstd-1.1.3
-              make && sudo make install
+             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+             mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
+             tar zxvf zstd-1.1.3.tar.gz
+             cd zstd-1.1.3
+             make && sudo make install
 
 * **OS X**:
     * Install latest C++ compiler that supports C++ 11:
@@ -120,94 +100,9 @@ to build a portable binary, add `PORTABLE=1` before your make commands, like thi
             * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
     * run `brew install rocksdb`
 
-* **FreeBSD** (11.01):
-
-    * You can either install RocksDB from the Ports system using `cd /usr/ports/databases/rocksdb && make install`, or you can follow the details below to install dependencies and compile from source code:
-
-    * Install the dependencies for RocksDB:
-
-        export BATCH=YES
-        cd /usr/ports/devel/gmake && make install
-        cd /usr/ports/devel/gflags && make install
-
-        cd /usr/ports/archivers/snappy && make install
-        cd /usr/ports/archivers/bzip2 && make install
-        cd /usr/ports/archivers/liblz4 && make install
-        cd /usr/ports/archivesrs/zstd && make install
-
-        cd /usr/ports/devel/git && make install
-
-
-    * Install the dependencies for RocksJava (optional):
-
-        export BATCH=yes
-        cd /usr/ports/java/openjdk7 && make install
-
-    * Build RocksDB from source:
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-        cd rocksdb
-        export JAVA_HOME=/usr/local/openjdk7
-        gmake rocksdbjava
-
-* **OpenBSD** (6.3/-current):
-
-    * As RocksDB is not available in the ports yet you have to build it on your own:
-
-    * Install the dependencies for RocksDB:
-
-        pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch 
-
-    * Build RocksDB from source:
-
-        cd ~
-        git clone https://github.com/facebook/rocksdb.git
-        cd rocksdb
-        gmake static_lib
-
-    * Build RocksJava from source (optional):
-
-        cd rocksdb
-        export JAVA_HOME=/usr/local/jdk-1.8.0
-        export PATH=$PATH:/usr/local/jdk-1.8.0/bin
-        gmake rocksdbjava
-
 * **iOS**:
   * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
 
 * **Windows**:
   * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
-       * run `vcpkg install rocksdb:x64-windows`
-
-* **AIX 6.1**
-    * Install AIX Toolbox rpms with gcc
-    * Use these environment variables:
-  
-             export PORTABLE=1
-             export CC=gcc
-             export AR="ar -X64"
-             export EXTRA_ARFLAGS=-X64
-             export EXTRA_CFLAGS=-maix64
-             export EXTRA_CXXFLAGS=-maix64
-             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
-             export LIBPATH=/opt/freeware/lib
-             export JAVA_HOME=/usr/java8_64
-             export PATH=/opt/freeware/bin:$PATH
-  
-* **Solaris Sparc**
-    * Install GCC 4.8.2 and higher.
-    * Use these environment variables:
-
-             export CC=gcc
-             export EXTRA_CFLAGS=-m64
-             export EXTRA_CXXFLAGS=-m64
-             export EXTRA_LDFLAGS=-m64
-             export PORTABLE=1
-             export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
-

LANGUAGE-BINDINGS.md

@@ -1,24 +1,13 @@
 This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
 
 * Java - https://github.com/facebook/rocksdb/tree/master/java
-* Python
-    * http://python-rocksdb.readthedocs.io/en/latest/
-    * http://pyrocksdb.readthedocs.org/en/latest/ (unmaintained)
+* Python - http://pyrocksdb.readthedocs.org/en/latest/
 * Perl - https://metacpan.org/pod/RocksDB
 * Node.js - https://npmjs.org/package/rocksdb
 * Go - https://github.com/tecbot/gorocksdb
 * Ruby - http://rubygems.org/gems/rocksdb-ruby
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
-* C#
-    * https://github.com/warrenfalk/rocksdb-sharp
-    * https://github.com/curiosity-ai/rocksdb-sharp
-* Rust
-    * https://github.com/pingcap/rust-rocksdb (used in production fork of https://github.com/spacejam/rust-rocksdb)
-    * https://github.com/spacejam/rust-rocksdb
-    * https://github.com/bh1xuw/rust-rocks
+* C# - https://github.com/warrenfalk/rocksdb-sharp
+* Rust - https://github.com/spacejam/rust-rocksdb
 * D programming language - https://github.com/b1naryth1ef/rocksdb
-* Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
-* Elixir - https://github.com/urbint/rox
-* Nim - https://github.com/status-im/nim-rocksdb
-* Swift and Objective-C (iOS/OSX) - https://github.com/iabudiab/ObjectiveRocks 

PLUGINS.md

@@ -1,3 +0,0 @@
-This is the list of all known third-party plugins for RocksDB. If something is missing, please open a pull request to add it.
-
-* [Dedupfs](https://github.com/ajkr/dedupfs): an example for plugin developers to reference

README.md

@@ -1,20 +1,19 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
 
-[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb)
-[![TravisCI Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
-[![Appveyor Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
-[![PPC64le Build Status](http://140-211-168-68-openstack.osuosl.org:8080/buildStatus/icon?job=rocksdb&style=plastic)](http://140-211-168-68-openstack.osuosl.org:8080/job/rocksdb)
+[![Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
+[![Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
+
 
 RocksDB is developed and maintained by Facebook Database Engineering Team.
-It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com)
+It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
 and Jeff Dean (jeff@google.com)
 
 This code is a library that forms the core building block for a fast
-key-value server, especially suited for storing data on flash drives.
+key value server, especially suited for storing data on flash drives.
 It has a Log-Structured-Merge-Database (LSM) design with flexible tradeoffs
 between Write-Amplification-Factor (WAF), Read-Amplification-Factor (RAF)
 and Space-Amplification-Factor (SAF). It has multi-threaded compactions,
-making it especially suitable for storing multiple terabytes of data in a
+making it specially suitable for storing multiple terabytes of data in a
 single database.
 
 Start with example usage here: https://github.com/facebook/rocksdb/tree/master/examples
@@ -25,8 +24,4 @@ The public interface is in `include/`.  Callers should not include or
 rely on the details of any other header files in this package.  Those
 internal APIs may be changed without warning.
 
-Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/ and https://rocksdb.slack.com/
-
-## License
-
-RocksDB is dual-licensed under both the GPLv2 (found in the COPYING file in the root directory) and Apache 2.0 License (found in the LICENSE.Apache file in the root directory).  You may select, at your option, one of the above-listed licenses.
+Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/

ROCKSDB_LITE.md

@@ -5,7 +5,7 @@ RocksDBLite is a project focused on mobile use cases, which don't need a lot of
 Some examples of the features disabled by ROCKSDB_LITE:
 * compiled-in support for LDB tool
 * No backupable DB
-* No support for replication (which we provide in form of TransactionalIterator)
+* No support for replication (which we provide in form of TrasactionalIterator)
 * No advanced monitoring tools
 * No special-purpose memtables that are highly optimized for specific use cases
 * No Transactions

USERS.md

@@ -5,15 +5,12 @@ At Facebook, we use RocksDB as storage engines in multiple data management servi
 
 1. MyRocks -- https://github.com/MySQLOnRocksDB/mysql-5.6
 2. MongoRocks -- https://github.com/mongodb-partners/mongo-rocks
-3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[1] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
-4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[1]
+3. ZippyDB --  Facebook's distributed key-value store with Paxos-style replication, built on top of RocksDB.[*] https://www.youtube.com/watch?v=DfiN7pG0D0khtt
+4. Laser -- Laser is a high query throughput, low (millisecond) latency, key-value storage service built on top of RocksDB.[*]
 4. Dragon -- a distributed graph query engine. https://code.facebook.com/posts/1737605303120405/dragon-a-distributed-graph-query-engine/
-5. Stylus -- a low-level stream processing framework writtenin C++.[1]
-6. LogDevice -- a distributed data store for logs [2]
+5. Stylus -- a low-level stream processing framework writtenin C++.[*]
 
-[1] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
-
-[2] https://code.facebook.com/posts/357056558062811/logdevice-a-distributed-data-store-for-logs/
+[*] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
 
 ## LinkedIn
 Two different use cases at Linkedin are using RocksDB as a storage engine:
@@ -26,11 +23,8 @@ Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasu
 ## Yahoo
 Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
 
-## Baidu
-[Apache Doris](http://doris.apache.org/master/en/) is a MPP analytical database engine released by Baidu. It [uses RocksDB](http://doris.apache.org/master/en/administrator-guide/operation/tablet-meta-tool.html) to manage its tablet's metadata.
-
 ## CockroachDB
-CockroachDB is an open-source geo-replicated transactional database. They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
+CockroachDB is an open-source geo-replicated transactional database (still in development). They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
 
 ## DNANexus
 DNANexus is using RocksDB to speed up processing of genomics data.
@@ -47,16 +41,12 @@ Tango is using RocksDB as a graph storage to store all users' connection data an
 Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
 Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
 
-## Santander UK/Cloudera Profession Services
+## Santanader UK/Cloudera Profession Services
 Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/
 
 ## Airbnb
 Airbnb is using RocksDB as a storage engine for their personalized search service. You can learn more about it here: https://www.youtube.com/watch?v=ASQ6XMtogMs
 
-## Alluxio
-[Alluxio](https://www.alluxio.io) uses RocksDB to serve and scale file system metadata to beyond 1 Billion files. The detailed design and implementation is described in this engineering blog:
-https://www.alluxio.io/blog/scalable-metadata-service-in-alluxio-storing-billions-of-files/
-
 ## Pinterest
 Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo
 
@@ -70,7 +60,7 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 [VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
 
 ## quasardb
-[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
 quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## Netflix
@@ -86,32 +76,7 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 [Dgraph](https://github.com/dgraph-io/dgraph) is an open-source, scalable, distributed, low latency, high throughput Graph database .They use RocksDB to store state locally on a machine.
 
 ## Uber
-[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
+[Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and and scalable task queue.
 
 ## 360 Pika
-[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been used in many companies.
-
-## LzLabs
-LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
-
-## ProfaneDB
-[ProfaneDB](https://profanedb.gitlab.io/) is a database for Protocol Buffers, and uses RocksDB for storage. It is accessible via gRPC, and the schema is defined using directly `.proto` files.
-
-## IOTA Foundation
- [IOTA Foundation](https://www.iota.org/) is using RocksDB in the [IOTA Reference Implementation (IRI)](https://github.com/iotaledger/iri) to store the local state of the Tangle. The Tangle is the first open-source distributed ledger powering the future of the Internet of Things.
-
-## Avrio Project
- [Avrio Project](http://avrio-project.github.io/avrio.network/) is using RocksDB in [Avrio ](https://github.com/avrio-project/avrio) to store blocks, account balances and data and other blockchain-releated data. Avrio is a multiblockchain decentralized cryptocurrency empowering monetary transactions.
-
-## Crux
-[Crux](https://github.com/juxt/crux) is a document database that uses RocksDB for local [EAV](https://en.wikipedia.org/wiki/Entity%E2%80%93attribute%E2%80%93value_model) index storage to enable point-in-time bitemporal Datalog queries. The "unbundled" architecture uses Kafka to provide horizontal scalability.
-
-## Nebula Graph
-[Nebula Graph](https://github.com/vesoft-inc/nebula) is a distributed, scalable, lightning-fast, open source graph database capable of hosting super large scale graphs with dozens of billions of vertices (nodes) and trillions of edges, with milliseconds of latency.
-
-## YugabyteDB
-[YugabyteDB](https://www.yugabyte.com/) is an open source, high performance, distributed SQL database that uses RocksDB as its storage layer. For more information, please see https://github.com/yugabyte/yugabyte-db/.
-
-## ArangoDB
-[ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its sotrage engine.
-
+[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company

Vagrantfile

@@ -14,11 +14,6 @@ Vagrant.configure("2") do |config|
     box.vm.box = "chef/centos-6.5"
   end
 
-  config.vm.define "centos7" do |box|
-    box.vm.box = "centos/7"
-    box.vm.provision "shell", path: "build_tools/setup_centos7.sh"
-  end
-
   config.vm.define "FreeBSD10" do |box|
     box.vm.guest = :freebsd
     box.vm.box = "robin/freebsd-10"

WINDOWS_PORT.md

@@ -43,9 +43,9 @@ We plan to use this port for our business purposes here at Bing and this provide
 
 * Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`)
 * All posix specific headers were replaced to port/port.h which worked well
-* Replaced `dirent.h` for `port/port_dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
+* Replaced `dirent.h` for `port/dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
 * Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port`
-* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `zu` on posix systems and to `Iu` on windows.
+* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `%z` on posix systems and to Iu on windows.
 * in class member initialization were moved to a __ctors in some cases
 * `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file.
 * `constexpr` for functions was replaced to a template specialization (1 place)

appveyor.yml

@@ -1,77 +1,15 @@
 version: 1.0.{build}
-
-image: Visual Studio 2019
-
-environment:
-  JAVA_HOME: C:\Program Files\Java\jdk1.8.0
-  THIRDPARTY_HOME: $(APPVEYOR_BUILD_FOLDER)\thirdparty
-  SNAPPY_HOME: $(THIRDPARTY_HOME)\snappy-1.1.7
-  SNAPPY_INCLUDE: $(SNAPPY_HOME);$(SNAPPY_HOME)\build
-  SNAPPY_LIB_DEBUG: $(SNAPPY_HOME)\build\Debug\snappy.lib
-  SNAPPY_LIB_RELEASE: $(SNAPPY_HOME)\build\Release\snappy.lib
-  LZ4_HOME: $(THIRDPARTY_HOME)\lz4-1.8.3
-  LZ4_INCLUDE: $(LZ4_HOME)\lib
-  LZ4_LIB_DEBUG: $(LZ4_HOME)\visual\VS2010\bin\x64_Debug\liblz4_static.lib
-  LZ4_LIB_RELEASE: $(LZ4_HOME)\visual\VS2010\bin\x64_Release\liblz4_static.lib
-  ZSTD_HOME: $(THIRDPARTY_HOME)\zstd-1.4.0
-  ZSTD_INCLUDE: $(ZSTD_HOME)\lib;$(ZSTD_HOME)\lib\dictBuilder
-  ZSTD_LIB_DEBUG: $(ZSTD_HOME)\build\VS2010\bin\x64_Debug\libzstd_static.lib
-  ZSTD_LIB_RELEASE: $(ZSTD_HOME)\build\VS2010\bin\x64_Release\libzstd_static.lib
-  matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      CMAKE_GENERATOR: Visual Studio 14 Win64
-      DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio 14.0\Common7\IDE\devenv.com
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      CMAKE_GENERATOR: Visual Studio 15 Win64
-      DEV_ENV: C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\Common7\IDE\devenv.com
-
-install:
-  - md %THIRDPARTY_HOME%
-  - echo "Building Snappy dependency..."
-  - cd %THIRDPARTY_HOME%
-  - curl --fail --silent --show-error --output snappy-1.1.7.zip --location https://github.com/google/snappy/archive/1.1.7.zip
-  - unzip snappy-1.1.7.zip
-  - cd snappy-1.1.7
-  - mkdir build
-  - cd build
-  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT%
-  - msbuild Snappy.sln /p:Configuration=Debug /p:Platform=x64
-  - msbuild Snappy.sln /p:Configuration=Release /p:Platform=x64
-  - echo "Building LZ4 dependency..."
-  - cd %THIRDPARTY_HOME%
-  - curl --fail --silent --show-error --output lz4-1.8.3.zip --location https://github.com/lz4/lz4/archive/v1.8.3.zip
-  - unzip lz4-1.8.3.zip
-  - cd lz4-1.8.3\visual\VS2010
-  - ps: $CMD="$Env:DEV_ENV"; & $CMD lz4.sln /upgrade
-  - msbuild lz4.sln /p:Configuration=Debug /p:Platform=x64
-  - msbuild lz4.sln /p:Configuration=Release /p:Platform=x64
-  - echo "Building ZStd dependency..."
-  - cd %THIRDPARTY_HOME%
-  - curl --fail --silent --show-error --output zstd-1.4.0.zip --location https://github.com/facebook/zstd/archive/v1.4.0.zip
-  - unzip zstd-1.4.0.zip
-  - cd zstd-1.4.0\build\VS2010
-  - ps: $CMD="$Env:DEV_ENV"; & $CMD zstd.sln /upgrade
-  - msbuild zstd.sln /p:Configuration=Debug /p:Platform=x64
-  - msbuild zstd.sln /p:Configuration=Release /p:Platform=x64
-
+image: Visual Studio 2015
 before_build:
-  - md %APPVEYOR_BUILD_FOLDER%\build
-  - cd %APPVEYOR_BUILD_FOLDER%\build
-  - if DEFINED CMAKE_PLATEFORM_NAME (set "PLATEFORM_OPT=-A %CMAKE_PLATEFORM_NAME%")
-  - cmake .. -G "%CMAKE_GENERATOR%" %PLATEFORM_OPT% %CMAKE_OPT% -DCMAKE_BUILD_TYPE=Debug -DOPTDBG=1 -DPORTABLE=1 -DSNAPPY=1 -DLZ4=1 -DZSTD=1 -DXPRESS=1 -DJNI=1 -DWITH_ALL_TESTS=0
-  - cd ..
-
+- md %APPVEYOR_BUILD_FOLDER%\build
+- cd %APPVEYOR_BUILD_FOLDER%\build
+- cmake -G "Visual Studio 14 2015 Win64" -DOPTDBG=1 -DXPRESS=1 ..
+- cd ..
 build:
   project: build\rocksdb.sln
   parallel: true
-  verbosity: normal
-
+  verbosity: minimal
 test:
-
 test_script:
-  - ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,env_basic_test -Concurrency 8
-
-on_failure:
-  - cmd: 7z a build-failed.zip %APPVEYOR_BUILD_FOLDER%\build\ && appveyor PushArtifact build-failed.zip
+- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
 

buckifier/buckify_rocksdb.py

@@ -1,37 +1,16 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-try:
-    from builtins import str
-except ImportError:
-    from __builtin__ import str
 from targets_builder import TARGETSBuilder
-import json
+from optparse import OptionParser
 import os
 import fnmatch
 import sys
+import tempfile
 
 from util import ColorString
-
-# This script generates TARGETS file for Buck.
-# Buck is a build tool specifying dependencies among different build targets.
-# User can pass extra dependencies as a JSON object via command line, and this
-# script can include these dependencies in the generate TARGETS file.
-# Usage:
-# $python3 buckifier/buckify_rocksdb.py
-# (This generates a TARGET file without user-specified dependency for unit
-# tests.)
-# $python3 buckifier/buckify_rocksdb.py \
-#        '{"fake": {
-#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],
-#                      "extra_compiler_flags": ["-DROCKSDB_LITE", "-Os"]
-#                  }
-#         }'
-# (Generated TARGETS file has test_dep and mock1 as dependencies for RocksDB
-# unit tests, and will use the extra_compiler_flags to compile the unit test
-# source.)
+import util
 
 # tests to export as libraries for inclusion in other projects
 _EXPORTED_TEST_LIBS = ["env_basic_test"]
@@ -48,8 +27,8 @@ def parse_src_mk(repo_path):
         if '=' in line:
             current_src = line.split('=')[0].strip()
             src_files[current_src] = []
-        elif '.c' in line:
-            src_path = line.split('\\')[0].strip()
+        elif '.cc' in line:
+            src_path = line.split('.cc')[0].strip() + '.cc'
             src_files[current_src].append(src_path)
     return src_files
 
@@ -57,7 +36,7 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
     cc_files = []
-    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
+    for root, dirnames, filenames in os.walk(repo_path):
         root = root[(len(repo_path) + 1):]
         if "java" in root:
             # Skip java
@@ -69,155 +48,97 @@ def get_cc_files(repo_path):
     return cc_files
 
 
-# Get non_parallel tests from Makefile
-def get_non_parallel_tests(repo_path):
+# Get tests from Makefile
+def get_tests(repo_path):
     Makefile = repo_path + "/Makefile"
 
-    s = set({})
+    # Dictionary TEST_NAME => IS_PARALLEL
+    tests = {}
 
-    found_non_parallel_tests = False
+    found_tests = False
     for line in open(Makefile):
         line = line.strip()
-        if line.startswith("NON_PARALLEL_TEST ="):
-            found_non_parallel_tests = True
-        elif found_non_parallel_tests:
+        if line.startswith("TESTS ="):
+            found_tests = True
+        elif found_tests:
             if line.endswith("\\"):
                 # remove the trailing \
                 line = line[:-1]
                 line = line.strip()
-                s.add(line)
+                tests[line] = False
             else:
-                # we consumed all the non_parallel tests
+                # we consumed all the tests
                 break
 
-    return s
+    found_parallel_tests = False
+    for line in open(Makefile):
+        line = line.strip()
+        if line.startswith("PARALLEL_TEST ="):
+            found_parallel_tests = True
+        elif found_parallel_tests:
+            if line.endswith("\\"):
+                # remove the trailing \
+                line = line[:-1]
+                line = line.strip()
+                tests[line] = True
+            else:
+                # we consumed all the parallel tests
+                break
 
-# Parse extra dependencies passed by user from command line
-def get_dependencies():
-    deps_map = {
-        '': {
-            'extra_deps': [],
-            'extra_compiler_flags': []
-        }
-    }
-    if len(sys.argv) < 2:
-        return deps_map
-
-    def encode_dict(data):
-        rv = {}
-        for k, v in data.items():
-            if isinstance(v, dict):
-                v = encode_dict(v)
-            rv[k] = v
-        return rv
-    extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
-    for target_alias, deps in extra_deps.items():
-        deps_map[target_alias] = deps
-    return deps_map
+    return tests
 
 
 # Prepare TARGETS file for buck
-def generate_targets(repo_path, deps_map):
+def generate_targets(repo_path):
     print(ColorString.info("Generating TARGETS"))
     # parsed src.mk file
     src_mk = parse_src_mk(repo_path)
     # get all .cc files
     cc_files = get_cc_files(repo_path)
-    # get non_parallel tests from Makefile
-    non_parallel_tests = get_non_parallel_tests(repo_path)
+    # get tests from Makefile
+    tests = get_tests(repo_path)
 
-    if src_mk is None or cc_files is None or non_parallel_tests is None:
+    if src_mk is None or cc_files is None or tests is None:
         return False
 
-    extra_argv = ""
-    if len(sys.argv) >= 2:
-        # Heuristically quote and canonicalize whitespace for inclusion
-        # in how the file was generated.
-        extra_argv = " '{0}'".format(" ".join(sys.argv[1].split()))
-
-    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path, extra_argv)
-
+    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
     # rocksdb_lib
     TARGETS.add_library(
         "rocksdb_lib",
         src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] +
         src_mk["TOOL_LIB_SOURCES"])
-    # rocksdb_whole_archive_lib
-    TARGETS.add_library(
-        "rocksdb_whole_archive_lib",
-        src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] +
-        src_mk["TOOL_LIB_SOURCES"],
-        deps=None,
-        headers=None,
-        extra_external_deps="",
-        link_whole=True)
     # rocksdb_test_lib
     TARGETS.add_library(
         "rocksdb_test_lib",
         src_mk.get("MOCK_LIB_SOURCES", []) +
         src_mk.get("TEST_LIB_SOURCES", []) +
-        src_mk.get("EXP_LIB_SOURCES", []) +
-        src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-        extra_external_deps=""" + [
-        ("googletest", None, "gtest"),
-    ]""")
+        src_mk.get("EXP_LIB_SOURCES", []),
+        [":rocksdb_lib"])
     # rocksdb_tools_lib
     TARGETS.add_library(
         "rocksdb_tools_lib",
         src_mk.get("BENCH_LIB_SOURCES", []) +
-        src_mk.get("ANALYZER_LIB_SOURCES", []) +
-        ["test_util/testutil.cc"],
+        ["util/testutil.cc"],
         [":rocksdb_lib"])
-    # rocksdb_stress_lib
-    TARGETS.add_rocksdb_library(
-        "rocksdb_stress_lib",
-        src_mk.get("ANALYZER_LIB_SOURCES", [])
-        + src_mk.get('STRESS_LIB_SOURCES', [])
-        + ["test_util/testutil.cc"])
 
-    print("Extra dependencies:\n{0}".format(json.dumps(deps_map)))
+    # test for every test we found in the Makefile
+    for test in tests:
+        match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+        if len(match_src) == 0:
+            print(ColorString.warning("Cannot find .cc file for %s" % test))
+            continue
+        elif len(match_src) > 1:
+            print(ColorString.warning("Found more than one .cc for %s" % test))
+            print(match_src)
+            continue
 
-    # Dictionary test executable name -> relative source file path
-    test_source_map = {}
-    print(src_mk)
+        assert(len(match_src) == 1)
+        is_parallel = tests[test]
+        TARGETS.register_test(test, match_src[0], is_parallel)
 
-    # c_test.c is added through TARGETS.add_c_test(). If there
-    # are more than one .c test file, we need to extend
-    # TARGETS.add_c_test() to include other C tests too.
-    for test_src in src_mk.get("TEST_MAIN_SOURCES_C", []):
-        if test_src != 'db/c_test.c':
-            print("Don't know how to deal with " + test_src)
-            return False
-    TARGETS.add_c_test()
-
-    for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
-        test = test_src.split('.c')[0].strip().split('/')[-1].strip()
-        test_source_map[test] = test_src
-        print("" + test + " " + test_src)
-
-    for target_alias, deps in deps_map.items():
-        for test, test_src in sorted(test_source_map.items()):
-            if len(test) == 0:
-                print(ColorString.warning("Failed to get test name for %s" % test_src))
-                continue
-
-            test_target_name = \
-                test if not target_alias else test + "_" + target_alias
-            TARGETS.register_test(
-                test_target_name,
-                test_src,
-                test not in non_parallel_tests,
-                json.dumps(deps['extra_deps']),
-                json.dumps(deps['extra_compiler_flags']))
-
-            if test in _EXPORTED_TEST_LIBS:
-                test_library = "%s_lib" % test_target_name
-                TARGETS.add_library(test_library, [test_src], [":rocksdb_test_lib"])
+        if test in _EXPORTED_TEST_LIBS:
+            test_library = "%s_lib" % test
+            TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
     TARGETS.flush_tests()
 
     print(ColorString.info("Generated TARGETS Summary:"))
@@ -236,16 +157,14 @@ def get_rocksdb_path():
 
     return rocksdb_path
 
-
 def exit_with_error(msg):
     print(ColorString.error(msg))
     sys.exit(1)
 
 
 def main():
-    deps_map = get_dependencies()
     # Generate TARGETS file for buck
-    ok = generate_targets(get_rocksdb_path(), deps_map)
+    ok = generate_targets(get_rocksdb_path())
     if not ok:
         exit_with_error("Failed to generate TARGETS files")
 

buckifier/check_buck_targets.sh

@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# If clang_format_diff.py command is not specfied, we assume we are able to
-# access directly without any path.
-
-TGT_DIFF=`git diff TARGETS | head -n 1`
-
-if [ ! -z "$TGT_DIFF" ]
-then
-  echo "TARGETS file has uncommitted changes. Skip this check."
-  exit 0
-fi
-
-echo Backup original TARGETS file.
-
-cp TARGETS TARGETS.bkp
-
-${PYTHON:-python3} buckifier/buckify_rocksdb.py
-
-TGT_DIFF=`git diff TARGETS | head -n 1`
-
-if [ -z "$TGT_DIFF" ]
-then
-  mv TARGETS.bkp TARGETS
-  exit 0
-else
-  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update TARGETS file."
-  echo "Do not manually update TARGETS file."
-  ${PYTHON:-python3} --version
-  mv TARGETS.bkp TARGETS
-  exit 1
-fi

buckifier/rocks_test_runner.sh

@@ -1,6 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
-# shellcheck disable=SC2068
 TEST_TMPDIR="$TEST_DIR" $@ && rm -rf "$TEST_DIR"

buckifier/targets_builder.py

@@ -1,17 +1,12 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-try:
-    from builtins import object
-    from builtins import str
-except ImportError:
-    from __builtin__ import object
-    from __builtin__ import str
 import targets_cfg
+import pprint
 
-def pretty_list(lst, indent=8):
+# TODO(tec): replace this with PrettyPrinter
+def pretty_list(lst, indent=6):
     if lst is None or len(lst) == 0:
         return ""
 
@@ -19,106 +14,52 @@ def pretty_list(lst, indent=8):
         return "\"%s\"" % lst[0]
 
     separator = "\",\n%s\"" % (" " * indent)
-    res = separator.join(sorted(lst))
-    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 4))
+    res = separator.join(lst)
+    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 2))
     return res
 
 
-class TARGETSBuilder(object):
-    def __init__(self, path, extra_argv):
+class TARGETSBuilder:
+    def __init__(self, path):
         self.path = path
-        self.targets_file = open(path, 'wb')
-        header = targets_cfg.rocksdb_target_header_template.format(
-            extra_argv=extra_argv)
-        self.targets_file.write(header.encode("utf-8"))
+        self.targets_file = open(path, 'w')
+        self.targets_file.write(targets_cfg.rocksdb_target_header)
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
-        self.tests_cfg = ""
+        self.tests_cfg = []
 
     def __del__(self):
         self.targets_file.close()
 
-    def add_library(self, name, srcs, deps=None, headers=None,
-                    extra_external_deps="", link_whole=False):
-        headers_attr_prefix = ""
+    def add_library(self, name, srcs, deps=None, headers=None):
         if headers is None:
-            headers_attr_prefix = "auto_"
             headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
-        self.targets_file.write(targets_cfg.library_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
-            headers=headers,
-            deps=pretty_list(deps),
-            extra_external_deps=extra_external_deps,
-            link_whole=link_whole).encode("utf-8"))
-        self.total_lib = self.total_lib + 1
-
-    def add_rocksdb_library(self, name, srcs, headers=None):
-        headers_attr_prefix = ""
-        if headers is None:
-            headers_attr_prefix = "auto_"
-            headers = "AutoHeaders.RECURSIVE_GLOB"
-        else:
-            headers = "[" + pretty_list(headers) + "]"
-        self.targets_file.write(targets_cfg.rocksdb_library_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            headers_attr_prefix=headers_attr_prefix,
-            headers=headers).encode("utf-8"))
+        self.targets_file.write(targets_cfg.library_template % (
+            name,
+            headers,
+            pretty_list(srcs),
+            pretty_list(deps)))
         self.total_lib = self.total_lib + 1
 
     def add_binary(self, name, srcs, deps=None):
-        self.targets_file.write(targets_cfg.binary_template.format(
-            name=name,
-            srcs=pretty_list(srcs),
-            deps=pretty_list(deps)).encode("utf-8"))
+        self.targets_file.write(targets_cfg.binary_template % (
+            name,
+            pretty_list(srcs),
+            pretty_list(deps)))
         self.total_bin = self.total_bin + 1
 
-    def add_c_test(self):
-        self.targets_file.write(b"""
-cpp_binary(
-    name = "c_test_bin",
-    srcs = ["db/c_test.c"],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [":rocksdb_test_lib"],
-) if not is_opt_mode else None
-
-custom_unittest(
-    name = "c_test",
-    command = [
-        native.package_name() + "/buckifier/rocks_test_runner.sh",
-        "$(location :{})".format("c_test_bin"),
-    ],
-    type = "simple",
-) if not is_opt_mode else None
-""")
-
-    def register_test(self,
-                      test_name,
-                      src,
-                      is_parallel,
-                      extra_deps,
-                      extra_compiler_flags):
+    def register_test(self, test_name, src, is_parallel):
         exec_mode = "serial"
         if is_parallel:
             exec_mode = "parallel"
-        self.tests_cfg += targets_cfg.test_cfg_template % (
-            test_name,
-            str(src),
-            str(exec_mode),
-            extra_deps,
-            extra_compiler_flags)
+        self.tests_cfg.append([test_name, str(src), str(exec_mode)])
 
         self.total_test = self.total_test + 1
 
     def flush_tests(self):
-        self.targets_file.write(targets_cfg.unittests_template.format(tests=self.tests_cfg).encode("utf-8"))
-        self.tests_cfg = ""
+        self.targets_file.write(targets_cfg.unittests_template % (
+            pprint.PrettyPrinter().pformat(self.tests_cfg)
+        ))
+
+        self.tests_cfg = []

buckifier/targets_cfg.py

@@ -1,220 +1,112 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-
-rocksdb_target_header_template = \
-    """# This file \100generated by:
-#$ python3 buckifier/buckify_rocksdb.py{extra_argv}
-# --> DO NOT EDIT MANUALLY <--
-# This file is a Facebook-specific integration for buck builds, so can
-# only be validated by Facebook employees.
-#
-load("@fbcode_macros//build_defs:auto_headers.bzl", "AutoHeaders")
-load("@fbcode_macros//build_defs:cpp_library.bzl", "cpp_library")
-load(":defs.bzl", "test_binary")
-
-REPO_PATH = package_name() + "/"
-
-ROCKSDB_COMPILER_FLAGS = [
-    "-fno-builtin-memcmp",
-    # Needed to compile in fbcode
-    "-Wno-expansion-to-defined",
-    # Added missing flags from output of build_detect_platform
-    "-Wnarrowing",
-    "-DROCKSDB_NO_DYNAMIC_EXTENSION",
+rocksdb_target_header = """REPO_PATH = "internal_repo_rocksdb/repo/"
+BUCK_BINS = "buck-out/gen/" + REPO_PATH
+TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
+rocksdb_compiler_flags = [
+  "-msse",
+  "-msse4.2",
+  "-fno-builtin-memcmp",
+  "-DROCKSDB_PLATFORM_POSIX",
+  "-DROCKSDB_LIB_IO_POSIX",
+  "-DROCKSDB_FALLOCATE_PRESENT",
+  "-DROCKSDB_MALLOC_USABLE_SIZE",
+  "-DOS_LINUX",
+  # Flags to enable libs we include
+  "-DSNAPPY",
+  "-DZLIB",
+  "-DBZIP2",
+  "-DLZ4",
+  "-DZSTD",
+  "-DGFLAGS=gflags",
+  "-DNUMA",
+  "-DTBB",
+  # Needed to compile in fbcode
+  "-Wno-expansion-to-defined",
 ]
 
-ROCKSDB_EXTERNAL_DEPS = [
-    ("bzip2", None, "bz2"),
-    ("snappy", None, "snappy"),
-    ("zlib", None, "z"),
-    ("gflags", None, "gflags"),
-    ("lz4", None, "lz4"),
-    ("zstd", None, "zstd"),
+rocksdb_external_deps = [
+  ('bzip2', None, 'bz2'),
+  ('snappy', None, "snappy"),
+  ('zlib', None, 'z'),
+  ('gflags', None, 'gflags'),
+  ('lz4', None, 'lz4'),
+  ('zstd', None),
+  ('tbb', None),
+  ("numa", "2.0.8", "numa"),
+  ("googletest", None, "gtest"),
 ]
 
-ROCKSDB_OS_DEPS = [
-    (
-        "linux",
-        ["third-party//numa:numa", "third-party//liburing:uring", "third-party//tbb:tbb"],
-    ),
-    (
-        "macos",
-        ["third-party//tbb:tbb"],
-    ),
+rocksdb_preprocessor_flags = [
+  # Directories with files for #include
+  "-I" + REPO_PATH + "include/",
+  "-I" + REPO_PATH,
 ]
-
-ROCKSDB_OS_PREPROCESSOR_FLAGS = [
-    (
-        "linux",
-        [
-            "-DOS_LINUX",
-            "-DROCKSDB_FALLOCATE_PRESENT",
-            "-DROCKSDB_MALLOC_USABLE_SIZE",
-            "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
-            "-DROCKSDB_RANGESYNC_PRESENT",
-            "-DROCKSDB_SCHED_GETCPU_PRESENT",
-            "-DROCKSDB_IOURING_PRESENT",
-            "-DHAVE_SSE42",
-            "-DLIBURING",
-            "-DNUMA",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "macos",
-        [
-            "-DOS_MACOSX",
-            "-DROCKSDB_PLATFORM_POSIX",
-            "-DROCKSDB_LIB_IO_POSIX",
-            "-DTBB",
-        ],
-    ),
-    (
-        "windows",
-        [ "-DOS_WIN", "-DWIN32", "-D_MBCS", "-DWIN64", "-DNOMINMAX" ]
-    ),
-]
-
-ROCKSDB_PREPROCESSOR_FLAGS = [
-    "-DROCKSDB_SUPPORT_THREAD_LOCAL",
-
-    # Flags to enable libs we include
-    "-DSNAPPY",
-    "-DZLIB",
-    "-DBZIP2",
-    "-DLZ4",
-    "-DZSTD",
-    "-DZSTD_STATIC_LINKING_ONLY",
-    "-DGFLAGS=gflags",
-
-    # Added missing flags from output of build_detect_platform
-    "-DROCKSDB_BACKTRACE",
-]
-
-# Directories with files for #include
-ROCKSDB_INCLUDE_PATHS = [
-    "",
-    "include",
-]
-
-ROCKSDB_ARCH_PREPROCESSOR_FLAGS = {{
-    "x86_64": [
-        "-DHAVE_PCLMUL",
-    ],
-}}
-
-build_mode = read_config("fbcode", "build_mode")
-
-is_opt_mode = build_mode.startswith("opt")
-
-# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
-# doesn't harm and avoid forgetting to add it.
-ROCKSDB_COMPILER_FLAGS += (["-DNDEBUG"] if is_opt_mode else [])
-
-sanitizer = read_config("fbcode", "sanitizer")
-
-# Do not enable jemalloc if sanitizer presents. RocksDB will further detect
-# whether the binary is linked with jemalloc at runtime.
-ROCKSDB_OS_PREPROCESSOR_FLAGS += ([(
-    "linux",
-    ["-DROCKSDB_JEMALLOC"],
-)] if sanitizer == "" else [])
-
-ROCKSDB_OS_DEPS += ([(
-    "linux",
-    ["third-party//jemalloc:headers"],
-)] if sanitizer == "" else [])
-
-ROCKSDB_LIB_DEPS = [
-    ":rocksdb_lib",
-    ":rocksdb_test_lib",
-] if not is_opt_mode else [":rocksdb_lib"]
 """
 
 
 library_template = """
 cpp_library(
-    name = "{name}",
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS{extra_external_deps},
-    link_whole = {link_whole},
-)
-"""
-
-rocksdb_library_template = """
-cpp_library(
-    name = "{name}",
-    srcs = [{srcs}],
-    {headers_attr_prefix}headers = {headers},
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    os_deps = ROCKSDB_OS_DEPS,
-    os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = ROCKSDB_LIB_DEPS,
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
+    name = "%s",
+    headers = %s,
+    srcs = [%s],
+    deps = [%s],
+    preprocessor_flags = rocksdb_preprocessor_flags,
+    compiler_flags = rocksdb_compiler_flags,
+    external_deps = rocksdb_external_deps,
 )
 """
 
 binary_template = """
 cpp_binary(
-    name = "{name}",
-    srcs = [{srcs}],
-    arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-    compiler_flags = ROCKSDB_COMPILER_FLAGS,
-    preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-    include_paths = ROCKSDB_INCLUDE_PATHS,
-    deps = [{deps}],
-    external_deps = ROCKSDB_EXTERNAL_DEPS,
+  name = "%s",
+  srcs = [%s],
+  deps = [%s],
+  preprocessor_flags = rocksdb_preprocessor_flags,
+  compiler_flags = rocksdb_compiler_flags,
+  external_deps = rocksdb_external_deps,
 )
 """
 
-test_cfg_template = """    [
-        "%s",
-        "%s",
-        "%s",
-        %s,
-        %s,
-    ],
-"""
-
 unittests_template = """
-# [test_name, test_src, test_type, extra_deps, extra_compiler_flags]
-ROCKS_TESTS = [
-{tests}]
+# [test_name, test_src, test_type]
+ROCKS_TESTS = %s
+
 
 # Generate a test rule for each entry in ROCKS_TESTS
-# Do not build the tests in opt mode, since SyncPoint and other test code
-# will not be included.
-[
-    cpp_unittest(
-        name = test_name,
-        srcs = [test_cc],
-        arch_preprocessor_flags = ROCKSDB_ARCH_PREPROCESSOR_FLAGS,
-        os_preprocessor_flags = ROCKSDB_OS_PREPROCESSOR_FLAGS,
-        compiler_flags = ROCKSDB_COMPILER_FLAGS + extra_compiler_flags,
-        preprocessor_flags = ROCKSDB_PREPROCESSOR_FLAGS,
-        include_paths = ROCKSDB_INCLUDE_PATHS,
-        deps = [":rocksdb_test_lib"] + extra_deps,
-        external_deps = ROCKSDB_EXTERNAL_DEPS + [
-            ("googletest", None, "gtest"),
-        ],
+for test_cfg in ROCKS_TESTS:
+    test_name = test_cfg[0]
+    test_cc = test_cfg[1]
+    ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
+    test_bin = test_name + "_bin"
+
+    cpp_binary (
+      name = test_bin,
+      srcs = [test_cc],
+      deps = [":rocksdb_test_lib"],
+      preprocessor_flags = rocksdb_preprocessor_flags,
+      compiler_flags = rocksdb_compiler_flags,
+      external_deps = rocksdb_external_deps,
     )
-    for test_name, test_cc, parallelism, extra_deps, extra_compiler_flags in ROCKS_TESTS
-    if not is_opt_mode
-]
+
+    custom_unittest(
+      name = test_name,
+      type = ttype,
+      deps = [":" + test_bin],
+      command = [TEST_RUNNER, BUCK_BINS + test_bin]
+    )
+
+custom_unittest(
+    name = "make_rocksdbjavastatic",
+    type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
+)
+
+custom_unittest(
+    name = "make_rocksdb_lite_release",
+    type = "simple",
+    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
+)
 """

buckifier/util.py

@@ -1,4 +1,3 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 """
 This module keeps commonly used components.
 """
@@ -6,16 +5,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
-try:
-    from builtins import object
-except ImportError:
-    from __builtin__ import object
 import subprocess
-import sys
 import os
 import time
 
-class ColorString(object):
+class ColorString:
     """ Generate colorful strings on terminal """
     HEADER = '\033[95m'
     BLUE = '\033[94m'
@@ -26,13 +20,7 @@ class ColorString(object):
 
     @staticmethod
     def _make_color_str(text, color):
-        # In Python2, default encoding for unicode string is ASCII
-        if sys.version_info.major <= 2:
-            return "".join(
-                [color, text.encode('utf-8'), ColorString.ENDC])
-        # From Python3, default encoding for unicode string is UTF-8
-        return "".join(
-            [color, text, ColorString.ENDC])
+        return "".join([color, text.encode('utf-8'), ColorString.ENDC])
 
     @staticmethod
     def ok(text):

build_tools/amalgamate.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 # amalgamate.py creates an amalgamation from a unity build.
 # It can be run with either Python 2 or 3.

build_tools/build_detect_platform

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/sh
 #
 # Detects OS we're compiling on and outputs a file specified by the first
 # argument, which in turn gets read while processing Makefile.
@@ -9,7 +9,6 @@
 #   PLATFORM_LDFLAGS            Linker flags
 #   JAVA_LDFLAGS                Linker flags for RocksDBJava
 #   JAVA_STATIC_LDFLAGS         Linker flags for RocksDBJava static build
-#   JAVAC_ARGS                  Arguments for javac
 #   PLATFORM_SHARED_EXT         Extension for shared libraries
 #   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
 #   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
@@ -17,8 +16,6 @@
 #   PLATFORM_CXXFLAGS           C++ compiler flags.  Will contain:
 #   PLATFORM_SHARED_VERSIONED   Set to 'true' if platform supports versioned
 #                               shared libraries, empty otherwise.
-#   FIND			Command for the find utility
-#   WATCH			Command for the watch utility
 #
 # The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
 #
@@ -28,7 +25,6 @@
 #       -DZSTD                      if the ZSTD library is present
 #       -DNUMA                      if the NUMA library is present
 #       -DTBB                       if the TBB library is present
-#       -DMEMKIND                   if the memkind library is present
 #
 # Using gflags in rocksdb:
 # Our project depends on gflags, which requires users to take some extra steps
@@ -45,13 +41,8 @@ if test -z "$OUTPUT"; then
   exit 1
 fi
 
-# we depend on C++11, but should be compatible with newer standards
-if [ "$ROCKSDB_CXX_STANDARD" ]; then
-  PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
-else
-  PLATFORM_CXXFLAGS="-std=c++11"
-fi
-
+# we depend on C++11
+PLATFORM_CXXFLAGS="-std=c++11"
 # we currently depend on POSIX platform
 COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
 
@@ -60,22 +51,7 @@ if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
     # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ -n "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
-      # we need this to build with MySQL. Don't use for other purposes.
-      source "$PWD/build_tools/fbcode_config4.8.1.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_5xx" ]; then
-      source "$PWD/build_tools/fbcode_config.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007" ]; then
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
-    elif [ -n "$ROCKSDB_FBCODE_BUILD_WITH_PLATFORM009" ]; then
-      source "$PWD/build_tools/fbcode_config_platform009.sh"
-    elif [ -z "$USE_CLANG" ]; then
-        # Still use platform007 for gcc by default for build break on
-        # some hosts.
-      source "$PWD/build_tools/fbcode_config_platform007.sh"
-    else
-      source "$PWD/build_tools/fbcode_config_platform009.sh"
-    fi
+    source "$PWD/build_tools/fbcode_config.sh"
 fi
 
 # Delete existing output, if it exists
@@ -83,33 +59,11 @@ rm -f "$OUTPUT"
 touch "$OUTPUT"
 
 if test -z "$CC"; then
-    if [ -x "$(command -v cc)" ]; then
-        CC=cc
-    elif [ -x "$(command -v clang)" ]; then
-        CC=clang
-    else
-        CC=cc
-    fi
+   CC=cc
 fi
 
 if test -z "$CXX"; then
-    if [ -x "$(command -v g++)" ]; then
-        CXX=g++
-    elif [ -x "$(command -v clang++)" ]; then
-        CXX=clang++
-    else
-        CXX=g++
-    fi
-fi
-
-if test -z "$AR"; then
-    if [ -x "$(command -v gcc-ar)" ]; then
-        AR=gcc-ar
-    elif [ -x "$(command -v llvm-ar)" ]; then
-        AR=llvm-ar
-    else
-        AR=ar
-    fi
+    CXX=g++
 fi
 
 # Detect OS
@@ -126,15 +80,7 @@ if test -z "$CLANG_SCAN_BUILD"; then
 fi
 
 if test -z "$CLANG_ANALYZER"; then
-    CLANG_ANALYZER=$(command -v clang++ 2> /dev/null)
-fi
-
-if test -z "$FIND"; then
-    FIND=find
-fi
-
-if test -z "$WATCH"; then
-    WATCH=watch
+    CLANG_ANALYZER=$(which clang++ 2> /dev/null)
 fi
 
 COMMON_FLAGS="$COMMON_FLAGS ${CFLAGS}"
@@ -171,61 +117,22 @@ case "$TARGET_OS" in
         COMMON_FLAGS="$COMMON_FLAGS -DOS_LINUX"
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
-        fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
-        if test $ROCKSDB_USE_IO_URING; then
-            # check for liburing
-            $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o /dev/null 2>/dev/null  <<EOF
-              #include <liburing.h>
-              int main() {
-                struct io_uring ring;
-                io_uring_queue_init(1, &ring, 0);
-                return 0;
-              }
-EOF
-            if [ "$?" = 0 ]; then
-                PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring"
-                COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT"
-            fi
-        fi
-        if test -z "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
-          USE_FOLLY_DISTRIBUTED_MUTEX=1
         fi
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
         ;;
     SunOS)
         PLATFORM=OS_SOLARIS
-        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS -m64"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -static-libstdc++ -static-libgcc -m64"
+        COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_SOLARIS"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/sunos/sunos_specific.cc
         ;;
-    AIX)
-        PLATFORM=OS_AIX
-        CC=gcc
-        COMMON_FLAGS="$COMMON_FLAGS -maix64 -pthread -fno-builtin-memcmp -D_REENTRANT -DOS_AIX -D__STDC_FORMAT_MACROS"
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread -lpthread -lrt -maix64 -static-libstdc++ -static-libgcc"
-        # PORT_FILES=port/aix/aix_specific.cc
-        ;;
     FreeBSD)
         PLATFORM=OS_FREEBSD
-        CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_FREEBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         # PORT_FILES=port/freebsd/freebsd_specific.cc
         ;;
-    GNU/kFreeBSD)
-        PLATFORM=OS_GNU_KFREEBSD
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_GNU_KFREEBSD"
-        if [ -z "$USE_CLANG" ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
-        fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
-        # PORT_FILES=port/gnu_kfreebsd/gnu_kfreebsd_specific.cc
-        ;;
     NetBSD)
         PLATFORM=OS_NETBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
@@ -234,12 +141,9 @@ EOF
         ;;
     OpenBSD)
         PLATFORM=OS_OPENBSD
-	CXX=clang++
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_OPENBSD"
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -pthread"
         # PORT_FILES=port/openbsd/openbsd_specific.cc
-	FIND=gfind
-	WATCH=gnuwatch
         ;;
     DragonFly)
         PLATFORM=OS_DRAGONFLYBSD
@@ -254,8 +158,6 @@ EOF
         COMMON_FLAGS="$COMMON_FLAGS -DCYGWIN"
         if [ -z "$USE_CLANG" ]; then
             COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
@@ -275,25 +177,20 @@ esac
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
 JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
 JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
-JAVAC_ARGS="-source 7"
 
 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Cross-compiling; do not try any compilation tests.
     # Also don't need any compilation tests if compiling on fbcode
-    if [ "$FBCODE_BUILD" = "true" ]; then
-      # Enable backtrace on fbcode since the necessary libraries are present
-      COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-    fi
     true
 else
     if ! test $ROCKSDB_DISABLE_FALLOCATE; then
         # Test whether fallocate is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <fcntl.h>
           #include <linux/falloc.h>
           int main() {
       int fd = open("/dev/null", 0);
-      fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
+      fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
           }
 EOF
         if [ "$?" = 0 ]; then
@@ -301,301 +198,184 @@ EOF
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_SNAPPY; then
-        # Test whether Snappy library is installed
-        # http://code.google.com/p/snappy/
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <snappy.h>
-          int main() {}
+    # Test whether Snappy library is installed
+    # http://code.google.com/p/snappy/
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <snappy.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DSNAPPY"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lsnappy"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lsnappy"
     fi
 
-    if ! test $ROCKSDB_DISABLE_GFLAGS; then
-        # Test whether gflags library is installed
-        # http://gflags.github.io/gflags/
-        # check if the namespace is gflags
-        if $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-          #include <gflags/gflags.h>
-          using namespace GFLAGS_NAMESPACE;
-          int main() {}
+    # Test whether gflags library is installed
+    # http://gflags.github.io/gflags/
+    # check if the namespace is gflags
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+      #include <gflags/gflags.h>
+      using namespace gflags;
+      int main() {}
 EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=gflags"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+    else
+      # check if namespace is google
+      $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
+        #include <gflags/gflags.h>
+        using namespace google;
+        int main() {}
+EOF
+      if [ "$?" = 0 ]; then
+          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
           PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is gflags
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace gflags;
-            int main() {}
-EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is google
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace google;
-            int main() {}
-EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        fi
+      fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_ZLIB; then
-        # Test whether zlib library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <zlib.h>
-          int main() {}
+    # Test whether zlib library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zlib.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZLIB"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lz"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lz"
     fi
 
-    if ! test $ROCKSDB_DISABLE_BZIP; then
-        # Test whether bzip library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <bzlib.h>
-          int main() {}
+    # Test whether bzip library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <bzlib.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DBZIP2"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbz2"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lbz2"
     fi
 
-    if ! test $ROCKSDB_DISABLE_LZ4; then
-        # Test whether lz4 library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <lz4.h>
-          #include <lz4hc.h>
-          int main() {}
+    # Test whether lz4 library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <lz4.h>
+      #include <lz4hc.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DLZ4"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -llz4"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -llz4"
     fi
 
-    if ! test $ROCKSDB_DISABLE_ZSTD; then
-        # Test whether zstd library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <zstd.h>
-          int main() {}
+    # Test whether zstd library is installed
+    $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <zstd.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DZSTD"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lzstd"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lzstd"
     fi
 
-    if ! test $ROCKSDB_DISABLE_NUMA; then
-        # Test whether numa is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
-          #include <numa.h>
-          #include <numaif.h>
-          int main() {}
+    # Test whether numa is available
+    $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
+      #include <numa.h>
+      #include <numaif.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DNUMA"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lnuma"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -lnuma"
     fi
 
-    if ! test $ROCKSDB_DISABLE_TBB; then
-        # Test whether tbb is available
-        $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
-          #include <tbb/tbb.h>
-          int main() {}
+    # Test whether tbb is available
+    $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
+      #include <tbb/tbb.h>
+      int main() {}
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DTBB"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DTBB"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltbb"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ltbb"
     fi
 
-    if ! test $ROCKSDB_DISABLE_JEMALLOC; then
-        # Test whether jemalloc is available
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -ljemalloc \
-          2>/dev/null; then
-            # This will enable some preprocessor identifiers in the Makefile
-            JEMALLOC=1
-            # JEMALLOC can be enabled either using the flag (like here) or by
-            # providing direct link to the jemalloc library
-            WITH_JEMALLOC_FLAG=1
-            # check for JEMALLOC installed with HomeBrew
-            if [ "$PLATFORM" == "OS_MACOSX" ]; then
-                if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then
-                    JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ')
-                    JEMALLOC_INCLUDE="-I/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/include"
-                    JEMALLOC_LIB="/usr/local/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
-                    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $JEMALLOC_LIB"
-                    JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS $JEMALLOC_LIB"
-                fi
-            fi
-        fi
-    fi
-    if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
+    # Test whether jemalloc is available
+    if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
+      2>/dev/null; then
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ljemalloc"
+        JAVA_LDFLAGS="$JAVA_LDFLAGS -ljemalloc"
+        JEMALLOC=1
+    else
         # jemalloc is not available. Let's try tcmalloc
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null \
+        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
           -ltcmalloc 2>/dev/null; then
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
             JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then
-        # Test whether malloc_usable_size is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <malloc.h>
-          int main() {
-            size_t res = malloc_usable_size(0);
-            (void)res;
-            return 0;
-          }
+    # Test whether malloc_usable_size is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <malloc.h>
+      int main() {
+        size_t res = malloc_usable_size(0);
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_MALLOC_USABLE_SIZE"
     fi
 
-    if ! test $ROCKSDB_DISABLE_MEMKIND; then
-        # Test whether memkind library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -lmemkind -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <memkind.h>
-          int main() {
-            memkind_malloc(MEMKIND_DAX_KMEM, 1024);
-            return 0;
-          }
+    # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <pthread.h>
+      int main() {
+        int x = PTHREAD_MUTEX_ADAPTIVE_NP;
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DMEMKIND"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lmemkind"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lmemkind"
-        fi
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
     fi
 
-    if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
-        # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <pthread.h>
-          int main() {
-            int x = PTHREAD_MUTEX_ADAPTIVE_NP;
-            (void)x;
-            return 0;
-          }
+    # Test whether backtrace is available
+    $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      #include <execinfo.h>>
+      int main() {
+        void* frames[1];
+        backtrace_symbols(frames, backtrace(frames, 1));
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_PTHREAD_ADAPTIVE_MUTEX"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_BACKTRACE; then
-        # Test whether backtrace is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+    if [ "$?" = 0 ]; then
+        COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
+    else
+        # Test whether execinfo library is installed
+        $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <execinfo.h>
           int main() {
             void* frames[1];
             backtrace_symbols(frames, backtrace(frames, 1));
-            return 0;
           }
 EOF
         if [ "$?" = 0 ]; then
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-        else
-            # Test whether execinfo library is installed
-            $CXX $PLATFORM_CXXFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
-              #include <execinfo.h>
-              int main() {
-                void* frames[1];
-                backtrace_symbols(frames, backtrace(frames, 1));
-              }
-EOF
-            if [ "$?" = 0 ]; then
-                COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-                PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
-                JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
-            fi
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lexecinfo"
+            JAVA_LDFLAGS="$JAVA_LDFLAGS -lexecinfo"
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_PG; then
-        # Test if -pg is supported
-        $CXX $PLATFORM_CXXFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          int main() {
-            return 0;
-          }
+    # Test if -pg is supported
+    $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
+      int main() {
+        return 0;
+      }
 EOF
-        if [ "$?" = 0 ]; then
-            PROFILING_FLAGS=-pg
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_SYNC_FILE_RANGE; then
-        # Test whether sync_file_range is supported for compatibility with an old glibc
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <fcntl.h>
-          int main() {
-            int fd = open("/dev/null", 0);
-            sync_file_range(fd, 0, 1024, SYNC_FILE_RANGE_WRITE);
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_RANGESYNC_PRESENT"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then
-        # Test whether sched_getcpu is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <sched.h>
-          int main() {
-            int cpuid = sched_getcpu();
-            (void)cpuid;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_AUXV_GETAUXVAL; then
-        # Test whether getauxval is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <sys/auxv.h>
-          int main() {
-            uint64_t auxv = getauxval(AT_HWCAP);
-            (void)auxv;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_AUXV_GETAUXVAL_PRESENT"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then
-        # Test whether c++17 aligned-new is supported
-        $CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o /dev/null 2>/dev/null <<EOF
-            struct alignas(1024) t {int a;};
-            int main() {}
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
-        fi
+    if [ "$?" = 0 ]; then
+        PROFILING_FLAGS=-pg
     fi
 fi
 
@@ -603,7 +383,7 @@ fi
 # -Wshorten-64-to-32 breaks compilation on FreeBSD i386
 if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
   # Test whether -Wshorten-64-to-32 is available
-  $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
+  $CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
     int main() {}
 EOF
   if [ "$?" = 0 ]; then
@@ -615,11 +395,11 @@ fi
 
 if test "$USE_HDFS"; then
   if test -z "$JAVA_HOME"; then
-    echo "JAVA_HOME has to be set for HDFS usage." >&2
+    echo "JAVA_HOME has to be set for HDFS usage."
     exit 1
   fi
-  HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS -I$HADOOP_HOME/include"
-  HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64 -L$HADOOP_HOME/lib/native"
+  HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS"
+  HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64"
   HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
   HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
   COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
@@ -627,202 +407,24 @@ if test "$USE_HDFS"; then
   JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
 fi
 
-if test "0$PORTABLE" -eq 0; then
+if [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
+  # Intel SSE instructions breaks compilation on FreeBSD i386
+  unset USE_SSE
+fi
+
+if test "$USE_SSE"; then
+  # if Intel SSE instruction set is supported, set USE_SSE=1
+  COMMON_FLAGS="$COMMON_FLAGS -msse -msse4.2 "
+elif test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
     POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
   elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
     COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep -e^arm -e^aarch64`"; then
-    # TODO: Handle this with approprite options.
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^aarch64`"; then
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif [ "$TARGET_OS" == "IOS" ]; then
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif [ "$TARGET_OS" == "AIX" ] || [ "$TARGET_OS" == "SunOS" ]; then
-    # TODO: Not sure why we don't use -march=native on these OSes
-    if test "$USE_SSE"; then
-      TRY_SSE_ETC="1"
-    fi
   else
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
   fi
-else
-  # PORTABLE=1
-  if test "$USE_SSE"; then
-    TRY_SSE_ETC="1"
-  fi
-
-  if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
-    # For portability compile for macOS 10.12 (2016) or newer
-    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.12"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_SHARED_LDFLAGS="$PLATFORM_SHARED_LDFLAGS -mmacosx-version-min=10.12"
-    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.12"
-    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.12"
-    JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-  fi
-fi
-
-if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
-  # check for GNU libc on ppc64
-  $CXX -x c++ - -o /dev/null 2>/dev/null <<EOF
-    #include <stdio.h>
-    #include <stdlib.h>
-    #include <gnu/libc-version.h>
-
-    int main(int argc, char *argv[]) {
-      printf("GNU libc version: %s\n", gnu_get_libc_version());
-      return 0;
-    }
-EOF
-  if [ "$?" != 0 ]; then
-      PPC_LIBC_IS_GNU=0
-  fi
-fi
-
-if test "$TRY_SSE_ETC"; then
-  # The USE_SSE flag now means "attempt to compile with widely-available
-  # Intel architecture extensions utilized by specific optimizations in the
-  # source code." It's a qualifier on PORTABLE=1 that means "mostly portable."
-  # It doesn't even really check that your current CPU is compatible.
-  #
-  # SSE4.2 available since nehalem, ca. 2008-2010
-  # Includes POPCNT for BitsSetToOne, BitParity
-  TRY_SSE42="-msse4.2"
-  # PCLMUL available since westmere, ca. 2010-2011
-  TRY_PCLMUL="-mpclmul"
-  # AVX2 available since haswell, ca. 2013-2015
-  TRY_AVX2="-mavx2"
-  # BMI available since haswell, ca. 2013-2015
-  # Primarily for TZCNT for CountTrailingZeroBits
-  TRY_BMI="-mbmi"
-  # LZCNT available since haswell, ca. 2013-2015
-  # For FloorLog2
-  TRY_LZCNT="-mlzcnt"
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_SSE42 -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <nmmintrin.h>
-  int main() {
-    volatile uint32_t x = _mm_crc32_u32(0, 0);
-    (void)x;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_SSE42 -DHAVE_SSE42"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_PCLMUL -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <wmmintrin.h>
-  int main() {
-    const auto a = _mm_set_epi64x(0, 0);
-    const auto b = _mm_set_epi64x(0, 0);
-    const auto c = _mm_clmulepi64_si128(a, b, 0x00);
-    auto d = _mm_cvtsi128_si64(c);
-    (void)d;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_PCLMUL -DHAVE_PCLMUL"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use PCLMUL intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_AVX2 -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main() {
-    const auto a = _mm256_setr_epi32(0, 1, 2, 3, 4, 7, 6, 5);
-    const auto b = _mm256_permutevar8x32_epi32(a, a);
-    (void)b;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_AVX2 -DHAVE_AVX2"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use AVX2 intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_BMI -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main(int argc, char *argv[]) {
-    (void)argv;
-    return (int)_tzcnt_u64((uint64_t)argc);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_BMI -DHAVE_BMI"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use BMI intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS $TRY_LZCNT -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  #include <immintrin.h>
-  int main(int argc, char *argv[]) {
-    (void)argv;
-    return (int)_lzcnt_u64((uint64_t)argc);
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS $TRY_LZCNT -DHAVE_LZCNT"
-elif test "$USE_SSE"; then
-  echo "warning: USE_SSE specified but compiler could not use LZCNT intrinsics, disabling" >&2
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #include <cstdint>
-  int main() {
-    uint64_t a = 0xffffFFFFffffFFFF;
-    __uint128_t b = __uint128_t(a) * a;
-    a = static_cast<uint64_t>(b >> 64);
-    (void)a;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_UINT128_EXTENSION"
-fi
-
-# iOS doesn't support thread-local storage, but this check would erroneously
-# succeed because the cross-compiler flags are added by the Makefile, not this
-# script.
-if [ "$PLATFORM" != IOS ]; then
-  $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
-  #if defined(_MSC_VER) && !defined(__thread)
-  #define __thread __declspec(thread)
-  #endif
-  int main() {
-    static __thread int tls;
-    (void)tls;
-  }
-EOF
-  if [ "$?" = 0 ]; then
-    COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
-  fi
-fi
-
-
-if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then
-  $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null <<EOF
-  void dummy_func() {}
-EOF
-  if [ "$?" = 0 ]; then
-    $CXX $COMMON_FLAGS $PLATFORM_SHARED_LDFLAGS test_dl.o -o /dev/null 2>/dev/null
-    if [ "$?" = 0 ]; then
-      EXEC_LDFLAGS+="-ldl"
-      rm -f test_dl.o
-    fi
-  fi
 fi
 
 PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS"
@@ -836,16 +438,10 @@ ROCKSDB_PATCH=`build_tools/version.sh patch`
 
 echo "CC=$CC" >> "$OUTPUT"
 echo "CXX=$CXX" >> "$OUTPUT"
-echo "AR=$AR" >> "$OUTPUT"
 echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
-echo "PLATFORM_CMAKE_FLAGS=$PLATFORM_CMAKE_FLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CCFLAGS=$JAVA_STATIC_DEPS_CCFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CXXFLAGS=$JAVA_STATIC_DEPS_CXXFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_LDFLAGS=$JAVA_STATIC_DEPS_LDFLAGS" >> "$OUTPUT"
-echo "JAVAC_ARGS=$JAVAC_ARGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
@@ -862,22 +458,7 @@ echo "ROCKSDB_PATCH=$ROCKSDB_PATCH" >> "$OUTPUT"
 echo "CLANG_SCAN_BUILD=$CLANG_SCAN_BUILD" >> "$OUTPUT"
 echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
-echo "FIND=$FIND" >> "$OUTPUT"
-echo "WATCH=$WATCH" >> "$OUTPUT"
-# This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
   echo "JEMALLOC=1" >> "$OUTPUT"
 fi
-# Indicates that jemalloc should be enabled using -ljemalloc flag
-# The alternative is to porvide a direct link to the library via JEMALLOC_LIB
-# and JEMALLOC_INCLUDE
-if test -n "$WITH_JEMALLOC_FLAG"; then
-  echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
-fi
 echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"
-if test -n "$USE_FOLLY_DISTRIBUTED_MUTEX"; then
-  echo "USE_FOLLY_DISTRIBUTED_MUTEX=$USE_FOLLY_DISTRIBUTED_MUTEX" >> "$OUTPUT"
-fi
-if test -n "$PPC_LIBC_IS_GNU"; then
-  echo "PPC_LIBC_IS_GNU=$PPC_LIBC_IS_GNU" >> "$OUTPUT"
-fi

build_tools/cont_integration.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+#
+# Copyright (c) 2016, Facebook. All rights reserved.
+#
+# Overall wrapper script for RocksDB continuous builds. The implementation is a
+# trivial pulling scheme. We loop infinitely, check if any new changes have been
+# committed, if yes then trigger a Sandcastle run, and finally go to sleep again
+# for a certain interval.
+#
+
+SRC_GIT_REPO=/data/git/rocksdb-public
+error=0
+
+function log {
+  DATE=`date +%Y-%m-%d:%H:%M:%S`
+  echo $DATE $@
+}
+
+function log_err {
+  log "ERROR: $@ Error code: $error."
+}
+
+function update_repo_status {
+  # Update the parent first.
+  pushd $SRC_GIT_REPO
+
+  # This is a fatal error. Something in the environment isn't right and we will
+  # terminate the execution.
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "Where is $SRC_GIT_REPO?"
+    exit $error
+  fi
+
+  HTTPS_PROXY=fwdproxy:8080 git fetch -f
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git fetch -f failed."
+    popd
+    return $error
+  fi
+
+  git update-ref refs/heads/master refs/remotes/origin/master
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git update-ref failed."
+    popd
+    return $error
+  fi
+
+  popd
+
+  # We're back in an instance-specific directory. Get the latest changes.
+  git pull --rebase
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git pull --rebase failed."
+    return $error
+  fi
+}
+
+#
+# Execution starts here.
+#
+
+# Path to the determinator from the root of the RocksDB repo.
+CONTRUN_DETERMINATOR=./arcanist_util/config/RocksDBCommonHelper.php
+
+# Value of the previous commit.
+PREV_COMMIT=
+
+log "Starting to monitor for new RocksDB changes ..."
+log "Running under `pwd` as `whoami`."
+
+# Paranoia. Make sure that we're using the right branch.
+git checkout master
+
+error=$?
+if [ ! $error -eq 0 ]; then
+  log_err "This is not good. Can't checkout master. Bye-bye!"
+  exit 1
+fi
+
+# We'll run forever and let the execution environment terminate us if we'll
+# exceed whatever timeout is set for the job.
+while true;
+do
+  # Get the latest changes committed.
+  update_repo_status
+
+  error=$?
+  if [  $error -eq 0 ]; then
+    LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'`
+
+    log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'."
+
+    if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then
+      log "There were no changes since the last time I checked. Going to sleep."
+    else
+      if [ ! -z "$LAST_COMMIT" ]; then
+        log "New code has been committed or previous commit not known. " \
+            "Will trigger the tests."
+
+        PREV_COMMIT=$LAST_COMMIT
+        log "Updated previous commit to '$PREV_COMMIT'."
+
+        #
+        # This is where we'll trigger the Sandcastle run. The values for
+        # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're
+        # running in.
+        #
+        POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR
+
+        error=$?
+        if [ $error -eq 0 ]; then
+          log "Sandcastle run successfully triggered."
+        else
+          log_err "Failed to trigger Sandcastle run."
+        fi
+      else
+        log_err "Previous commit not updated. Don't know what the last one is."
+      fi
+    fi
+  else
+    log_err "Getting latest changes failed. Will skip running tests for now."
+  fi
+
+  # Always sleep, even if errors happens while trying to determine the latest
+  # commit. This will prevent us terminating in case of transient errors.
+  log "Will go to sleep for 5 minutes."
+  sleep 5m
+done

build_tools/dependencies.sh

@@ -1,19 +1,19 @@
 # Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/5.x/centos7-native/c447969
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1bd23f9917738974ad0ff305aa23eb5f93f18305/9.0.0/centos7-native/c9f9104
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/5.x/gcc-5-glibc-2.23/339d858
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.23/gcc-5-glibc-2.23/ca1d1c0
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/gcc-5-glibc-2.23/9bc6787
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/gcc-5-glibc-2.23/9bc6787
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/gcc-5-glibc-2.23/9bc6787
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/gcc-5-glibc-2.23/9bc6787
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/gcc-5-glibc-2.23/03859b5
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/gcc-5-glibc-2.23/9bc6787
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/gcc-5-glibc-2.23/0c8f76d
-NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/gcc-5-glibc-2.23/9bc6787
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/gcc-5-glibc-2.23/b443de1
-TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/gcc-5-glibc-2.23/9bc6787
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/2e3cb7d119b3cea5f1e738cc13a1ac69f49eb875/2.29.1/centos7-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/gcc-5-glibc-2.23/9bc6787
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.2.3/gcc-5-glibc-2.23/65372bd
+GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
+NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
+TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
+LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/dependencies_4.8.1.sh

@@ -1,5 +1,3 @@
-# shellcheck disable=SC2148
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
 CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
 LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc

build_tools/dependencies_platform007.sh

@@ -1,20 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/7331085db891a2ef4a88a48a751d834e8d68f4cb/7.x/centos7-native/b2ef2b6
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/963d9aeda70cc4779885b1277484fe7544a04e3e/9.0.0/platform007/9e92d53/
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/6ace84e956873d53638c738b6f65f3f469cca74c/7.x/platform007/5620abc
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/192b0f42d63dcf6210d6ceae387b49af049e6e0c/2.26/platform007/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/7f9bdaada18f59bc27ec2b0871eb8a6144343aef/1.1.3/platform007/ca4da3d
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/2d9f0b9a4274cc21f61272a9e89bdb859bce8f1f/1.2.8/platform007/ca4da3d
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/dc49a21c5fceec6456a7a28a94dcd16690af1337/1.0.6/platform007/ca4da3d
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/0f607f8fc442ea7d6b876931b1898bb573d5e5da/1.9.1/platform007/ca4da3d
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/ca22bc441a4eb709e9e0b1f9fec9750fed7b31c5/1.4.x/platform007/15a3614
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/0b9929d2588991c65a57168bf88aff2db87c5d48/2.2.0/platform007/ca4da3d
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/c26f08f47ac35fc31da2633b7da92d6b863246eb/master/platform007/c26c002
-NUMA_BASE=/mnt/gvfs/third-party2/numa/3f3fb57a5ccc5fd21c66416c0b83e0aa76a05376/2.0.11/platform007/ca4da3d
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/40c73d874898b386a71847f1b99115d93822d11f/1.4/platform007/6f3e0a9
-TBB_BASE=/mnt/gvfs/third-party2/tbb/4ce8e8dba77cdbd81b75d6f0c32fd7a1b76a11ec/2018_U5/platform007/ca4da3d
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/79427253fd0d42677255aacfe6d13bfe63f752eb/20190828/platform007/ca4da3d
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/fb251ecd2f5ae16f8671f7014c246e52a748fe0b/fb/platform007/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/ab9f09bba370e7066cafd4eb59752db93f2e8312/2.29.1/platform007/15a3614
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d42d152a15636529b0861ec493927200ebebca8e/3.15.0/platform007/ca4da3d
-LUA_BASE=/mnt/gvfs/third-party2/lua/f0cd714433206d5139df61659eb7b28b1dea6683/5.3.4/platform007/5007832

build_tools/dependencies_platform009.sh

@@ -1,20 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/1795efe5f06778c15a92c8f9a2aba5dc496d9d4d/9.x/centos7-native/3bed279
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/7318eaac22659b6ff2fe43918e4b69fd0772a8a7/9.0.0/platform009/651ee30
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/4959b39cfbe5965a37c861c4c327fa7c5c759b87/9.x/platform009/9202ce7
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/45ce3375cdc77ecb2520bbf8f0ecddd3f98efd7a/2.30/platform009/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/be4de3205e029101b18aa8103daa696c2bef3b19/1.1.3/platform009/7f3b187
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/3c160ac5c67e257501e24c6c1d00ad5e01d73db6/1.2.8/platform009/7f3b187
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/73a237ac5bc0a5f5d67b39b8d253cfebaab88684/1.0.6/platform009/7f3b187
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/ec6573523b0ce55ef6373a4801189027cf07bb2c/1.9.1/platform009/7f3b187
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/64c58a207d2495e83abc57a500a956df09b79a7c/1.4.x/platform009/ba86d1f
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/824d0a8a5abb5b121afd1b35fc3896407ea50092/2.2.0/platform009/7f3b187
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/d9aef9feb850b168a68736420f217b01cce11a89/master/platform009/c305944
-NUMA_BASE=/mnt/gvfs/third-party2/numa/0af65f71e23a67bf65dc91b11f95caa39325c432/2.0.11/platform009/7f3b187
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/02486dac347645d31dce116f44e1de3177315be2/1.4/platform009/5191652
-TBB_BASE=/mnt/gvfs/third-party2/tbb/2e0ec671e550bfca347300bf3f789d9c0fff24ad/2018_U5/platform009/7f3b187
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/70dbd9cfee63a25611417d09433a86d7711b3990/20200729/platform009/7f3b187
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/32b8a2407b634df3f8f948ba373fc4acc6a18296/fb/platform009/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/08634589372fa5f237bfd374e8c644a8364e78c1/2.32/platform009/ba86d1f/
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/6ae525939ad02e5e676855082fbbc7828dbafeac/3.15.0/platform009/7f3b187
-LUA_BASE=/mnt/gvfs/third-party2/lua/162efd9561a3d21f6869f4814011e9cf1b3ff4dc/5.3.4/platform009/a6271c4

build_tools/dockerbuild.sh

@@ -1,3 +1,2 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make

build_tools/error_filter.py

@@ -64,12 +64,8 @@ class MatchErrorParser(ErrorParserBase):
 
 class CompilerErrorParser(MatchErrorParser):
     def __init__(self):
-        # format (compile error):
-        #   '<filename>:<line #>:<column #>: error: <error msg>'
-        # format (link error):
-        #   '<filename>:<line #>: error: <error msg>'
-        # The below regex catches both
-        super(CompilerErrorParser, self).__init__(r'\S+:\d+: error:')
+        # format: '<filename>:<line #>:<column #>: error: <error msg>'
+        super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:')
 
 
 class ScanBuildErrorParser(MatchErrorParser):
@@ -132,17 +128,11 @@ _TEST_NAME_TO_PARSERS = {
     'lite': [CompilerErrorParser],
     'lite_test': [CompilerErrorParser, GTestErrorParser],
     'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
-    'stress_crash_with_atomic_flush': [CompilerErrorParser, DbCrashErrorParser],
-    'stress_crash_with_txn': [CompilerErrorParser, DbCrashErrorParser],
     'write_stress': [CompilerErrorParser, WriteStressErrorParser],
     'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
     'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    'asan_crash_with_atomic_flush': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    'asan_crash_with_txn': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
     'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
     'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    'ubsan_crash_with_atomic_flush': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    'ubsan_crash_with_txn': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
     'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
     'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
     'format_compatible': [CompilerErrorParser, CompatErrorParser],

build_tools/fb_compile_mongo.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+
 # fail early
 set -e
 

build_tools/fbcode_config.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Set environment variables so that we can compile rocksdb using
 # fbcode settings.  It uses the latest g++ and clang compilers and also
@@ -14,7 +13,7 @@ source "$BASEDIR/dependencies.sh"
 CFLAGS=""
 
 # libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include"
+LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
 LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
 
 # glibc
@@ -52,7 +51,7 @@ if test -z $PIC_BUILD; then
 else
   ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
 fi
-CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY"
+CFLAGS+=" -DZSTD"
 
 # location of gflags headers and libraries
 GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
@@ -86,10 +85,8 @@ else
 fi
 CFLAGS+=" -DTBB"
 
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
@@ -109,24 +106,22 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
   JEMALLOC=1
 else
   # clang
   CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
   CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/5.x/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
   CFLAGS+=" -isystem $LIBGCC_INCLUDE"
   CFLAGS+=" -isystem $CLANG_INCLUDE"
@@ -142,9 +137,9 @@ CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
 EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-5-glibc-2.23/lib/ld.so"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
 EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-5-glibc-2.23/lib"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
 # required by libtbb
 EXEC_LDFLAGS+=" -ldl"
 
@@ -154,14 +149,4 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GF
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 
-LUA_PATH="$LUA_BASE"
-
-if test -z $PIC_BUILD; then
-  LUA_LIB=" $LUA_PATH/lib/liblua.a"
-else
-  LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
-fi
-
-USE_FOLLY_DISTRIBUTED_MUTEX=1
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD 

build_tools/fbcode_config4.8.1.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Set environment variables so that we can compile rocksdb using
 # fbcode settings.  It uses the latest g++ compiler and also
@@ -53,10 +52,8 @@ LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
 TBB_INCLUDE=" -isystem $TBB_BASE/include/"
 TBB_LIBS="$TBB_BASE/lib/libtbb.a"
 
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
@@ -69,7 +66,6 @@ if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  CXX="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -82,7 +78,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
 
@@ -98,7 +93,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE"
 CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
 CXXFLAGS+=" $CFLAGS"
 

build_tools/fbcode_config_platform007.sh

@@ -1,172 +0,0 @@
-#!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform007.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/7.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-if test -z $PIC_BUILD; then
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
-else
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
-fi
-CFLAGS+=" -DLIBURING"
-
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
-
-  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/7.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform007/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform007/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-USE_FOLLY_DISTRIBUTED_MUTEX=1
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/fbcode_config_platform009.sh

@@ -1,172 +0,0 @@
-#!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform009.sh"
-
-CFLAGS=""
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/9.3.0"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-
-# snappy
-SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-if test -z $PIC_BUILD; then
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-else
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-fi
-CFLAGS+=" -DSNAPPY"
-
-if test -z $PIC_BUILD; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-  CFLAGS+=" -DZLIB"
-
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-  CFLAGS+=" -DBZIP2"
-
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-if test -z $PIC_BUILD; then
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-else
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-fi
-CFLAGS+=" -DZSTD"
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-if test -z $PIC_BUILD; then
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
-else
-  GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags_pic.a"
-fi
-CFLAGS+=" -DGFLAGS=gflags"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc.a"
-
-if test -z $PIC_BUILD; then
-  # location of numa
-  NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-  NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
-  CFLAGS+=" -DNUMA"
-
-  # location of libunwind
-  LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
-fi
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-if test -z $PIC_BUILD; then
-  TBB_LIBS="$TBB_BASE/lib/libtbb.a"
-else
-  TBB_LIBS="$TBB_BASE/lib/libtbb_pic.a"
-fi
-CFLAGS+=" -DTBB"
-
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-if test -z $PIC_BUILD; then
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing.a"
-else
-  LIBURING_LIBS="$LIBURING_BASE/lib/liburing_pic.a"
-fi
-CFLAGS+=" -DLIBURING"
-
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_SRC/llvm/clang/tools/scan-build/bin/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
-
-  CFLAGS+=" -B$BINUTILS/gold"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
-
-  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-
-  CFLAGS+=" -B$BINUTILS/gold -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/9.x/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42 -DROCKSDB_IOURING_PRESENT"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS"
-EXEC_LDFLAGS+=" -B$BINUTILS/gold"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform009/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform009/lib"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-# lua not supported because it's on track for deprecation, I think
-LUA_PATH=
-LUA_LIB=
-
-USE_FOLLY_DISTRIBUTED_MUTEX=1
-
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/format-diff.sh

@@ -1,100 +1,36 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
+if [ -z $CLANG_FORMAT_DIFF ]
+then
+CLANG_FORMAT_DIFF="clang-format-diff.py"
+fi
 
-print_usage () {
-  echo "Usage:"
-  echo "format-diff.sh [OPTIONS]"
-  echo "-c: check only."
-  echo "-h: print this message."
-}
+# Check clang-format-diff.py
+if ! which $CLANG_FORMAT_DIFF &> /dev/null
+then
+  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
+  echo "You can download clang-format-diff.py by running: "
+  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
+  echo "You can download clang-format by running: "
+  echo "    brew install clang-format"
+  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  exit 128
+fi
 
-while getopts ':ch' OPTION; do
-  case "$OPTION" in
-    c)
-      CHECK_ONLY=1
-      ;;
-    h)
-      print_usage
-      exit 1
-      ;;
-    ?)
-      print_usage
-      exit 1
-      ;;
-  esac
-done
+# Check argparse, a library that clang-format-diff.py requires.
+python 2>/dev/null << EOF
+import argparse
+EOF
 
-REPO_ROOT="$(git rev-parse --show-toplevel)"
-
-if [ "$CLANG_FORMAT_DIFF" ]; then
-  echo "Note: CLANG_FORMAT_DIFF='$CLANG_FORMAT_DIFF'"
-  # Dry run to confirm dependencies like argparse
-  if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-    true #Good
-  else
-    exit 128
-  fi
-else
-  # First try directly executing the possibilities
-  if clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=clang-format-diff.py
-  elif $REPO_ROOT/clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=$REPO_ROOT/clang-format-diff.py
-  else
-    # This probably means we need to directly invoke the interpreter.
-    # But first find clang-format-diff.py
-    if [ -f "$REPO_ROOT/clang-format-diff.py" ]; then
-      CFD_PATH="$REPO_ROOT/clang-format-diff.py"
-    elif which clang-format-diff.py &> /dev/null; then
-      CFD_PATH="$(which clang-format-diff.py)"
-    else
-      echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-      echo "You can download clang-format-diff.py by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      echo "You can download clang-format by running:"
-      echo "    brew install clang-format"
-      echo "  Or"
-      echo "    apt install clang-format"
-      echo "  This might work too:"
-      echo "    yum install git-clang-format"
-      echo "Then make sure clang-format is available and executable from \$PATH:"
-      echo "    clang-format --version"
-      exit 128
-    fi
-    # Check argparse pre-req on interpreter, or it will fail
-    if echo import argparse | ${PYTHON:-python3}; then
-      true # Good
-    else
-      echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
-      echo "installed. You can try either of the follow ways to install it:"
-      echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
-      echo "  2. easy_install argparse (if you have easy_install)"
-      echo "  3. pip install argparse (if you have pip)"
-      exit 129
-    fi
-    # Unfortunately, some machines have a Python2 clang-format-diff.py
-    # installed but only a Python3 interpreter installed. Unfortunately,
-    # automatic 2to3 migration is insufficient, so suggest downloading latest.
-    if grep -q "print '" "$CFD_PATH" && \
-       ${PYTHON:-python3} --version | grep -q 'ython 3'; then
-      echo "You have clang-format-diff.py for Python 2 but are using a Python 3"
-      echo "interpreter (${PYTHON:-python3})."
-      echo "You can download clang-format-diff.py for Python 3 by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      exit 130
-    fi
-    CLANG_FORMAT_DIFF="${PYTHON:-python3} $CFD_PATH"
-    # This had better work after all those checks
-    if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-      true #Good
-    else
-      exit 128
-    fi
-  fi
+if [ "$?" != 0 ]
+then
+  echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
+  echo "installed. You can try either of the follow ways to install it:"
+  echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
+  echo "  2. easy_install argparse (if you have easy_install)"
+  echo "  3. pip install argparse (if you have pip)"
+  exit 129
 fi
 
 # TODO(kailiu) following work is not complete since we still need to figure
@@ -117,54 +53,34 @@ fi
 set -e
 
 uncommitted_code=`git diff HEAD`
+LAST_MASTER=`git merge-base master HEAD`
 
 # If there's no uncommitted changes, we assume user are doing post-commit
-# format check, in which case we'll try to check the modified lines vs. the
-# facebook/rocksdb.git master branch. Otherwise, we'll check format of the
-# uncommitted code only.
+# format check, in which case we'll check the modified lines since last commit
+# from master. Otherwise, we'll check format of the uncommitted code only.
 if [ -z "$uncommitted_code" ]
 then
-  # Attempt to get name of facebook/rocksdb.git remote.
-  [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE="$(git remote -v | grep 'facebook/rocksdb.git' | head -n 1 | cut -f 1)"
-  # Fall back on 'origin' if that fails
-  [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE=origin
-  # Use master branch from that remote
-  [ "$FORMAT_UPSTREAM" ] || FORMAT_UPSTREAM="$FORMAT_REMOTE/master"
-  # Get the common ancestor with that remote branch. Everything after that
-  # common ancestor would be considered the contents of a pull request, so
-  # should be relevant for formatting fixes.
-  FORMAT_UPSTREAM_MERGE_BASE="$(git merge-base "$FORMAT_UPSTREAM" HEAD)"
-  # Get the differences
-  diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -p 1)
-  echo "Checking format of changes not yet in $FORMAT_UPSTREAM..."
+  # Check the format of last commit
+  diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
 else
   # Check the format of uncommitted lines,
   diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
-  echo "Checking format of uncommitted changes..."
 fi
 
 if [ -z "$diffs" ]
 then
   echo "Nothing needs to be reformatted!"
   exit 0
-elif [ $CHECK_ONLY ]
-then
-  echo "Your change has unformatted code. Please run make format!"
-  if [ $VERBOSE_CHECK ]; then
-    clang-format --version
-    echo "$diffs"
-  fi
-  exit 1
 fi
 
 # Highlight the insertion/deletion from the clang-format-diff.py's output
 COLOR_END="\033[0m"
-COLOR_RED="\033[0;31m"
-COLOR_GREEN="\033[0;32m"
+COLOR_RED="\033[0;31m" 
+COLOR_GREEN="\033[0;32m" 
 
 echo -e "Detect lines that doesn't follow the format rules:\r"
 # Add the color to the diff. lines added will be green; lines removed will be red.
-echo "$diffs" |
+echo "$diffs" | 
   sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
   sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
 
@@ -187,9 +103,9 @@ fi
 # Do in-place format adjustment.
 if [ -z "$uncommitted_code" ]
 then
-  git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
 else
-  git diff -U0 HEAD | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
 fi
 echo "Files reformatted!"
 

build_tools/gnu_parallel

@@ -5082,8 +5082,8 @@ sub openoutputfiles {
 	# Set reading FD if using --group (--ungroup does not need)
 	for my $fdno (1,2) {
 	    # Re-open the file for reading
-	    # so fdw can be closed separately
-	    # and fdr can be seeked separately (for --line-buffer)
+	    # so fdw can be closed seperately
+	    # and fdr can be seeked seperately (for --line-buffer)
 	    open(my $fdr,"<", $self->fh($fdno,'name')) ||
 		::die_bug("fdr: Cannot open ".$self->fh($fdno,'name'));
 	    $self->set_fh($fdno,'r',$fdr);
@@ -5801,7 +5801,7 @@ sub workdir {
 		    . "-" . $self->seq();
 	    } else {
 		$workdir = $opt::workdir;
-		# Rsync treats /./ special. We don't want that
+		# Rsync treats /./ special. We dont want that
 		$workdir =~ s:/\./:/:g; # Remove /./
 		$workdir =~ s:/+$::; # Remove ending / if any
 		$workdir =~ s:^\./::g; # Remove starting ./ if any

build_tools/make_package.sh

@@ -1,6 +1,4 @@
-# shellcheck disable=SC1113
 #/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 set -e
 
@@ -30,14 +28,12 @@ function package() {
     if dpkg --get-selections | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
-      # shellcheck disable=SC2068
       apt-get install $@ -y
     fi
   elif [[ $OS = "centos" ]]; then
     if rpm -qa | grep --quiet $1; then
       log "$1 is already installed. skipping."
     else
-      # shellcheck disable=SC2068
       yum install $@ -y
     fi
   fi
@@ -56,7 +52,6 @@ function gem_install() {
   if gem list | grep --quiet $1; then
     log "$1 is already installed. skipping."
   else
-    # shellcheck disable=SC2068
     gem install $@
   fi
 }
@@ -103,27 +98,19 @@ function main() {
   gem_install fpm
 
   make static_lib
-  LIBDIR=/usr/lib
-  if [[ $FPM_OUTPUT = "rpm" ]]; then
-      LIBDIR=$(rpm --eval '%_libdir')
-  fi
-
-  rm -rf package
-  make install DESTDIR=package PREFIX=/usr LIBDIR=$LIBDIR
-
+  make install INSTALL_PATH=package
   fpm \
     -s dir \
     -t $FPM_OUTPUT \
-    -C package \
     -n rocksdb \
     -v $1 \
+    --prefix /usr \
     --url http://rocksdb.org/ \
     -m rocksdb@fb.com \
     --license BSD \
     --vendor Facebook \
     --description "RocksDB is an embeddable persistent key-value store for fast storage." \
-    usr
+    package
 }
 
-# shellcheck disable=SC2068
 main $@

build_tools/precommit_checker.py

@@ -1,5 +1,4 @@
-#!/usr/bin/env python2.7
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
 
 from __future__ import absolute_import
 from __future__ import division

build_tools/regression_build_test.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 
 set -e
 
@@ -378,7 +377,7 @@ function send_to_ods {
     echo >&2 "ERROR: Key $key doesn't have a value."
     return
   fi
-  curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
+  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
     --connect-timeout 60
 }
 

build_tools/run_ci_db_test.ps1

@@ -1,17 +1,13 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # This script enables you running RocksDB tests by running
 # All the tests concurrently and utilizing all the cores
 Param(
-  [switch]$EnableJE = $false,  # Look for and use test executable, append _je to listed exclusions
+  [switch]$EnableJE = $false,  # Look for and use _je executable, append _je to listed exclusions
   [switch]$RunAll = $false,    # Will attempt discover all *_test[_je].exe binaries and run all
                                # of them as Google suites. I.e. It will run test cases concurrently
                                # except those mentioned as $Run, those will run as individual test cases
                                # And any execlued with $ExcludeExes or $ExcludeCases
                                # It will also not run any individual test cases
                                # excluded but $ExcludeCasese
-  [switch]$RunAllExe = $false, # Look for and use test exdcutables, append _je to exclusions automatically
-                               # It will attempt to run them in parallel w/o breaking them up on individual
-                               # test cases. Those listed with $ExcludeExes will be excluded
   [string]$SuiteRun = "",      # Split test suites in test cases and run in parallel, not compatible with $RunAll
   [string]$Run = "",           # Run specified executables in parallel but do not split to test cases
   [string]$ExcludeCases = "",  # Exclude test cases, expects a comma separated list, no spaces
@@ -43,18 +39,13 @@ $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
 $RunOnly.Add("merge_test") | Out-Null
 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
 $RunOnly.Add("backupable_db_test") | Out-Null # Disabled
-$RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
+
 
 if($RunAll -and $SuiteRun -ne "") {
     Write-Error "$RunAll and $SuiteRun are not compatible"
     exit 1
 }
 
-if($RunAllExe -and $Run -ne "") {
-    Write-Error "$RunAllExe and $Run are not compatible"
-    exit 1
-}
-
 # If running under Appveyor assume that root
 [string]$Appveyor = $Env:APPVEYOR_BUILD_FOLDER
 if($Appveyor -ne "") {
@@ -68,7 +59,7 @@ $BinariesFolder = -Join($RootFolder, "\build\Debug\")
 
 if($WorkFolder -eq "") {
 
-    # If TEST_TMPDIR is set use it
+    # If TEST_TMPDIR is set use it    
     [string]$var = $Env:TEST_TMPDIR
     if($var -eq "") {
         $WorkFolder = -Join($RootFolder, "\db_tests\")
@@ -93,7 +84,7 @@ $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeCases -ne "") {
     Write-Host "ExcludeCases: $ExcludeCases"
     $l = $ExcludeCases -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeCasesSet.Add($t) | Out-Null
     }
 }
@@ -102,7 +93,7 @@ $ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeExes -ne "") {
     Write-Host "ExcludeExe: $ExcludeExes"
     $l = $ExcludeExes -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeExesSet.Add($t) | Out-Null
     }
 }
@@ -118,10 +109,6 @@ if($ExcludeExes -ne "") {
 #   MultiThreaded/MultiThreadedDBTest.
 #     MultiThreaded/0  # GetParam() = 0
 #     MultiThreaded/1  # GetParam() = 1
-#   RibbonTypeParamTest/0.  # TypeParam = struct DefaultTypesAndSettings
-#     CompactnessAndBacktrackAndFpRate
-#     Extremes
-#     FindOccupancyForSuccessRate
 #
 # into this:
 #
@@ -129,9 +116,6 @@ if($ExcludeExes -ne "") {
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-#   RibbonTypeParamTest/0.CompactnessAndBacktrackAndFpRate
-#   RibbonTypeParamTest/0.Extremes
-#   RibbonTypeParamTest/0.FindOccupancyForSuccessRate
 #
 # Output into the parameter in a form TestName -> Log File Name
 function ExtractTestCases([string]$GTestExe, $HashTable) {
@@ -145,17 +129,20 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
     ForEach( $l in $Tests) {
 
-      # remove trailing comment if any
-      $l = $l -replace '\s+\#.*',''
       # Leading whitespace is fine
       $l = $l -replace '^\s+',''
+      # but no whitespace any other place
+      if($l -match "\s+") {
+        continue
+      }
       # Trailing dot is a test group but no whitespace
-      if ($l -match "\.$" -and $l -notmatch "\s+") {
+      elseif ( $l -match "\.$" ) {
         $Group = $l
       }  else {
         # Otherwise it is a test name, remove leading space
         $test = $l
-        # create a log name
+        # remove trailing comment if any and create a log name
+        $test = $test -replace '\s+\#.*',''
         $test = "$Group$test"
 
         if($ExcludeCasesSet.Contains($test)) {
@@ -236,11 +223,13 @@ $TestExes = [ordered]@{}
 if($Run -ne "") {
 
   $test_list = $Run -split ' '
+
   ForEach($t in $test_list) {
 
     if($EnableJE) {
       $t += "_je"
     }
+
     MakeAndAdd -token $t -HashTable $TestExes
   }
 
@@ -248,38 +237,6 @@ if($Run -ne "") {
      Write-Error "Failed to extract tests from $Run"
      exit 1
   }
-} elseif($RunAllExe) {
-  # Discover all the test binaries
-  if($EnableJE) {
-    $pattern = "*_test_je.exe"
-  } else {
-    $pattern = "*_test.exe"
-  }
-
-  $search_path = -join ($BinariesFolder, $pattern)
-  Write-Host "Binaries Search Path: $search_path"
-
-  $DiscoveredExe = @()
-  dir -Path $search_path | ForEach-Object {
-     $DiscoveredExe += ($_.Name)
-  }
-
-  # Remove exclusions
-  ForEach($e in $DiscoveredExe) {
-    $e = $e -replace '.exe$', ''
-    $bare_name = $e -replace '_je$', ''
-
-    if($ExcludeExesSet.Contains($bare_name)) {
-      Write-Warning "Test $e is excluded"
-      continue
-    }
-    MakeAndAdd -token $e -HashTable $TestExes
-  }
-
-  if($TestExes.Count -lt 1) {
-     Write-Error "Failed to discover test executables"
-     exit 1
-  }
 }
 
 # Ordered by exe @{ Exe = @{ TestCase = LogName }}
@@ -288,7 +245,9 @@ $CasesToRun = [ordered]@{}
 if($SuiteRun -ne "") {
   $suite_list = $SuiteRun -split ' '
   ProcessSuites -ListOfSuites $suite_list -HashOfHashes $CasesToRun
-} elseif ($RunAll) {
+}
+
+if($RunAll) {
 # Discover all the test binaries
   if($EnableJE) {
     $pattern = "*_test_je.exe"
@@ -296,12 +255,13 @@ if($SuiteRun -ne "") {
     $pattern = "*_test.exe"
   }
 
+
   $search_path = -join ($BinariesFolder, $pattern)
   Write-Host "Binaries Search Path: $search_path"
 
   $ListOfExe = @()
   dir -Path $search_path | ForEach-Object {
-     $ListOfExe += ($_.Name)
+     $ListOfExe += ($_.Name)     
   }
 
   # Exclude those in RunOnly from running as suites
@@ -316,7 +276,7 @@ if($SuiteRun -ne "") {
       continue
     }
 
-    if($RunOnly.Contains($bare_name)) {
+    if($RunOnly.Contains($e)) {
       MakeAndAdd -token $e -HashTable $TestExes
     } else {
       $ListOfSuites += $bare_name
@@ -327,6 +287,8 @@ if($SuiteRun -ne "") {
 }
 
 
+Write-Host "Attempting to start: $NumTestsToStart tests"
+
 # Invoke a test with a filter and redirect all output
 $InvokeTestCase = {
     param($exe, $test, $log);
@@ -345,7 +307,7 @@ $InvokeTestAsync = {
 # Test limiting factor here
 [int]$count = 0
 # Overall status
-[bool]$script:success = $true;
+[bool]$success = $true;
 
 function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 {
@@ -356,7 +318,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
-          ($TestCmds.Count -gt 0) -or
+          ($TestCmds.Count -gt 0) -or 
            ($Suites.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
@@ -371,7 +333,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
             $Cases = @{}
 
             if($Suites.Count -gt 0) {
-              # Will the first one
+              # Will teh first one
               ForEach($e in $Suites.Keys) {
                 $exe_name = $e
                 $Cases = $Suites[$e]
@@ -403,7 +365,6 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
                  break
                }
 
-              Write-Host "Starting $exe_name"
               [string]$Exe =  -Join ($BinariesFolder, $exe_name)
               $job = Start-Job -Name $exe_name -ScriptBlock $InvokeTestAsync -ArgumentList @($Exe,$log_path)
               $JobToLog.Add($job, $log_path)
@@ -434,7 +395,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
         $log_content = @(Get-Content $log)
 
         if($completed.State -ne "Completed") {
-            $script:success = $false
+            $success = $false
             Write-Warning $message
             $log_content | Write-Warning
         } else {
@@ -458,7 +419,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
             }
 
             if(!$pass_found) {
-                $script:success = $false;
+                $success = $false;
                 Write-Warning $message
                 $log_content | Write-Warning
             } else {
@@ -476,13 +437,13 @@ RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
 
 $EndDate = (Get-Date)
 
-New-TimeSpan -Start $StartDate -End $EndDate |
-  ForEach-Object {
+New-TimeSpan -Start $StartDate -End $EndDate | 
+  ForEach-Object { 
     "Elapsed time: {0:g}" -f $_
   }
 
 
-if(!$script:success) {
+if(!$success) {
 # This does not succeed killing off jobs quick
 # So we simply exit
 #    Remove-Job -Job $jobs -Force
@@ -492,4 +453,4 @@ if(!$script:success) {
 
  exit 0
 
-
+ 

build_tools/setup_centos7.sh

@@ -1,45 +0,0 @@
-#!/bin/bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-set -ex
-
-ROCKSDB_VERSION="6.7.3"
-ZSTD_VERSION="1.4.4"
-
-echo "This script configures CentOS with everything needed to build and run RocksDB"
-
-yum update -y && yum install epel-release -y
-
-yum install -y \
-  wget \
-  gcc-c++ \
-  snappy snappy-devel \
-  zlib zlib-devel \
-  bzip2 bzip2-devel \
-  lz4-devel \
-  libasan \
-  gflags
-
-mkdir -pv /usr/local/rocksdb-${ROCKSDB_VERSION}
-ln -sfT /usr/local/rocksdb-${ROCKSDB_VERSION} /usr/local/rocksdb
-
-wget -qO /tmp/zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/v${ZSTD_VERSION}.tar.gz
-wget -qO /tmp/rocksdb-${ROCKSDB_VERSION}.tar.gz https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz
-
-cd /tmp
-
-tar xzvf zstd-${ZSTD_VERSION}.tar.gz
-tar xzvf rocksdb-${ROCKSDB_VERSION}.tar.gz -C /usr/local/
-
-echo "Installing ZSTD..."
-pushd zstd-${ZSTD_VERSION}
-make && make install
-popd
-
-echo "Compiling RocksDB..."
-cd /usr/local/rocksdb
-chown -R vagrant:vagrant /usr/local/rocksdb/
-sudo -u vagrant make static_lib
-cd examples/
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ make all
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ ./c_simple_example
-

build_tools/update_dependencies.sh

@@ -1,17 +1,10 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Update dependencies.sh file with the latest avaliable versions
 
 BASEDIR=$(dirname $0)
 OUTPUT=""
 
-function log_header()
-{
-  echo "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved." >> "$OUTPUT"
-}
-
-
 function log_variable()
 {
   echo "$1=${!1}" >> "$OUTPUT"
@@ -45,7 +38,6 @@ function get_lib_base()
     # platform is not provided, use latest gcc
     result=`ls -dr1v $result/gcc-*[^fb]/ | head -n1`
   else
-    echo $lib_platform
     result="$result/$lib_platform/"
   fi
   
@@ -61,48 +53,7 @@ function get_lib_base()
 }
 
 ###########################################################
-#                platform007 dependencies                 #
-###########################################################
-
-OUTPUT="$BASEDIR/dependencies_platform007.sh"
-
-rm -f "$OUTPUT"
-touch "$OUTPUT"
-
-echo "Writing dependencies to $OUTPUT"
-
-# Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/7.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
-
-log_header
-log_variable GCC_BASE
-log_variable CLANG_BASE
-
-# Libraries locations
-get_lib_base libgcc     7.x     platform007
-get_lib_base glibc      2.26    platform007
-get_lib_base snappy     LATEST  platform007
-get_lib_base zlib       LATEST  platform007
-get_lib_base bzip2      LATEST  platform007
-get_lib_base lz4        LATEST  platform007
-get_lib_base zstd       LATEST  platform007
-get_lib_base gflags     LATEST  platform007
-get_lib_base jemalloc   LATEST  platform007
-get_lib_base numa       LATEST  platform007
-get_lib_base libunwind  LATEST  platform007
-get_lib_base tbb        LATEST  platform007
-get_lib_base liburing   LATEST  platform007
-
-get_lib_base kernel-headers fb platform007
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST platform007
-get_lib_base lua        5.3.4  platform007
-
-git diff $OUTPUT
-
-###########################################################
-#                   5.x dependencies                      #
+#                   4.9.x dependencies                    #
 ###########################################################
 
 OUTPUT="$BASEDIR/dependencies.sh"
@@ -113,31 +64,30 @@ touch "$OUTPUT"
 echo "Writing dependencies to $OUTPUT"
 
 # Compilers locations
-GCC_BASE=`readlink -f $TP2_LATEST/gcc/5.x/centos7-native/*/`
-CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos7-native/*/`
+GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.9.x/centos6-native/*/`
+CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
-log_header
 log_variable GCC_BASE
 log_variable CLANG_BASE
 
 # Libraries locations
-get_lib_base libgcc     5.x     gcc-5-glibc-2.23
-get_lib_base glibc      2.23    gcc-5-glibc-2.23
-get_lib_base snappy     LATEST  gcc-5-glibc-2.23
-get_lib_base zlib       LATEST  gcc-5-glibc-2.23
-get_lib_base bzip2      LATEST  gcc-5-glibc-2.23
-get_lib_base lz4        LATEST  gcc-5-glibc-2.23
-get_lib_base zstd       LATEST  gcc-5-glibc-2.23
-get_lib_base gflags     LATEST  gcc-5-glibc-2.23
-get_lib_base jemalloc   LATEST  gcc-5-glibc-2.23
-get_lib_base numa       LATEST  gcc-5-glibc-2.23
-get_lib_base libunwind  LATEST  gcc-5-glibc-2.23
-get_lib_base tbb        LATEST  gcc-5-glibc-2.23
+get_lib_base libgcc     4.9.x
+get_lib_base glibc      2.20
+get_lib_base snappy     LATEST gcc-4.9-glibc-2.20
+get_lib_base zlib       LATEST
+get_lib_base bzip2      LATEST
+get_lib_base lz4        LATEST
+get_lib_base zstd       LATEST
+get_lib_base gflags     LATEST
+get_lib_base jemalloc   LATEST
+get_lib_base numa       LATEST
+get_lib_base libunwind  LATEST
+get_lib_base tbb        4.0_update2 gcc-4.9-glibc-2.20
 
-get_lib_base kernel-headers 4.0.9-36_fbk5_2933_gd092e3f gcc-5-glibc-2.23
-get_lib_base binutils   LATEST centos7-native
-get_lib_base valgrind   LATEST gcc-5-glibc-2.23
-get_lib_base lua        5.2.3 gcc-5-glibc-2.23
+get_lib_base kernel-headers LATEST 
+get_lib_base binutils   LATEST centos6-native 
+get_lib_base valgrind   3.10.0 gcc-4.9-glibc-2.20
+get_lib_base lua        5.2.3 gcc-4.9-glibc-2.20
 
 git diff $OUTPUT
 
@@ -156,7 +106,6 @@ echo "Writing 4.8.1 dependencies to $OUTPUT"
 GCC_BASE=`readlink -f $TP2_LATEST/gcc/4.8.1/centos6-native/*/`
 CLANG_BASE=`readlink -f $TP2_LATEST/llvm-fb/stable/centos6-native/*/`
 
-log_header
 log_variable GCC_BASE
 log_variable CLANG_BASE
 

build_tools/version.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/sh
 if [ "$#" = "0" ]; then
   echo "Usage: $0 major|minor|patch|full"
   exit 1

cache/cache.cc

@@ -1,63 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "rocksdb/cache.h"
-
-#include "cache/lru_cache.h"
-#include "rocksdb/utilities/options_type.h"
-#include "util/string_util.h"
-
-namespace ROCKSDB_NAMESPACE {
-#ifndef ROCKSDB_LITE
-static std::unordered_map<std::string, OptionTypeInfo>
-    lru_cache_options_type_info = {
-        {"capacity",
-         {offsetof(struct LRUCacheOptions, capacity), OptionType::kSizeT,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
-        {"num_shard_bits",
-         {offsetof(struct LRUCacheOptions, num_shard_bits), OptionType::kInt,
-          OptionVerificationType::kNormal, OptionTypeFlags::kMutable}},
-        {"strict_capacity_limit",
-         {offsetof(struct LRUCacheOptions, strict_capacity_limit),
-          OptionType::kBoolean, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable}},
-        {"high_pri_pool_ratio",
-         {offsetof(struct LRUCacheOptions, high_pri_pool_ratio),
-          OptionType::kDouble, OptionVerificationType::kNormal,
-          OptionTypeFlags::kMutable}},
-};
-#endif  // ROCKSDB_LITE
-
-Status Cache::CreateFromString(const ConfigOptions& config_options,
-                               const std::string& value,
-                               std::shared_ptr<Cache>* result) {
-  Status status;
-  std::shared_ptr<Cache> cache;
-  if (value.find('=') == std::string::npos) {
-    cache = NewLRUCache(ParseSizeT(value));
-  } else {
-#ifndef ROCKSDB_LITE
-    LRUCacheOptions cache_opts;
-    status = OptionTypeInfo::ParseStruct(
-        config_options, "", &lru_cache_options_type_info, "", value,
-        reinterpret_cast<char*>(&cache_opts));
-    if (status.ok()) {
-      cache = NewLRUCache(cache_opts);
-    }
-#else
-    (void)config_options;
-    status = Status::NotSupported("Cannot load cache in LITE mode ", value);
-#endif  //! ROCKSDB_LITE
-  }
-  if (status.ok()) {
-    result->swap(cache);
-  }
-  return status;
-}
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_bench.cc

@@ -3,6 +3,9 @@
 //  COPYING file in the root directory) and Apache 2.0 License
 //  (found in the LICENSE.Apache file in the root directory).
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
 #ifndef GFLAGS
 #include <cstdio>
 int main() {
@@ -11,68 +14,59 @@ int main() {
 }
 #else
 
+#include <inttypes.h>
 #include <sys/types.h>
+#include <stdio.h>
+#include <gflags/gflags.h>
 
-#include <cinttypes>
-#include <cstdio>
-#include <limits>
-
-#include "port/port.h"
-#include "rocksdb/cache.h"
 #include "rocksdb/db.h"
+#include "rocksdb/cache.h"
 #include "rocksdb/env.h"
-#include "rocksdb/system_clock.h"
-#include "util/coding.h"
-#include "util/gflags_compat.h"
-#include "util/hash.h"
+#include "port/port.h"
 #include "util/mutexlock.h"
 #include "util/random.h"
 
-using GFLAGS_NAMESPACE::ParseCommandLineFlags;
+using GFLAGS::ParseCommandLineFlags;
 
-static constexpr uint32_t KiB = uint32_t{1} << 10;
-static constexpr uint32_t MiB = KiB << 10;
-static constexpr uint64_t GiB = MiB << 10;
+static const uint32_t KB = 1024;
 
-DEFINE_uint32(threads, 16, "Number of concurrent threads to run.");
-DEFINE_uint64(cache_size, 1 * GiB,
-              "Number of bytes to use as a cache of uncompressed data.");
-DEFINE_uint32(num_shard_bits, 6, "shard_bits.");
+DEFINE_int32(threads, 16, "Number of concurrent threads to run.");
+DEFINE_int64(cache_size, 8 * KB * KB,
+             "Number of bytes to use as a cache of uncompressed data.");
+DEFINE_int32(num_shard_bits, 4, "shard_bits.");
 
-DEFINE_double(resident_ratio, 0.25,
-              "Ratio of keys fitting in cache to keyspace.");
-DEFINE_uint64(ops_per_thread, 0,
-              "Number of operations per thread. (Default: 5 * keyspace size)");
-DEFINE_uint32(value_bytes, 8 * KiB, "Size of each value added.");
+DEFINE_int64(max_key, 1 * KB * KB * KB, "Max number of key to place in cache");
+DEFINE_uint64(ops_per_thread, 1200000, "Number of operations per thread.");
 
-DEFINE_uint32(skew, 5, "Degree of skew in key selection");
-DEFINE_bool(populate_cache, true, "Populate cache before operations");
-
-DEFINE_uint32(lookup_insert_percent, 87,
-              "Ratio of lookup (+ insert on not found) to total workload "
-              "(expressed as a percentage)");
-DEFINE_uint32(insert_percent, 2,
-              "Ratio of insert to total workload (expressed as a percentage)");
-DEFINE_uint32(lookup_percent, 10,
-              "Ratio of lookup to total workload (expressed as a percentage)");
-DEFINE_uint32(erase_percent, 1,
-              "Ratio of erase to total workload (expressed as a percentage)");
+DEFINE_bool(populate_cache, false, "Populate cache before operations");
+DEFINE_int32(insert_percent, 40,
+             "Ratio of insert to total workload (expressed as a percentage)");
+DEFINE_int32(lookup_percent, 50,
+             "Ratio of lookup to total workload (expressed as a percentage)");
+DEFINE_int32(erase_percent, 10,
+             "Ratio of erase to total workload (expressed as a percentage)");
 
 DEFINE_bool(use_clock_cache, false, "");
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 class CacheBench;
 namespace {
+void deleter(const Slice& key, void* value) {
+    delete reinterpret_cast<char *>(value);
+}
+
 // State shared by all concurrent executions of the same benchmark.
 class SharedState {
  public:
   explicit SharedState(CacheBench* cache_bench)
       : cv_(&mu_),
+        num_threads_(FLAGS_threads),
         num_initialized_(0),
         start_(false),
         num_done_(0),
-        cache_bench_(cache_bench) {}
+        cache_bench_(cache_bench) {
+  }
 
   ~SharedState() {}
 
@@ -96,9 +90,13 @@ class SharedState {
     num_done_++;
   }
 
-  bool AllInitialized() const { return num_initialized_ >= FLAGS_threads; }
+  bool AllInitialized() const {
+    return num_initialized_ >= num_threads_;
+  }
 
-  bool AllDone() const { return num_done_ >= FLAGS_threads; }
+  bool AllDone() const {
+    return num_done_ >= num_threads_;
+  }
 
   void SetStart() {
     start_ = true;
@@ -112,6 +110,7 @@ class SharedState {
   port::Mutex mu_;
   port::CondVar cv_;
 
+  const uint64_t num_threads_;
   uint64_t num_initialized_;
   bool start_;
   uint64_t num_done_;
@@ -122,69 +121,17 @@ class SharedState {
 // Per-thread state for concurrent executions of the same benchmark.
 struct ThreadState {
   uint32_t tid;
-  Random64 rnd;
+  Random rnd;
   SharedState* shared;
 
   ThreadState(uint32_t index, SharedState* _shared)
       : tid(index), rnd(1000 + index), shared(_shared) {}
 };
-
-struct KeyGen {
-  char key_data[27];
-
-  Slice GetRand(Random64& rnd, uint64_t max_key) {
-    uint64_t raw = rnd.Next();
-    // Skew according to setting
-    for (uint32_t i = 0; i < FLAGS_skew; ++i) {
-      raw = std::min(raw, rnd.Next());
-    }
-    uint64_t key = FastRange64(raw, max_key);
-    // Variable size and alignment
-    size_t off = key % 8;
-    key_data[0] = char{42};
-    EncodeFixed64(key_data + 1, key);
-    key_data[9] = char{11};
-    EncodeFixed64(key_data + 10, key);
-    key_data[18] = char{4};
-    EncodeFixed64(key_data + 19, key);
-    return Slice(&key_data[off], sizeof(key_data) - off);
-  }
-};
-
-char* createValue(Random64& rnd) {
-  char* rv = new char[FLAGS_value_bytes];
-  // Fill with some filler data, and take some CPU time
-  for (uint32_t i = 0; i < FLAGS_value_bytes; i += 8) {
-    EncodeFixed64(rv + i, rnd.Next());
-  }
-  return rv;
-}
-
-void deleter(const Slice& /*key*/, void* value) {
-  delete[] static_cast<char*>(value);
-}
 }  // namespace
 
 class CacheBench {
-  static constexpr uint64_t kHundredthUint64 =
-      std::numeric_limits<uint64_t>::max() / 100U;
-
  public:
-  CacheBench()
-      : max_key_(static_cast<uint64_t>(FLAGS_cache_size / FLAGS_resident_ratio /
-                                       FLAGS_value_bytes)),
-        lookup_insert_threshold_(kHundredthUint64 *
-                                 FLAGS_lookup_insert_percent),
-        insert_threshold_(lookup_insert_threshold_ +
-                          kHundredthUint64 * FLAGS_insert_percent),
-        lookup_threshold_(insert_threshold_ +
-                          kHundredthUint64 * FLAGS_lookup_percent),
-        erase_threshold_(lookup_threshold_ +
-                         kHundredthUint64 * FLAGS_erase_percent) {
-    if (erase_threshold_ != 100U * kHundredthUint64) {
-      fprintf(stderr, "Percentages must add to 100.\n");
-      exit(1);
-    }
+  CacheBench() : num_threads_(FLAGS_threads) {
     if (FLAGS_use_clock_cache) {
       cache_ = NewClockCache(FLAGS_cache_size, FLAGS_num_shard_bits);
       if (!cache_) {
@@ -194,32 +141,30 @@ class CacheBench {
     } else {
       cache_ = NewLRUCache(FLAGS_cache_size, FLAGS_num_shard_bits);
     }
-    if (FLAGS_ops_per_thread == 0) {
-      FLAGS_ops_per_thread = 5 * max_key_;
-    }
   }
 
   ~CacheBench() {}
 
   void PopulateCache() {
-    Random64 rnd(1);
-    KeyGen keygen;
-    for (uint64_t i = 0; i < 2 * FLAGS_cache_size; i += FLAGS_value_bytes) {
-      cache_->Insert(keygen.GetRand(rnd, max_key_), createValue(rnd),
-                     FLAGS_value_bytes, &deleter);
+    Random rnd(1);
+    for (int64_t i = 0; i < FLAGS_cache_size; i++) {
+      uint64_t rand_key = rnd.Next() % FLAGS_max_key;
+      // Cast uint64* to be char*, data would be copied to cache
+      Slice key(reinterpret_cast<char*>(&rand_key), 8);
+      // do insert
+      cache_->Insert(key, new char[10], 1, &deleter);
     }
   }
 
   bool Run() {
-    ROCKSDB_NAMESPACE::Env* env = ROCKSDB_NAMESPACE::Env::Default();
-    const auto& clock = env->GetSystemClock();
+    rocksdb::Env* env = rocksdb::Env::Default();
 
     PrintEnv();
     SharedState shared(this);
-    std::vector<std::unique_ptr<ThreadState> > threads(FLAGS_threads);
-    for (uint32_t i = 0; i < FLAGS_threads; i++) {
-      threads[i].reset(new ThreadState(i, &shared));
-      env->StartThread(ThreadBody, threads[i].get());
+    std::vector<ThreadState*> threads(num_threads_);
+    for (uint32_t i = 0; i < num_threads_; i++) {
+      threads[i] = new ThreadState(i, &shared);
+      env->StartThread(ThreadBody, threads[i]);
     }
     {
       MutexLock l(shared.GetMutex());
@@ -227,7 +172,7 @@ class CacheBench {
         shared.GetCondVar()->Wait();
       }
       // Record start time
-      uint64_t start_time = clock->NowMicros();
+      uint64_t start_time = env->NowMicros();
 
       // Start all threads
       shared.SetStart();
@@ -239,7 +184,7 @@ class CacheBench {
       }
 
       // Record end time
-      uint64_t end_time = clock->NowMicros();
+      uint64_t end_time = env->NowMicros();
       double elapsed = static_cast<double>(end_time - start_time) * 1e-6;
       uint32_t qps = static_cast<uint32_t>(
           static_cast<double>(FLAGS_threads * FLAGS_ops_per_thread) / elapsed);
@@ -250,15 +195,10 @@ class CacheBench {
 
  private:
   std::shared_ptr<Cache> cache_;
-  const uint64_t max_key_;
-  // Cumulative thresholds in the space of a random uint64_t
-  const uint64_t lookup_insert_threshold_;
-  const uint64_t insert_threshold_;
-  const uint64_t lookup_threshold_;
-  const uint64_t erase_threshold_;
+  uint32_t num_threads_;
 
   static void ThreadBody(void* v) {
-    ThreadState* thread = static_cast<ThreadState*>(v);
+    ThreadState* thread = reinterpret_cast<ThreadState*>(v);
     SharedState* shared = thread->shared;
 
     {
@@ -283,82 +223,44 @@ class CacheBench {
   }
 
   void OperateCache(ThreadState* thread) {
-    // To use looked-up values
-    uint64_t result = 0;
-    // To hold handles for a non-trivial amount of time
-    Cache::Handle* handle = nullptr;
-    KeyGen gen;
     for (uint64_t i = 0; i < FLAGS_ops_per_thread; i++) {
-      Slice key = gen.GetRand(thread->rnd, max_key_);
-      uint64_t random_op = thread->rnd.Next();
-      if (random_op < lookup_insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
-        // do lookup
-        handle = cache_->Lookup(key);
-        if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
-        } else {
-          // do insert
-          cache_->Insert(key, createValue(thread->rnd), FLAGS_value_bytes,
-                         &deleter, &handle);
-        }
-      } else if (random_op < insert_threshold_) {
-        if (handle) {
-          cache_->Release(handle);
-          handle = nullptr;
-        }
+      uint64_t rand_key = thread->rnd.Next() % FLAGS_max_key;
+      // Cast uint64* to be char*, data would be copied to cache
+      Slice key(reinterpret_cast<char*>(&rand_key), 8);
+      int32_t prob_op = thread->rnd.Uniform(100);
+      if (prob_op >= 0 && prob_op < FLAGS_insert_percent) {
         // do insert
-        cache_->Insert(key, createValue(thread->rnd), FLAGS_value_bytes,
-                       &deleter, &handle);
-      } else if (random_op < lookup_threshold_) {
+        cache_->Insert(key, new char[10], 1, &deleter);
+      } else if (prob_op -= FLAGS_insert_percent &&
+                 prob_op < FLAGS_lookup_percent) {
+        // do lookup
+        auto handle = cache_->Lookup(key);
         if (handle) {
           cache_->Release(handle);
-          handle = nullptr;
         }
-        // do lookup
-        handle = cache_->Lookup(key);
-        if (handle) {
-          // do something with the data
-          result += NPHash64(static_cast<char*>(cache_->Value(handle)),
-                             FLAGS_value_bytes);
-        }
-      } else if (random_op < erase_threshold_) {
+      } else if (prob_op -= FLAGS_lookup_percent &&
+                 prob_op < FLAGS_erase_percent) {
         // do erase
         cache_->Erase(key);
-      } else {
-        // Should be extremely unlikely (noop)
-        assert(random_op >= kHundredthUint64 * 100U);
       }
     }
-    if (handle) {
-      cache_->Release(handle);
-      handle = nullptr;
-    }
   }
 
   void PrintEnv() const {
     printf("RocksDB version     : %d.%d\n", kMajorVersion, kMinorVersion);
-    printf("Number of threads   : %u\n", FLAGS_threads);
+    printf("Number of threads   : %d\n", FLAGS_threads);
     printf("Ops per thread      : %" PRIu64 "\n", FLAGS_ops_per_thread);
     printf("Cache size          : %" PRIu64 "\n", FLAGS_cache_size);
-    printf("Num shard bits      : %u\n", FLAGS_num_shard_bits);
-    printf("Max key             : %" PRIu64 "\n", max_key_);
-    printf("Resident ratio      : %g\n", FLAGS_resident_ratio);
-    printf("Skew degree         : %u\n", FLAGS_skew);
-    printf("Populate cache      : %d\n", int{FLAGS_populate_cache});
-    printf("Lookup+Insert pct   : %u%%\n", FLAGS_lookup_insert_percent);
-    printf("Insert percentage   : %u%%\n", FLAGS_insert_percent);
-    printf("Lookup percentage   : %u%%\n", FLAGS_lookup_percent);
-    printf("Erase percentage    : %u%%\n", FLAGS_erase_percent);
+    printf("Num shard bits      : %d\n", FLAGS_num_shard_bits);
+    printf("Max key             : %" PRIu64 "\n", FLAGS_max_key);
+    printf("Populate cache      : %d\n", FLAGS_populate_cache);
+    printf("Insert percentage   : %d%%\n", FLAGS_insert_percent);
+    printf("Lookup percentage   : %d%%\n", FLAGS_lookup_percent);
+    printf("Erase percentage    : %d%%\n", FLAGS_erase_percent);
     printf("----------------------------\n");
   }
 };
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 int main(int argc, char** argv) {
   ParseCommandLineFlags(&argc, &argv, true);
@@ -368,11 +270,9 @@ int main(int argc, char** argv) {
     exit(1);
   }
 
-  ROCKSDB_NAMESPACE::CacheBench bench;
+  rocksdb::CacheBench bench;
   if (FLAGS_populate_cache) {
     bench.PopulateCache();
-    printf("Population complete\n");
-    printf("----------------------------\n");
   }
   if (bench.Run()) {
     return 0;

cache/cache_helpers.h

@@ -1,114 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-
-#include "rocksdb/cache.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Returns the cached value given a cache handle.
-template <typename T>
-T* GetFromCacheHandle(Cache* cache, Cache::Handle* handle) {
-  assert(cache);
-  assert(handle);
-
-  return static_cast<T*>(cache->Value(handle));
-}
-
-// Simple generic deleter for Cache (to be used with Cache::Insert).
-template <typename T>
-void DeleteCacheEntry(const Slice& /* key */, void* value) {
-  delete static_cast<T*>(value);
-}
-
-// Turns a T* into a Slice so it can be used as a key with Cache.
-template <typename T>
-Slice GetSlice(const T* t) {
-  return Slice(reinterpret_cast<const char*>(t), sizeof(T));
-}
-
-// Generic resource management object for cache handles that releases the handle
-// when destroyed. Has unique ownership of the handle, so copying it is not
-// allowed, while moving it transfers ownership.
-template <typename T>
-class CacheHandleGuard {
- public:
-  CacheHandleGuard() = default;
-
-  CacheHandleGuard(Cache* cache, Cache::Handle* handle)
-      : cache_(cache),
-        handle_(handle),
-        value_(GetFromCacheHandle<T>(cache, handle)) {
-    assert(cache_ && handle_ && value_);
-  }
-
-  CacheHandleGuard(const CacheHandleGuard&) = delete;
-  CacheHandleGuard& operator=(const CacheHandleGuard&) = delete;
-
-  CacheHandleGuard(CacheHandleGuard&& rhs) noexcept
-      : cache_(rhs.cache_), handle_(rhs.handle_), value_(rhs.value_) {
-    assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_));
-
-    rhs.ResetFields();
-  }
-
-  CacheHandleGuard& operator=(CacheHandleGuard&& rhs) noexcept {
-    if (this == &rhs) {
-      return *this;
-    }
-
-    ReleaseHandle();
-
-    cache_ = rhs.cache_;
-    handle_ = rhs.handle_;
-    value_ = rhs.value_;
-
-    assert((!cache_ && !handle_ && !value_) || (cache_ && handle_ && value_));
-
-    rhs.ResetFields();
-
-    return *this;
-  }
-
-  ~CacheHandleGuard() { ReleaseHandle(); }
-
-  bool IsEmpty() const { return !handle_; }
-
-  Cache* GetCache() const { return cache_; }
-  Cache::Handle* GetCacheHandle() const { return handle_; }
-  T* GetValue() const { return value_; }
-
-  void Reset() {
-    ReleaseHandle();
-    ResetFields();
-  }
-
- private:
-  void ReleaseHandle() {
-    if (IsEmpty()) {
-      return;
-    }
-
-    assert(cache_);
-    cache_->Release(handle_);
-  }
-
-  void ResetFields() {
-    cache_ = nullptr;
-    handle_ = nullptr;
-    value_ = nullptr;
-  }
-
- private:
-  Cache* cache_ = nullptr;
-  Cache::Handle* handle_ = nullptr;
-  T* value_ = nullptr;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

cache/cache_test.cc

@@ -16,11 +16,11 @@
 #include <vector>
 #include "cache/clock_cache.h"
 #include "cache/lru_cache.h"
-#include "test_util/testharness.h"
 #include "util/coding.h"
 #include "util/string_util.h"
+#include "util/testharness.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 // Conversions between numeric keys/values and the types expected by Cache.
 static std::string EncodeKey(int k) {
@@ -40,9 +40,9 @@ static int DecodeValue(void* v) {
 const std::string kLRU = "lru";
 const std::string kClock = "clock";
 
-void dumbDeleter(const Slice& /*key*/, void* /*value*/) {}
+void dumbDeleter(const Slice& key, void* value) {}
 
-void eraseDeleter(const Slice& /*key*/, void* value) {
+void eraseDeleter(const Slice& key, void* value) {
   Cache* cache = reinterpret_cast<Cache*>(value);
   cache->Erase("foo");
 }
@@ -64,8 +64,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
 
   std::vector<int> deleted_keys_;
   std::vector<int> deleted_values_;
-  std::shared_ptr<Cache> cache_;
-  std::shared_ptr<Cache> cache2_;
+  shared_ptr<Cache> cache_;
+  shared_ptr<Cache> cache2_;
 
   CacheTest()
       : cache_(NewCache(kCacheSize, kNumShardBits, false)),
@@ -73,7 +73,8 @@ class CacheTest : public testing::TestWithParam<std::string> {
     current_ = this;
   }
 
-  ~CacheTest() override {}
+  ~CacheTest() {
+  }
 
   std::shared_ptr<Cache> NewCache(size_t capacity) {
     auto type = GetParam();
@@ -86,27 +87,19 @@ class CacheTest : public testing::TestWithParam<std::string> {
     return nullptr;
   }
 
-  std::shared_ptr<Cache> NewCache(
-      size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-      CacheMetadataChargePolicy charge_policy = kDontChargeCacheMetadata) {
+  std::shared_ptr<Cache> NewCache(size_t capacity, int num_shard_bits,
+                                  bool strict_capacity_limit) {
     auto type = GetParam();
     if (type == kLRU) {
-      LRUCacheOptions co;
-      co.capacity = capacity;
-      co.num_shard_bits = num_shard_bits;
-      co.strict_capacity_limit = strict_capacity_limit;
-      co.high_pri_pool_ratio = 0;
-      co.metadata_charge_policy = charge_policy;
-      return NewLRUCache(co);
+      return NewLRUCache(capacity, num_shard_bits, strict_capacity_limit);
     }
     if (type == kClock) {
-      return NewClockCache(capacity, num_shard_bits, strict_capacity_limit,
-                           charge_policy);
+      return NewClockCache(capacity, num_shard_bits, strict_capacity_limit);
     }
     return nullptr;
   }
 
-  int Lookup(std::shared_ptr<Cache> cache, int key) {
+  int Lookup(shared_ptr<Cache> cache, int key) {
     Cache::Handle* handle = cache->Lookup(EncodeKey(key));
     const int r = (handle == nullptr) ? -1 : DecodeValue(cache->Value(handle));
     if (handle != nullptr) {
@@ -115,16 +108,16 @@ class CacheTest : public testing::TestWithParam<std::string> {
     return r;
   }
 
-  void Insert(std::shared_ptr<Cache> cache, int key, int value,
-              int charge = 1) {
-    EXPECT_OK(cache->Insert(EncodeKey(key), EncodeValue(value), charge,
-                            &CacheTest::Deleter));
+  void Insert(shared_ptr<Cache> cache, int key, int value, int charge = 1) {
+    cache->Insert(EncodeKey(key), EncodeValue(value), charge,
+                  &CacheTest::Deleter);
   }
 
-  void Erase(std::shared_ptr<Cache> cache, int key) {
+  void Erase(shared_ptr<Cache> cache, int key) {
     cache->Erase(EncodeKey(key));
   }
 
+
   int Lookup(int key) {
     return Lookup(cache_, key);
   }
@@ -151,15 +144,10 @@ class CacheTest : public testing::TestWithParam<std::string> {
 };
 CacheTest* CacheTest::current_;
 
-class LRUCacheTest : public CacheTest {};
-
 TEST_P(CacheTest, UsageTest) {
-  // cache is std::shared_ptr and will be automatically cleaned up.
+  // cache is shared_ptr and will be automatically cleaned up.
   const uint64_t kCapacity = 100000;
-  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
-  auto precise_cache = NewCache(kCapacity, 0, false, kFullChargeCacheMetadata);
-  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(0, precise_cache->GetUsage());
+  auto cache = NewCache(kCapacity, 8, false);
 
   size_t usage = 0;
   char value[10] = "abcdef";
@@ -167,47 +155,32 @@ TEST_P(CacheTest, UsageTest) {
   for (int i = 1; i < 100; ++i) {
     std::string key(i, 'a');
     auto kv_size = key.size() + 5;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter);
     usage += kv_size;
     ASSERT_EQ(usage, cache->GetUsage());
-    ASSERT_LT(usage, precise_cache->GetUsage());
   }
 
-  cache->EraseUnRefEntries();
-  precise_cache->EraseUnRefEntries();
-  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(0, precise_cache->GetUsage());
-
   // make sure the cache will be overloaded
   for (uint64_t i = 1; i < kCapacity; ++i) {
     auto key = ToString(i);
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                  dumbDeleter);
   }
 
   // the usage should be close to the capacity
   ASSERT_GT(kCapacity, cache->GetUsage());
-  ASSERT_GT(kCapacity, precise_cache->GetUsage());
   ASSERT_LT(kCapacity * 0.95, cache->GetUsage());
-  ASSERT_LT(kCapacity * 0.95, precise_cache->GetUsage());
 }
 
 TEST_P(CacheTest, PinnedUsageTest) {
-  // cache is std::shared_ptr and will be automatically cleaned up.
-  const uint64_t kCapacity = 200000;
-  auto cache = NewCache(kCapacity, 8, false, kDontChargeCacheMetadata);
-  auto precise_cache = NewCache(kCapacity, 8, false, kFullChargeCacheMetadata);
+  // cache is shared_ptr and will be automatically cleaned up.
+  const uint64_t kCapacity = 100000;
+  auto cache = NewCache(kCapacity, 8, false);
 
   size_t pinned_usage = 0;
   char value[10] = "abcdef";
 
   std::forward_list<Cache::Handle*> unreleased_handles;
-  std::forward_list<Cache::Handle*> unreleased_handles_in_precise_cache;
 
   // Add entries. Unpin some of them after insertion. Then, pin some of them
   // again. Check GetPinnedUsage().
@@ -215,73 +188,40 @@ TEST_P(CacheTest, PinnedUsageTest) {
     std::string key(i, 'a');
     auto kv_size = key.size() + 5;
     Cache::Handle* handle;
-    Cache::Handle* handle_in_precise_cache;
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), kv_size,
-                            dumbDeleter, &handle));
-    assert(handle);
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    kv_size, dumbDeleter,
-                                    &handle_in_precise_cache));
-    assert(handle_in_precise_cache);
+    cache->Insert(key, reinterpret_cast<void*>(value), kv_size, dumbDeleter,
+                  &handle);
     pinned_usage += kv_size;
     ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-    ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage());
     if (i % 2 == 0) {
       cache->Release(handle);
-      precise_cache->Release(handle_in_precise_cache);
       pinned_usage -= kv_size;
       ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-      ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage());
     } else {
       unreleased_handles.push_front(handle);
-      unreleased_handles_in_precise_cache.push_front(handle_in_precise_cache);
     }
     if (i % 3 == 0) {
       unreleased_handles.push_front(cache->Lookup(key));
-      auto x = precise_cache->Lookup(key);
-      assert(x);
-      unreleased_handles_in_precise_cache.push_front(x);
       // If i % 2 == 0, then the entry was unpinned before Lookup, so pinned
       // usage increased
       if (i % 2 == 0) {
         pinned_usage += kv_size;
       }
       ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-      ASSERT_LT(pinned_usage, precise_cache->GetPinnedUsage());
     }
   }
-  auto precise_cache_pinned_usage = precise_cache->GetPinnedUsage();
-  ASSERT_LT(pinned_usage, precise_cache_pinned_usage);
 
   // check that overloading the cache does not change the pinned usage
   for (uint64_t i = 1; i < 2 * kCapacity; ++i) {
     auto key = ToString(i);
-    ASSERT_OK(cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
-                            dumbDeleter));
-    ASSERT_OK(precise_cache->Insert(key, reinterpret_cast<void*>(value),
-                                    key.size() + 5, dumbDeleter));
+    cache->Insert(key, reinterpret_cast<void*>(value), key.size() + 5,
+                  dumbDeleter);
   }
   ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-  ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
-
-  cache->EraseUnRefEntries();
-  precise_cache->EraseUnRefEntries();
-  ASSERT_EQ(pinned_usage, cache->GetPinnedUsage());
-  ASSERT_EQ(precise_cache_pinned_usage, precise_cache->GetPinnedUsage());
 
   // release handles for pinned entries to prevent memory leaks
   for (auto handle : unreleased_handles) {
     cache->Release(handle);
   }
-  for (auto handle : unreleased_handles_in_precise_cache) {
-    precise_cache->Release(handle);
-  }
-  ASSERT_EQ(0, cache->GetPinnedUsage());
-  ASSERT_EQ(0, precise_cache->GetPinnedUsage());
-  cache->EraseUnRefEntries();
-  precise_cache->EraseUnRefEntries();
-  ASSERT_EQ(0, cache->GetUsage());
-  ASSERT_EQ(0, precise_cache->GetUsage());
 }
 
 TEST_P(CacheTest, HitAndMiss) {
@@ -367,7 +307,7 @@ TEST_P(CacheTest, EvictionPolicy) {
   Insert(200, 201);
 
   // Frequently used entry must be kept around
-  for (int i = 0; i < kCacheSize * 2; i++) {
+  for (int i = 0; i < kCacheSize + 100; i++) {
     Insert(1000+i, 2000+i);
     ASSERT_EQ(101, Lookup(100));
   }
@@ -420,7 +360,7 @@ TEST_P(CacheTest, EvictionPolicyRef) {
   Insert(303, 104);
 
   // Insert entries much more than Cache capacity
-  for (int i = 0; i < kCacheSize * 2; i++) {
+  for (int i = 0; i < kCacheSize + 100; i++) {
     Insert(1000 + i, 2000 + i);
   }
 
@@ -530,42 +470,11 @@ class Value {
 };
 
 namespace {
-void deleter(const Slice& /*key*/, void* value) {
+void deleter(const Slice& key, void* value) {
   delete static_cast<Value *>(value);
 }
 }  // namespace
 
-TEST_P(CacheTest, ReleaseAndErase) {
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
-  Cache::Handle* handle;
-  Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
-                           &CacheTest::Deleter, &handle);
-  ASSERT_TRUE(s.ok());
-  ASSERT_EQ(5U, cache->GetCapacity());
-  ASSERT_EQ(1U, cache->GetUsage());
-  ASSERT_EQ(0U, deleted_keys_.size());
-  auto erased = cache->Release(handle, true);
-  ASSERT_TRUE(erased);
-  // This tests that deleter has been called
-  ASSERT_EQ(1U, deleted_keys_.size());
-}
-
-TEST_P(CacheTest, ReleaseWithoutErase) {
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
-  Cache::Handle* handle;
-  Status s = cache->Insert(EncodeKey(100), EncodeValue(100), 1,
-                           &CacheTest::Deleter, &handle);
-  ASSERT_TRUE(s.ok());
-  ASSERT_EQ(5U, cache->GetCapacity());
-  ASSERT_EQ(1U, cache->GetUsage());
-  ASSERT_EQ(0U, deleted_keys_.size());
-  auto erased = cache->Release(handle);
-  ASSERT_FALSE(erased);
-  // This tests that deleter is not called. When cache has free capacity it is
-  // not expected to immediately erase the released items.
-  ASSERT_EQ(0U, deleted_keys_.size());
-}
-
 TEST_P(CacheTest, SetCapacity) {
   // test1: increase capacity
   // lets create a cache with capacity 5,
@@ -611,10 +520,10 @@ TEST_P(CacheTest, SetCapacity) {
   }
 }
 
-TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
+TEST_P(CacheTest, SetStrictCapacityLimit) {
   // test1: set the flag to false. Insert more keys than capacity. See if they
   // all go through.
-  std::shared_ptr<Cache> cache = NewCache(5, 0, false);
+  std::shared_ptr<Cache> cache = NewLRUCache(5, 0, false);
   std::vector<Cache::Handle*> handles(10);
   Status s;
   for (size_t i = 0; i < 10; i++) {
@@ -623,7 +532,6 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
     ASSERT_OK(s);
     ASSERT_NE(nullptr, handles[i]);
   }
-  ASSERT_EQ(10, cache->GetUsage());
 
   // test2: set the flag to true. Insert and check if it fails.
   std::string extra_key = "extra";
@@ -633,14 +541,13 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   s = cache->Insert(extra_key, extra_value, 1, &deleter, &handle);
   ASSERT_TRUE(s.IsIncomplete());
   ASSERT_EQ(nullptr, handle);
-  ASSERT_EQ(10, cache->GetUsage());
 
   for (size_t i = 0; i < 10; i++) {
     cache->Release(handles[i]);
   }
 
   // test3: init with flag being true.
-  std::shared_ptr<Cache> cache2 = NewCache(5, 0, true);
+  std::shared_ptr<Cache> cache2 = NewLRUCache(5, 0, true);
   for (size_t i = 0; i < 5; i++) {
     std::string key = ToString(i + 1);
     s = cache2->Insert(key, new Value(i + 1), 1, &deleter, &handles[i]);
@@ -654,7 +561,7 @@ TEST_P(LRUCacheTest, SetStrictCapacityLimit) {
   s = cache2->Insert(extra_key, extra_value, 1, &deleter);
   // AS if the key have been inserted into cache but get evicted immediately.
   ASSERT_OK(s);
-  ASSERT_EQ(5, cache2->GetUsage());
+  ASSERT_EQ(5, cache->GetUsage());
   ASSERT_EQ(nullptr, cache2->Lookup(extra_key));
 
   for (size_t i = 0; i < 5; i++) {
@@ -749,25 +656,15 @@ TEST_P(CacheTest, DefaultShardBits) {
   ASSERT_EQ(6, sc->GetNumShardBits());
 }
 
-TEST_P(CacheTest, GetCharge) {
-  Insert(1, 2);
-  Cache::Handle* h1 = cache_->Lookup(EncodeKey(1));
-  ASSERT_EQ(2, DecodeValue(cache_->Value(h1)));
-  ASSERT_EQ(1, cache_->GetCharge(h1));
-  cache_->Release(h1);
-}
-
 #ifdef SUPPORT_CLOCK_CACHE
-std::shared_ptr<Cache> (*new_clock_cache_func)(
-    size_t, int, bool, CacheMetadataChargePolicy) = NewClockCache;
+shared_ptr<Cache> (*new_clock_cache_func)(size_t, int, bool) = NewClockCache;
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest,
                         testing::Values(kLRU, kClock));
 #else
 INSTANTIATE_TEST_CASE_P(CacheTestInstance, CacheTest, testing::Values(kLRU));
 #endif  // SUPPORT_CLOCK_CACHE
-INSTANTIATE_TEST_CASE_P(CacheTestInstance, LRUCacheTest, testing::Values(kLRU));
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);

cache/clock_cache.cc

@@ -11,37 +11,31 @@
 
 #ifndef SUPPORT_CLOCK_CACHE
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
-std::shared_ptr<Cache> NewClockCache(
-    size_t /*capacity*/, int /*num_shard_bits*/, bool /*strict_capacity_limit*/,
-    CacheMetadataChargePolicy /*metadata_charge_policy*/) {
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+                                     bool strict_capacity_limit) {
   // Clock cache not supported.
   return nullptr;
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 #else
 
 #include <assert.h>
 #include <atomic>
 #include <deque>
+#include <limits>
 
-// "tbb/concurrent_hash_map.h" requires RTTI if exception is enabled.
-// Disable it so users can chooose to disable RTTI.
-#ifndef ROCKSDB_USE_RTTI
-#define TBB_USE_EXCEPTIONS 0
-#endif
 #include "tbb/concurrent_hash_map.h"
 
 #include "cache/sharded_cache.h"
-#include "port/malloc.h"
 #include "port/port.h"
 #include "util/autovector.h"
 #include "util/mutexlock.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 namespace {
 
@@ -182,7 +176,7 @@ struct CacheHandle {
   void (*deleter)(const Slice&, void* value);
 
   // Flags and counters associated with the cache handle:
-  //   lowest bit: in-cache bit
+  //   lowest bit: n-cache bit
   //   second lowest bit: usage bit
   //   the rest bits: reference count
   // The handle is unused when flags equals to 0. The thread decreases the count
@@ -204,27 +198,6 @@ struct CacheHandle {
     deleter = a.deleter;
     return *this;
   }
-
-  inline static size_t CalcTotalCharge(
-      Slice key, size_t charge,
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    size_t meta_charge = 0;
-    if (metadata_charge_policy == kFullChargeCacheMetadata) {
-      meta_charge += sizeof(CacheHandle);
-#ifdef ROCKSDB_MALLOC_USABLE_SIZE
-      meta_charge +=
-          malloc_usable_size(static_cast<void*>(const_cast<char*>(key.data())));
-#else
-      meta_charge += key.size();
-#endif
-    }
-    return charge + meta_charge;
-  }
-
-  inline size_t CalcTotalCharge(
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    return CalcTotalCharge(key, charge, metadata_charge_policy);
-  }
 };
 
 // Key of hash map. We store hash value with the key for convenience.
@@ -257,35 +230,35 @@ struct CleanupContext {
 };
 
 // A cache shard which maintains its own CLOCK cache.
-class ClockCacheShard final : public CacheShard {
+class ClockCacheShard : public CacheShard {
  public:
   // Hash map type.
   typedef tbb::concurrent_hash_map<CacheKey, CacheHandle*, CacheKey> HashTable;
 
   ClockCacheShard();
-  ~ClockCacheShard() override;
+  ~ClockCacheShard();
 
   // Interfaces
-  void SetCapacity(size_t capacity) override;
-  void SetStrictCapacityLimit(bool strict_capacity_limit) override;
-  Status Insert(const Slice& key, uint32_t hash, void* value, size_t charge,
-                void (*deleter)(const Slice& key, void* value),
-                Cache::Handle** handle, Cache::Priority priority) override;
-  Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
+  virtual void SetCapacity(size_t capacity) override;
+  virtual void SetStrictCapacityLimit(bool strict_capacity_limit) override;
+  virtual Status Insert(const Slice& key, uint32_t hash, void* value,
+                        size_t charge,
+                        void (*deleter)(const Slice& key, void* value),
+                        Cache::Handle** handle,
+                        Cache::Priority priority) override;
+  virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
   // If the entry in in cache, increase reference count and return true.
   // Return false otherwise.
   //
   // Not necessary to hold mutex_ before being called.
-  bool Ref(Cache::Handle* handle) override;
-  bool Release(Cache::Handle* handle, bool force_erase = false) override;
-  void Erase(const Slice& key, uint32_t hash) override;
-  bool EraseAndConfirm(const Slice& key, uint32_t hash,
-                       CleanupContext* context);
-  size_t GetUsage() const override;
-  size_t GetPinnedUsage() const override;
-  void EraseUnRefEntries() override;
-  void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
-                              bool thread_safe) override;
+  virtual bool Ref(Cache::Handle* handle) override;
+  virtual void Release(Cache::Handle* handle) override;
+  virtual void Erase(const Slice& key, uint32_t hash) override;
+  virtual size_t GetUsage() const override;
+  virtual size_t GetPinnedUsage() const override;
+  virtual void EraseUnRefEntries() override;
+  virtual void ApplyToAllCacheEntries(void (*callback)(void*, size_t),
+                                      bool thread_safe) override;
 
  private:
   static const uint32_t kInCacheBit = 1;
@@ -301,17 +274,13 @@ class ClockCacheShard final : public CacheShard {
   // Decrease reference count of the entry. If this decreases the count to 0,
   // recycle the entry. If set_usage is true, also set the usage bit.
   //
-  // returns true if a value is erased.
-  //
   // Not necessary to hold mutex_ before being called.
-  bool Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
+  void Unref(CacheHandle* handle, bool set_usage, CleanupContext* context);
 
   // Unset in-cache bit of the entry. Recycle the handle if necessary.
   //
-  // returns true if a value is erased.
-  //
   // Has to hold mutex_ before being called.
-  bool UnsetInCache(CacheHandle* handle, CleanupContext* context);
+  void UnsetInCache(CacheHandle* handle, CleanupContext* context);
 
   // Put the handle back to recycle_ list, and put the value associated with
   // it into to-be-deleted list. It doesn't cleanup the key as it might be
@@ -341,8 +310,7 @@ class ClockCacheShard final : public CacheShard {
   CacheHandle* Insert(const Slice& key, uint32_t hash, void* value,
                       size_t change,
                       void (*deleter)(const Slice& key, void* value),
-                      bool hold_reference, CleanupContext* context,
-                      bool* overwritten);
+                      bool hold_reference, CleanupContext* context);
 
   // Guards list_, head_, and recycle_. In addition, updating table_ also has
   // to hold the mutex, to avoid the cache being in inconsistent state.
@@ -388,9 +356,7 @@ ClockCacheShard::~ClockCacheShard() {
   for (auto& handle : list_) {
     uint32_t flags = handle.flags.load(std::memory_order_relaxed);
     if (InCache(flags) || CountRefs(flags) > 0) {
-      if (handle.deleter != nullptr) {
-        (*handle.deleter)(handle.key, handle.value);
-      }
+      (*handle.deleter)(handle.key, handle.value);
       delete[] handle.key.data();
     }
   }
@@ -428,12 +394,11 @@ void ClockCacheShard::RecycleHandle(CacheHandle* handle,
   assert(!InCache(handle->flags) && CountRefs(handle->flags) == 0);
   context->to_delete_key.push_back(handle->key.data());
   context->to_delete_value.emplace_back(*handle);
-  size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
   handle->key.clear();
   handle->value = nullptr;
   handle->deleter = nullptr;
   recycle_.push_back(handle);
-  usage_.fetch_sub(total_charge, std::memory_order_relaxed);
+  usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
 }
 
 void ClockCacheShard::Cleanup(const CleanupContext& context) {
@@ -459,8 +424,7 @@ bool ClockCacheShard::Ref(Cache::Handle* h) {
                                             std::memory_order_relaxed)) {
       if (CountRefs(flags) == 0) {
         // No reference count before the operation.
-        size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
-        pinned_usage_.fetch_add(total_charge, std::memory_order_relaxed);
+        pinned_usage_.fetch_add(handle->charge, std::memory_order_relaxed);
       }
       return true;
     }
@@ -468,7 +432,7 @@ bool ClockCacheShard::Ref(Cache::Handle* h) {
   return false;
 }
 
-bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
+void ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
                             CleanupContext* context) {
   if (set_usage) {
     handle->flags.fetch_or(kUsageBit, std::memory_order_relaxed);
@@ -480,18 +444,16 @@ bool ClockCacheShard::Unref(CacheHandle* handle, bool set_usage,
   assert(CountRefs(flags) > 0);
   if (CountRefs(flags) == 1) {
     // this is the last reference.
-    size_t total_charge = handle->CalcTotalCharge(metadata_charge_policy_);
-    pinned_usage_.fetch_sub(total_charge, std::memory_order_relaxed);
+    pinned_usage_.fetch_sub(handle->charge, std::memory_order_relaxed);
     // Cleanup if it is the last reference.
     if (!InCache(flags)) {
       MutexLock l(&mutex_);
       RecycleHandle(handle, context);
     }
   }
-  return context->to_delete_value.size();
 }
 
-bool ClockCacheShard::UnsetInCache(CacheHandle* handle,
+void ClockCacheShard::UnsetInCache(CacheHandle* handle,
                                    CleanupContext* context) {
   mutex_.AssertHeld();
   // Use acquire-release semantics as previous operations on the cache entry
@@ -503,7 +465,6 @@ bool ClockCacheShard::UnsetInCache(CacheHandle* handle,
   if (InCache(flags) && CountRefs(flags) == 0) {
     RecycleHandle(handle, context);
   }
-  return context->to_delete_value.size();
 }
 
 bool ClockCacheShard::TryEvict(CacheHandle* handle, CleanupContext* context) {
@@ -565,12 +526,9 @@ void ClockCacheShard::SetStrictCapacityLimit(bool strict_capacity_limit) {
 CacheHandle* ClockCacheShard::Insert(
     const Slice& key, uint32_t hash, void* value, size_t charge,
     void (*deleter)(const Slice& key, void* value), bool hold_reference,
-    CleanupContext* context, bool* overwritten) {
-  assert(overwritten != nullptr && *overwritten == false);
-  size_t total_charge =
-      CacheHandle::CalcTotalCharge(key, charge, metadata_charge_policy_);
+    CleanupContext* context) {
   MutexLock l(&mutex_);
-  bool success = EvictFromCache(total_charge, context);
+  bool success = EvictFromCache(charge, context);
   bool strict = strict_capacity_limit_.load(std::memory_order_relaxed);
   if (!success && (strict || !hold_reference)) {
     context->to_delete_key.push_back(key.data());
@@ -599,16 +557,15 @@ CacheHandle* ClockCacheShard::Insert(
   handle->flags.store(flags, std::memory_order_relaxed);
   HashTable::accessor accessor;
   if (table_.find(accessor, CacheKey(key, hash))) {
-    *overwritten = true;
     CacheHandle* existing_handle = accessor->second;
     table_.erase(accessor);
     UnsetInCache(existing_handle, context);
   }
   table_.insert(HashTable::value_type(CacheKey(key, hash), handle));
   if (hold_reference) {
-    pinned_usage_.fetch_add(total_charge, std::memory_order_relaxed);
+    pinned_usage_.fetch_add(charge, std::memory_order_relaxed);
   }
-  usage_.fetch_add(total_charge, std::memory_order_relaxed);
+  usage_.fetch_add(charge, std::memory_order_relaxed);
   return handle;
 }
 
@@ -616,15 +573,14 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
                                size_t charge,
                                void (*deleter)(const Slice& key, void* value),
                                Cache::Handle** out_handle,
-                               Cache::Priority /*priority*/) {
+                               Cache::Priority priority) {
   CleanupContext context;
   HashTable::accessor accessor;
   char* key_data = new char[key.size()];
   memcpy(key_data, key.data(), key.size());
   Slice key_copy(key_data, key.size());
-  bool overwritten = false;
   CacheHandle* handle = Insert(key_copy, hash, value, charge, deleter,
-                               out_handle != nullptr, &context, &overwritten);
+                               out_handle != nullptr, &context);
   Status s;
   if (out_handle != nullptr) {
     if (handle == nullptr) {
@@ -633,10 +589,6 @@ Status ClockCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
       *out_handle = reinterpret_cast<Cache::Handle*>(handle);
     }
   }
-  if (overwritten) {
-    assert(s.ok());
-    s = Status::OkOverwritten();
-  }
   Cleanup(context);
   return s;
 }
@@ -666,34 +618,25 @@ Cache::Handle* ClockCacheShard::Lookup(const Slice& key, uint32_t hash) {
   return reinterpret_cast<Cache::Handle*>(handle);
 }
 
-bool ClockCacheShard::Release(Cache::Handle* h, bool force_erase) {
+void ClockCacheShard::Release(Cache::Handle* h) {
   CleanupContext context;
   CacheHandle* handle = reinterpret_cast<CacheHandle*>(h);
-  bool erased = Unref(handle, true, &context);
-  if (force_erase && !erased) {
-    erased = EraseAndConfirm(handle->key, handle->hash, &context);
-  }
+  Unref(handle, true, &context);
   Cleanup(context);
-  return erased;
 }
 
 void ClockCacheShard::Erase(const Slice& key, uint32_t hash) {
   CleanupContext context;
-  EraseAndConfirm(key, hash, &context);
-  Cleanup(context);
-}
-
-bool ClockCacheShard::EraseAndConfirm(const Slice& key, uint32_t hash,
-                                      CleanupContext* context) {
-  MutexLock l(&mutex_);
-  HashTable::accessor accessor;
-  bool erased = false;
-  if (table_.find(accessor, CacheKey(key, hash))) {
-    CacheHandle* handle = accessor->second;
-    table_.erase(accessor);
-    erased = UnsetInCache(handle, context);
+  {
+    MutexLock l(&mutex_);
+    HashTable::accessor accessor;
+    if (table_.find(accessor, CacheKey(key, hash))) {
+      CacheHandle* handle = accessor->second;
+      table_.erase(accessor);
+      UnsetInCache(handle, &context);
+    }
   }
-  return erased;
+  Cleanup(context);
 }
 
 void ClockCacheShard::EraseUnRefEntries() {
@@ -708,45 +651,41 @@ void ClockCacheShard::EraseUnRefEntries() {
   Cleanup(context);
 }
 
-class ClockCache final : public ShardedCache {
+class ClockCache : public ShardedCache {
  public:
-  ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-             CacheMetadataChargePolicy metadata_charge_policy)
+  ClockCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit)
       : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
     int num_shards = 1 << num_shard_bits;
     shards_ = new ClockCacheShard[num_shards];
-    for (int i = 0; i < num_shards; i++) {
-      shards_[i].set_metadata_charge_policy(metadata_charge_policy);
-    }
     SetCapacity(capacity);
     SetStrictCapacityLimit(strict_capacity_limit);
   }
 
-  ~ClockCache() override { delete[] shards_; }
+  virtual ~ClockCache() { delete[] shards_; }
 
-  const char* Name() const override { return "ClockCache"; }
+  virtual const char* Name() const override { return "ClockCache"; }
 
-  CacheShard* GetShard(int shard) override {
+  virtual CacheShard* GetShard(int shard) override {
     return reinterpret_cast<CacheShard*>(&shards_[shard]);
   }
 
-  const CacheShard* GetShard(int shard) const override {
+  virtual const CacheShard* GetShard(int shard) const override {
     return reinterpret_cast<CacheShard*>(&shards_[shard]);
   }
 
-  void* Value(Handle* handle) override {
+  virtual void* Value(Handle* handle) override {
     return reinterpret_cast<const CacheHandle*>(handle)->value;
   }
 
-  size_t GetCharge(Handle* handle) const override {
+  virtual size_t GetCharge(Handle* handle) const override {
     return reinterpret_cast<const CacheHandle*>(handle)->charge;
   }
 
-  uint32_t GetHash(Handle* handle) const override {
+  virtual uint32_t GetHash(Handle* handle) const override {
     return reinterpret_cast<const CacheHandle*>(handle)->hash;
   }
 
-  void DisownData() override { shards_ = nullptr; }
+  virtual void DisownData() override { shards_ = nullptr; }
 
  private:
   ClockCacheShard* shards_;
@@ -754,16 +693,15 @@ class ClockCache final : public ShardedCache {
 
 }  // end anonymous namespace
 
-std::shared_ptr<Cache> NewClockCache(
-    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    CacheMetadataChargePolicy metadata_charge_policy) {
+std::shared_ptr<Cache> NewClockCache(size_t capacity, int num_shard_bits,
+                                     bool strict_capacity_limit) {
   if (num_shard_bits < 0) {
     num_shard_bits = GetDefaultCacheShardBits(capacity);
   }
-  return std::make_shared<ClockCache>(
-      capacity, num_shard_bits, strict_capacity_limit, metadata_charge_policy);
+  return std::make_shared<ClockCache>(capacity, num_shard_bits,
+                                      strict_capacity_limit);
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 #endif  // SUPPORT_CLOCK_CACHE

cache/lru_cache.cc

@@ -7,23 +7,28 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
 #include "cache/lru_cache.h"
 
-#include <cassert>
-#include <cstdio>
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
 #include <string>
 
 #include "util/mutexlock.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
-LRUHandleTable::LRUHandleTable() : list_(nullptr), length_(0), elems_(0) {
+LRUHandleTable::LRUHandleTable() : length_(0), elems_(0), list_(nullptr) {
   Resize();
 }
 
 LRUHandleTable::~LRUHandleTable() {
   ApplyToAllCacheEntries([](LRUHandle* h) {
-    if (!h->HasRefs()) {
+    if (h->refs == 1) {
       h->Free();
     }
   });
@@ -94,40 +99,38 @@ void LRUHandleTable::Resize() {
   length_ = new_length;
 }
 
-LRUCacheShard::LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                             double high_pri_pool_ratio,
-                             bool use_adaptive_mutex,
-                             CacheMetadataChargePolicy metadata_charge_policy)
-    : capacity_(0),
-      high_pri_pool_usage_(0),
-      strict_capacity_limit_(strict_capacity_limit),
-      high_pri_pool_ratio_(high_pri_pool_ratio),
-      high_pri_pool_capacity_(0),
-      usage_(0),
-      lru_usage_(0),
-      mutex_(use_adaptive_mutex) {
-  set_metadata_charge_policy(metadata_charge_policy);
+LRUCacheShard::LRUCacheShard()
+    : usage_(0), lru_usage_(0), high_pri_pool_usage_(0) {
   // Make empty circular linked list
   lru_.next = &lru_;
   lru_.prev = &lru_;
   lru_low_pri_ = &lru_;
-  SetCapacity(capacity);
 }
 
+LRUCacheShard::~LRUCacheShard() {}
+
+bool LRUCacheShard::Unref(LRUHandle* e) {
+  assert(e->refs > 0);
+  e->refs--;
+  return e->refs == 0;
+}
+
+// Call deleter and free
+
 void LRUCacheShard::EraseUnRefEntries() {
   autovector<LRUHandle*> last_reference_list;
   {
     MutexLock l(&mutex_);
     while (lru_.next != &lru_) {
       LRUHandle* old = lru_.next;
-      // LRU list contains only elements which can be evicted
-      assert(old->InCache() && !old->HasRefs());
+      assert(old->InCache());
+      assert(old->refs ==
+             1);  // LRU list contains elements which may be evicted
       LRU_Remove(old);
       table_.Remove(old->key(), old->hash);
       old->SetInCache(false);
-      size_t total_charge = old->CalcTotalCharge(metadata_charge_policy_);
-      assert(usage_ >= total_charge);
-      usage_ -= total_charge;
+      Unref(old);
+      usage_ -= old->charge;
       last_reference_list.push_back(old);
     }
   }
@@ -139,41 +142,21 @@ void LRUCacheShard::EraseUnRefEntries() {
 
 void LRUCacheShard::ApplyToAllCacheEntries(void (*callback)(void*, size_t),
                                            bool thread_safe) {
-  const auto applyCallback = [&]() {
-    table_.ApplyToAllCacheEntries(
-        [callback](LRUHandle* h) { callback(h->value, h->charge); });
-  };
-
   if (thread_safe) {
-    MutexLock l(&mutex_);
-    applyCallback();
-  } else {
-    applyCallback();
+    mutex_.Lock();
+  }
+  table_.ApplyToAllCacheEntries(
+      [callback](LRUHandle* h) { callback(h->value, h->charge); });
+  if (thread_safe) {
+    mutex_.Unlock();
   }
 }
 
 void LRUCacheShard::TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri) {
-  MutexLock l(&mutex_);
   *lru = &lru_;
   *lru_low_pri = lru_low_pri_;
 }
 
-size_t LRUCacheShard::TEST_GetLRUSize() {
-  MutexLock l(&mutex_);
-  LRUHandle* lru_handle = lru_.next;
-  size_t lru_size = 0;
-  while (lru_handle != &lru_) {
-    lru_size++;
-    lru_handle = lru_handle->next;
-  }
-  return lru_size;
-}
-
-double LRUCacheShard::GetHighPriPoolRatio() {
-  MutexLock l(&mutex_);
-  return high_pri_pool_ratio_;
-}
-
 void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   assert(e->next != nullptr);
   assert(e->prev != nullptr);
@@ -183,27 +166,24 @@ void LRUCacheShard::LRU_Remove(LRUHandle* e) {
   e->next->prev = e->prev;
   e->prev->next = e->next;
   e->prev = e->next = nullptr;
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-  assert(lru_usage_ >= total_charge);
-  lru_usage_ -= total_charge;
+  lru_usage_ -= e->charge;
   if (e->InHighPriPool()) {
-    assert(high_pri_pool_usage_ >= total_charge);
-    high_pri_pool_usage_ -= total_charge;
+    assert(high_pri_pool_usage_ >= e->charge);
+    high_pri_pool_usage_ -= e->charge;
   }
 }
 
 void LRUCacheShard::LRU_Insert(LRUHandle* e) {
   assert(e->next == nullptr);
   assert(e->prev == nullptr);
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-  if (high_pri_pool_ratio_ > 0 && (e->IsHighPri() || e->HasHit())) {
+  if (high_pri_pool_ratio_ > 0 && e->IsHighPri()) {
     // Inset "e" to head of LRU list.
     e->next = &lru_;
     e->prev = lru_.prev;
     e->prev->next = e;
     e->next->prev = e;
     e->SetInHighPriPool(true);
-    high_pri_pool_usage_ += total_charge;
+    high_pri_pool_usage_ += e->charge;
     MaintainPoolSize();
   } else {
     // Insert "e" to the head of low-pri pool. Note that when
@@ -215,7 +195,7 @@ void LRUCacheShard::LRU_Insert(LRUHandle* e) {
     e->SetInHighPriPool(false);
     lru_low_pri_ = e;
   }
-  lru_usage_ += total_charge;
+  lru_usage_ += e->charge;
 }
 
 void LRUCacheShard::MaintainPoolSize() {
@@ -224,25 +204,21 @@ void LRUCacheShard::MaintainPoolSize() {
     lru_low_pri_ = lru_low_pri_->next;
     assert(lru_low_pri_ != &lru_);
     lru_low_pri_->SetInHighPriPool(false);
-    size_t total_charge =
-        lru_low_pri_->CalcTotalCharge(metadata_charge_policy_);
-    assert(high_pri_pool_usage_ >= total_charge);
-    high_pri_pool_usage_ -= total_charge;
+    high_pri_pool_usage_ -= lru_low_pri_->charge;
   }
 }
 
 void LRUCacheShard::EvictFromLRU(size_t charge,
                                  autovector<LRUHandle*>* deleted) {
-  while ((usage_ + charge) > capacity_ && lru_.next != &lru_) {
+  while (usage_ + charge > capacity_ && lru_.next != &lru_) {
     LRUHandle* old = lru_.next;
-    // LRU list contains only elements which can be evicted
-    assert(old->InCache() && !old->HasRefs());
+    assert(old->InCache());
+    assert(old->refs == 1);  // LRU list contains elements which may be evicted
     LRU_Remove(old);
     table_.Remove(old->key(), old->hash);
     old->SetInCache(false);
-    size_t old_total_charge = old->CalcTotalCharge(metadata_charge_policy_);
-    assert(usage_ >= old_total_charge);
-    usage_ -= old_total_charge;
+    Unref(old);
+    usage_ -= old->charge;
     deleted->push_back(old);
   }
 }
@@ -255,8 +231,8 @@ void LRUCacheShard::SetCapacity(size_t capacity) {
     high_pri_pool_capacity_ = capacity_ * high_pri_pool_ratio_;
     EvictFromLRU(0, &last_reference_list);
   }
-
-  // Free the entries outside of mutex for performance reasons
+  // we free the entries here outside of mutex for
+  // performance reasons
   for (auto entry : last_reference_list) {
     entry->Free();
   }
@@ -272,22 +248,21 @@ Cache::Handle* LRUCacheShard::Lookup(const Slice& key, uint32_t hash) {
   LRUHandle* e = table_.Lookup(key, hash);
   if (e != nullptr) {
     assert(e->InCache());
-    if (!e->HasRefs()) {
-      // The entry is in LRU since it's in hash and has no external references
+    if (e->refs == 1) {
       LRU_Remove(e);
     }
-    e->Ref();
-    e->SetHit();
+    e->refs++;
   }
   return reinterpret_cast<Cache::Handle*>(e);
 }
 
 bool LRUCacheShard::Ref(Cache::Handle* h) {
-  LRUHandle* e = reinterpret_cast<LRUHandle*>(h);
+  LRUHandle* handle = reinterpret_cast<LRUHandle*>(h);
   MutexLock l(&mutex_);
-  // To create another reference - entry must be already externally referenced
-  assert(e->HasRefs());
-  e->Ref();
+  if (handle->InCache() && handle->refs == 1) {
+    LRU_Remove(handle);
+  }
+  handle->refs++;
   return true;
 }
 
@@ -298,41 +273,41 @@ void LRUCacheShard::SetHighPriorityPoolRatio(double high_pri_pool_ratio) {
   MaintainPoolSize();
 }
 
-bool LRUCacheShard::Release(Cache::Handle* handle, bool force_erase) {
+void LRUCacheShard::Release(Cache::Handle* handle) {
   if (handle == nullptr) {
-    return false;
+    return;
   }
   LRUHandle* e = reinterpret_cast<LRUHandle*>(handle);
   bool last_reference = false;
   {
     MutexLock l(&mutex_);
-    last_reference = e->Unref();
-    if (last_reference && e->InCache()) {
+    last_reference = Unref(e);
+    if (last_reference) {
+      usage_ -= e->charge;
+    }
+    if (e->refs == 1 && e->InCache()) {
       // The item is still in cache, and nobody else holds a reference to it
-      if (usage_ > capacity_ || force_erase) {
+      if (usage_ > capacity_) {
+        // the cache is full
         // The LRU list must be empty since the cache is full
-        assert(lru_.next == &lru_ || force_erase);
-        // Take this opportunity and remove the item
+        assert(lru_.next == &lru_);
+        // take this opportunity and remove the item
         table_.Remove(e->key(), e->hash);
         e->SetInCache(false);
+        Unref(e);
+        usage_ -= e->charge;
+        last_reference = true;
       } else {
-        // Put the item back on the LRU list, and don't free it
+        // put the item on the list to be potentially freed
         LRU_Insert(e);
-        last_reference = false;
       }
     }
-    if (last_reference) {
-      size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-      assert(usage_ >= total_charge);
-      usage_ -= total_charge;
-    }
   }
 
-  // Free the entry here outside of mutex for performance reasons
+  // free outside of mutex
   if (last_reference) {
     e->Free();
   }
-  return last_reference;
 }
 
 Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
@@ -344,35 +319,34 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
   // It shouldn't happen very often though.
   LRUHandle* e = reinterpret_cast<LRUHandle*>(
       new char[sizeof(LRUHandle) - 1 + key.size()]);
-  Status s = Status::OK();
+  Status s;
   autovector<LRUHandle*> last_reference_list;
 
   e->value = value;
   e->deleter = deleter;
   e->charge = charge;
   e->key_length = key.size();
-  e->flags = 0;
   e->hash = hash;
-  e->refs = 0;
+  e->refs = (handle == nullptr
+                 ? 1
+                 : 2);  // One from LRUCache, one for the returned handle
   e->next = e->prev = nullptr;
   e->SetInCache(true);
   e->SetPriority(priority);
   memcpy(e->key_data, key.data(), key.size());
-  size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
 
   {
     MutexLock l(&mutex_);
 
     // Free the space following strict LRU policy until enough space
     // is freed or the lru list is empty
-    EvictFromLRU(total_charge, &last_reference_list);
+    EvictFromLRU(charge, &last_reference_list);
 
-    if ((usage_ + total_charge) > capacity_ &&
+    if (usage_ - lru_usage_ + charge > capacity_ &&
         (strict_capacity_limit_ || handle == nullptr)) {
       if (handle == nullptr) {
         // Don't insert the entry but still return ok, as if the entry inserted
         // into cache and get evicted immediately.
-        e->SetInCache(false);
         last_reference_list.push_back(e);
       } else {
         delete[] reinterpret_cast<char*>(e);
@@ -380,34 +354,32 @@ Status LRUCacheShard::Insert(const Slice& key, uint32_t hash, void* value,
         s = Status::Incomplete("Insert failed due to LRU cache being full.");
       }
     } else {
-      // Insert into the cache. Note that the cache might get larger than its
-      // capacity if not enough space was freed up.
+      // insert into the cache
+      // note that the cache might get larger than its capacity if not enough
+      // space was freed
       LRUHandle* old = table_.Insert(e);
-      usage_ += total_charge;
+      usage_ += e->charge;
       if (old != nullptr) {
-        s = Status::OkOverwritten();
-        assert(old->InCache());
         old->SetInCache(false);
-        if (!old->HasRefs()) {
-          // old is on LRU because it's in cache and its reference count is 0
+        if (Unref(old)) {
+          usage_ -= old->charge;
+          // old is on LRU because it's in cache and its reference count
+          // was just 1 (Unref returned 0)
           LRU_Remove(old);
-          size_t old_total_charge =
-              old->CalcTotalCharge(metadata_charge_policy_);
-          assert(usage_ >= old_total_charge);
-          usage_ -= old_total_charge;
           last_reference_list.push_back(old);
         }
       }
       if (handle == nullptr) {
         LRU_Insert(e);
       } else {
-        e->Ref();
         *handle = reinterpret_cast<Cache::Handle*>(e);
       }
+      s = Status::OK();
     }
   }
 
-  // Free the entries here outside of mutex for performance reasons
+  // we free the entries here outside of mutex for
+  // performance reasons
   for (auto entry : last_reference_list) {
     entry->Free();
   }
@@ -422,20 +394,18 @@ void LRUCacheShard::Erase(const Slice& key, uint32_t hash) {
     MutexLock l(&mutex_);
     e = table_.Remove(key, hash);
     if (e != nullptr) {
-      assert(e->InCache());
-      e->SetInCache(false);
-      if (!e->HasRefs()) {
-        // The entry is in LRU since it's in hash and has no external references
-        LRU_Remove(e);
-        size_t total_charge = e->CalcTotalCharge(metadata_charge_policy_);
-        assert(usage_ >= total_charge);
-        usage_ -= total_charge;
-        last_reference = true;
+      last_reference = Unref(e);
+      if (last_reference) {
+        usage_ -= e->charge;
       }
+      if (last_reference && e->InCache()) {
+        LRU_Remove(e);
+      }
+      e->SetInCache(false);
     }
   }
 
-  // Free the entry here outside of mutex for performance reasons
+  // mutex not held here
   // last_reference will only be true if e != nullptr
   if (last_reference) {
     e->Free();
@@ -465,32 +435,18 @@ std::string LRUCacheShard::GetPrintableOptions() const {
 }
 
 LRUCache::LRUCache(size_t capacity, int num_shard_bits,
-                   bool strict_capacity_limit, double high_pri_pool_ratio,
-                   std::shared_ptr<MemoryAllocator> allocator,
-                   bool use_adaptive_mutex,
-                   CacheMetadataChargePolicy metadata_charge_policy)
-    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit,
-                   std::move(allocator)) {
-  num_shards_ = 1 << num_shard_bits;
-  shards_ = reinterpret_cast<LRUCacheShard*>(
-      port::cacheline_aligned_alloc(sizeof(LRUCacheShard) * num_shards_));
-  size_t per_shard = (capacity + (num_shards_ - 1)) / num_shards_;
-  for (int i = 0; i < num_shards_; i++) {
-    new (&shards_[i])
-        LRUCacheShard(per_shard, strict_capacity_limit, high_pri_pool_ratio,
-                      use_adaptive_mutex, metadata_charge_policy);
+                   bool strict_capacity_limit, double high_pri_pool_ratio)
+    : ShardedCache(capacity, num_shard_bits, strict_capacity_limit) {
+  int num_shards = 1 << num_shard_bits;
+  shards_ = new LRUCacheShard[num_shards];
+  SetCapacity(capacity);
+  SetStrictCapacityLimit(strict_capacity_limit);
+  for (int i = 0; i < num_shards; i++) {
+    shards_[i].SetHighPriorityPoolRatio(high_pri_pool_ratio);
   }
 }
 
-LRUCache::~LRUCache() {
-  if (shards_ != nullptr) {
-    assert(num_shards_ > 0);
-    for (int i = 0; i < num_shards_; i++) {
-      shards_[i].~LRUCacheShard();
-    }
-    port::cacheline_aligned_free(shards_);
-  }
-}
+LRUCache::~LRUCache() { delete[] shards_; }
 
 CacheShard* LRUCache::GetShard(int shard) {
   return reinterpret_cast<CacheShard*>(&shards_[shard]);
@@ -512,50 +468,11 @@ uint32_t LRUCache::GetHash(Handle* handle) const {
   return reinterpret_cast<const LRUHandle*>(handle)->hash;
 }
 
-void LRUCache::DisownData() {
-// Do not drop data if compile with ASAN to suppress leak warning.
-#if defined(__clang__)
-#if !defined(__has_feature) || !__has_feature(address_sanitizer)
-  shards_ = nullptr;
-  num_shards_ = 0;
-#endif
-#else  // __clang__
-#ifndef __SANITIZE_ADDRESS__
-  shards_ = nullptr;
-  num_shards_ = 0;
-#endif  // !__SANITIZE_ADDRESS__
-#endif  // __clang__
-}
+void LRUCache::DisownData() { shards_ = nullptr; }
 
-size_t LRUCache::TEST_GetLRUSize() {
-  size_t lru_size_of_all_shards = 0;
-  for (int i = 0; i < num_shards_; i++) {
-    lru_size_of_all_shards += shards_[i].TEST_GetLRUSize();
-  }
-  return lru_size_of_all_shards;
-}
-
-double LRUCache::GetHighPriPoolRatio() {
-  double result = 0.0;
-  if (num_shards_ > 0) {
-    result = shards_[0].GetHighPriPoolRatio();
-  }
-  return result;
-}
-
-std::shared_ptr<Cache> NewLRUCache(const LRUCacheOptions& cache_opts) {
-  return NewLRUCache(cache_opts.capacity, cache_opts.num_shard_bits,
-                     cache_opts.strict_capacity_limit,
-                     cache_opts.high_pri_pool_ratio,
-                     cache_opts.memory_allocator, cache_opts.use_adaptive_mutex,
-                     cache_opts.metadata_charge_policy);
-}
-
-std::shared_ptr<Cache> NewLRUCache(
-    size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-    double high_pri_pool_ratio,
-    std::shared_ptr<MemoryAllocator> memory_allocator, bool use_adaptive_mutex,
-    CacheMetadataChargePolicy metadata_charge_policy) {
+std::shared_ptr<Cache> NewLRUCache(size_t capacity, int num_shard_bits,
+                                   bool strict_capacity_limit,
+                                   double high_pri_pool_ratio) {
   if (num_shard_bits >= 20) {
     return nullptr;  // the cache cannot be sharded into too many fine pieces
   }
@@ -566,9 +483,8 @@ std::shared_ptr<Cache> NewLRUCache(
   if (num_shard_bits < 0) {
     num_shard_bits = GetDefaultCacheShardBits(capacity);
   }
-  return std::make_shared<LRUCache>(
-      capacity, num_shard_bits, strict_capacity_limit, high_pri_pool_ratio,
-      std::move(memory_allocator), use_adaptive_mutex, metadata_charge_policy);
+  return std::make_shared<LRUCache>(capacity, num_shard_bits,
+                                    strict_capacity_limit, high_pri_pool_ratio);
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cache/lru_cache.h

@@ -12,40 +12,36 @@
 
 #include "cache/sharded_cache.h"
 
-#include "port/malloc.h"
 #include "port/port.h"
 #include "util/autovector.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
-// LRU cache implementation. This class is not thread-safe.
+// LRU cache implementation
 
 // An entry is a variable length heap-allocated structure.
 // Entries are referenced by cache and/or by any external entity.
-// The cache keeps all its entries in a hash table. Some elements
+// The cache keeps all its entries in table. Some elements
 // are also stored on LRU list.
 //
 // LRUHandle can be in these states:
 // 1. Referenced externally AND in hash table.
-//    In that case the entry is *not* in the LRU list
-//    (refs >= 1 && in_cache == true)
-// 2. Not referenced externally AND in hash table.
-//    In that case the entry is in the LRU list and can be freed.
-//    (refs == 0 && in_cache == true)
-// 3. Referenced externally AND not in hash table.
-//    In that case the entry is not in the LRU list and not in hash table.
-//    The entry can be freed when refs becomes 0.
-//    (refs >= 1 && in_cache == false)
+//  In that case the entry is *not* in the LRU. (refs > 1 && in_cache == true)
+// 2. Not referenced externally and in hash table. In that case the entry is
+// in the LRU and can be freed. (refs == 1 && in_cache == true)
+// 3. Referenced externally and not in hash table. In that case the entry is
+// in not on LRU and not in table. (refs >= 1 && in_cache == false)
 //
 // All newly created LRUHandles are in state 1. If you call
-// LRUCacheShard::Release on entry in state 1, it will go into state 2.
-// To move from state 1 to state 3, either call LRUCacheShard::Erase or
-// LRUCacheShard::Insert with the same key (but possibly different value).
+// LRUCacheShard::Release
+// on entry in state 1, it will go into state 2. To move from state 1 to
+// state 3, either call LRUCacheShard::Erase or LRUCacheShard::Insert with the
+// same key.
 // To move from state 2 to state 1, use LRUCacheShard::Lookup.
 // Before destruction, make sure that no handles are in state 1. This means
 // that any successful LRUCacheShard::Lookup/LRUCacheShard::Insert have a
-// matching LRUCache::Release (to move into state 2) or LRUCacheShard::Erase
-// (to move into state 3).
+// matching
+// RUCache::Release (to move into state 2) or LRUCacheShard::Erase (for state 3)
 
 struct LRUHandle {
   void* value;
@@ -55,95 +51,64 @@ struct LRUHandle {
   LRUHandle* prev;
   size_t charge;  // TODO(opt): Only allow uint32_t?
   size_t key_length;
-  // The hash of key(). Used for fast sharding and comparisons.
-  uint32_t hash;
-  // The number of external refs to this entry. The cache itself is not counted.
-  uint32_t refs;
+  uint32_t refs;     // a number of refs to this entry
+                     // cache itself is counted as 1
 
-  enum Flags : uint8_t {
-    // Whether this entry is referenced by the hash table.
-    IN_CACHE = (1 << 0),
-    // Whether this entry is high priority entry.
-    IS_HIGH_PRI = (1 << 1),
-    // Whether this entry is in high-pri pool.
-    IN_HIGH_PRI_POOL = (1 << 2),
-    // Whether this entry has had any lookups (hits).
-    HAS_HIT = (1 << 3),
-  };
+  // Include the following flags:
+  //   in_cache:    whether this entry is referenced by the hash table.
+  //   is_high_pri: whether this entry is high priority entry.
+  //   in_high_pro_pool: whether this entry is in high-pri pool.
+  char flags;
 
-  uint8_t flags;
+  uint32_t hash;     // Hash of key(); used for fast sharding and comparisons
 
-  // Beginning of the key (MUST BE THE LAST FIELD IN THIS STRUCT!)
-  char key_data[1];
+  char key_data[1];  // Beginning of key
 
-  Slice key() const { return Slice(key_data, key_length); }
-
-  // Increase the reference count by 1.
-  void Ref() { refs++; }
-
-  // Just reduce the reference count by 1. Return true if it was last reference.
-  bool Unref() {
-    assert(refs > 0);
-    refs--;
-    return refs == 0;
+  Slice key() const {
+    // For cheaper lookups, we allow a temporary Handle object
+    // to store a pointer to a key in "value".
+    if (next == this) {
+      return *(reinterpret_cast<Slice*>(value));
+    } else {
+      return Slice(key_data, key_length);
+    }
   }
 
-  // Return true if there are external refs, false otherwise.
-  bool HasRefs() const { return refs > 0; }
-
-  bool InCache() const { return flags & IN_CACHE; }
-  bool IsHighPri() const { return flags & IS_HIGH_PRI; }
-  bool InHighPriPool() const { return flags & IN_HIGH_PRI_POOL; }
-  bool HasHit() const { return flags & HAS_HIT; }
+  bool InCache() { return flags & 1; }
+  bool IsHighPri() { return flags & 2; }
+  bool InHighPriPool() { return flags & 4; }
 
   void SetInCache(bool in_cache) {
     if (in_cache) {
-      flags |= IN_CACHE;
+      flags |= 1;
     } else {
-      flags &= ~IN_CACHE;
+      flags &= ~1;
     }
   }
 
   void SetPriority(Cache::Priority priority) {
     if (priority == Cache::Priority::HIGH) {
-      flags |= IS_HIGH_PRI;
+      flags |= 2;
     } else {
-      flags &= ~IS_HIGH_PRI;
+      flags &= ~2;
     }
   }
 
   void SetInHighPriPool(bool in_high_pri_pool) {
     if (in_high_pri_pool) {
-      flags |= IN_HIGH_PRI_POOL;
+      flags |= 4;
     } else {
-      flags &= ~IN_HIGH_PRI_POOL;
+      flags &= ~4;
     }
   }
 
-  void SetHit() { flags |= HAS_HIT; }
-
   void Free() {
-    assert(refs == 0);
+    assert((refs == 1 && InCache()) || (refs == 0 && !InCache()));
     if (deleter) {
       (*deleter)(key(), value);
     }
     delete[] reinterpret_cast<char*>(this);
   }
-
-  // Calculate the memory usage by metadata
-  inline size_t CalcTotalCharge(
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    size_t meta_charge = 0;
-    if (metadata_charge_policy == kFullChargeCacheMetadata) {
-#ifdef ROCKSDB_MALLOC_USABLE_SIZE
-      meta_charge += malloc_usable_size(static_cast<void*>(this));
-#else
-      // This is the size that is used when a new handle is created
-      meta_charge += sizeof(LRUHandle) - 1 + key_length;
-#endif
-    }
-    return charge + meta_charge;
-  }
 };
 
 // We provide our own simple hash table since it removes a whole bunch
@@ -183,18 +148,16 @@ class LRUHandleTable {
 
   // The table consists of an array of buckets where each bucket is
   // a linked list of cache entries that hash into the bucket.
-  LRUHandle** list_;
   uint32_t length_;
   uint32_t elems_;
+  LRUHandle** list_;
 };
 
 // A single shard of sharded cache.
-class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
+class LRUCacheShard : public CacheShard {
  public:
-  LRUCacheShard(size_t capacity, bool strict_capacity_limit,
-                double high_pri_pool_ratio, bool use_adaptive_mutex,
-                CacheMetadataChargePolicy metadata_charge_policy);
-  virtual ~LRUCacheShard() override = default;
+  LRUCacheShard();
+  virtual ~LRUCacheShard();
 
   // Separate from constructor so caller can easily make an array of LRUCache
   // if current usage is more than new capacity, the function will attempt to
@@ -215,8 +178,7 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
                         Cache::Priority priority) override;
   virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) override;
   virtual bool Ref(Cache::Handle* handle) override;
-  virtual bool Release(Cache::Handle* handle,
-                       bool force_erase = false) override;
+  virtual void Release(Cache::Handle* handle) override;
   virtual void Erase(const Slice& key, uint32_t hash) override;
 
   // Although in some platforms the update of size_t is atomic, to make sure
@@ -235,13 +197,6 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
 
   void TEST_GetLRUList(LRUHandle** lru, LRUHandle** lru_low_pri);
 
-  //  Retrieves number of elements in LRU, for unit test purpose only
-  //  not threadsafe
-  size_t TEST_GetLRUSize();
-
-  //  Retrieves high pri pool ratio
-  double GetHighPriPoolRatio();
-
  private:
   void LRU_Remove(LRUHandle* e);
   void LRU_Insert(LRUHandle* e);
@@ -250,6 +205,10 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // high-pri pool is no larger than the size specify by high_pri_pool_pct.
   void MaintainPoolSize();
 
+  // Just reduce the reference count by 1.
+  // Return true if last reference
+  bool Unref(LRUHandle* e);
+
   // Free some space following strict LRU policy until enough space
   // to hold (usage_ + charge) is freed or the lru list is empty
   // This function is not thread safe - it needs to be executed while
@@ -259,6 +218,12 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Initialized before use.
   size_t capacity_;
 
+  // Memory size for entries residing in the cache
+  size_t usage_;
+
+  // Memory size for entries residing only in the LRU list
+  size_t lru_usage_;
+
   // Memory size for entries in high-pri pool.
   size_t high_pri_pool_usage_;
 
@@ -272,6 +237,11 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Remember the value to avoid recomputing each time.
   double high_pri_pool_capacity_;
 
+  // mutex_ protects the following state.
+  // We don't count mutex_ as the cache's internal state so semantically we
+  // don't mind mutex_ invoking the non-const actions.
+  mutable port::Mutex mutex_;
+
   // Dummy head of LRU list.
   // lru.prev is newest entry, lru.next is oldest entry.
   // LRU contains items which can be evicted, ie reference only by cache
@@ -280,43 +250,13 @@ class ALIGN_AS(CACHE_LINE_SIZE) LRUCacheShard final : public CacheShard {
   // Pointer to head of low-pri pool in LRU list.
   LRUHandle* lru_low_pri_;
 
-  // ------------^^^^^^^^^^^^^-----------
-  // Not frequently modified data members
-  // ------------------------------------
-  //
-  // We separate data members that are updated frequently from the ones that
-  // are not frequently updated so that they don't share the same cache line
-  // which will lead into false cache sharing
-  //
-  // ------------------------------------
-  // Frequently modified data members
-  // ------------vvvvvvvvvvvvv-----------
   LRUHandleTable table_;
-
-  // Memory size for entries residing in the cache
-  size_t usage_;
-
-  // Memory size for entries residing only in the LRU list
-  size_t lru_usage_;
-
-  // mutex_ protects the following state.
-  // We don't count mutex_ as the cache's internal state so semantically we
-  // don't mind mutex_ invoking the non-const actions.
-  mutable port::Mutex mutex_;
 };
 
-class LRUCache
-#ifdef NDEBUG
-    final
-#endif
-    : public ShardedCache {
+class LRUCache : public ShardedCache {
  public:
   LRUCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-           double high_pri_pool_ratio,
-           std::shared_ptr<MemoryAllocator> memory_allocator = nullptr,
-           bool use_adaptive_mutex = kDefaultToAdaptiveMutex,
-           CacheMetadataChargePolicy metadata_charge_policy =
-               kDontChargeCacheMetadata);
+           double high_pri_pool_ratio);
   virtual ~LRUCache();
   virtual const char* Name() const override { return "LRUCache"; }
   virtual CacheShard* GetShard(int shard) override;
@@ -326,14 +266,8 @@ class LRUCache
   virtual uint32_t GetHash(Handle* handle) const override;
   virtual void DisownData() override;
 
-  //  Retrieves number of elements in LRU, for unit test purpose only
-  size_t TEST_GetLRUSize();
-  //  Retrieves high pri pool ratio
-  double GetHighPriPoolRatio();
-
  private:
-  LRUCacheShard* shards_ = nullptr;
-  int num_shards_ = 0;
+  LRUCacheShard* shards_;
 };
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cache/lru_cache_test.cc

@@ -7,39 +7,26 @@
 
 #include <string>
 #include <vector>
-#include "port/port.h"
-#include "test_util/testharness.h"
+#include "util/testharness.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 class LRUCacheTest : public testing::Test {
  public:
   LRUCacheTest() {}
-  ~LRUCacheTest() override { DeleteCache(); }
+  ~LRUCacheTest() {}
 
-  void DeleteCache() {
-    if (cache_ != nullptr) {
-      cache_->~LRUCacheShard();
-      port::cacheline_aligned_free(cache_);
-      cache_ = nullptr;
-    }
-  }
-
-  void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0,
-                bool use_adaptive_mutex = kDefaultToAdaptiveMutex) {
-    DeleteCache();
-    cache_ = reinterpret_cast<LRUCacheShard*>(
-        port::cacheline_aligned_alloc(sizeof(LRUCacheShard)));
-    new (cache_) LRUCacheShard(capacity, false /*strict_capacity_limit*/,
-                               high_pri_pool_ratio, use_adaptive_mutex,
-                               kDontChargeCacheMetadata);
+  void NewCache(size_t capacity, double high_pri_pool_ratio = 0.0) {
+    cache_.reset(new LRUCacheShard());
+    cache_->SetCapacity(capacity);
+    cache_->SetStrictCapacityLimit(false);
+    cache_->SetHighPriorityPoolRatio(high_pri_pool_ratio);
   }
 
   void Insert(const std::string& key,
               Cache::Priority priority = Cache::Priority::LOW) {
-    EXPECT_OK(cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/,
-                             nullptr /*deleter*/, nullptr /*handle*/,
-                             priority));
+    cache_->Insert(key, 0 /*hash*/, nullptr /*value*/, 1 /*charge*/,
+                   nullptr /*deleter*/, nullptr /*handle*/, priority);
   }
 
   void Insert(char key, Cache::Priority priority = Cache::Priority::LOW) {
@@ -89,7 +76,7 @@ class LRUCacheTest : public testing::Test {
   }
 
  private:
-  LRUCacheShard* cache_ = nullptr;
+  std::unique_ptr<LRUCacheShard> cache_;
 };
 
 TEST_F(LRUCacheTest, BasicLRU) {
@@ -118,30 +105,7 @@ TEST_F(LRUCacheTest, BasicLRU) {
   ValidateLRUList({"e", "z", "d", "u", "v"});
 }
 
-TEST_F(LRUCacheTest, MidpointInsertion) {
-  // Allocate 2 cache entries to high-pri pool.
-  NewCache(5, 0.45);
-
-  Insert("a", Cache::Priority::LOW);
-  Insert("b", Cache::Priority::LOW);
-  Insert("c", Cache::Priority::LOW);
-  Insert("x", Cache::Priority::HIGH);
-  Insert("y", Cache::Priority::HIGH);
-  ValidateLRUList({"a", "b", "c", "x", "y"}, 2);
-
-  // Low-pri entries inserted to the tail of low-pri list (the midpoint).
-  // After lookup, it will move to the tail of the full list.
-  Insert("d", Cache::Priority::LOW);
-  ValidateLRUList({"b", "c", "d", "x", "y"}, 2);
-  ASSERT_TRUE(Lookup("d"));
-  ValidateLRUList({"b", "c", "x", "y", "d"}, 2);
-
-  // High-pri entries will be inserted to the tail of full list.
-  Insert("z", Cache::Priority::HIGH);
-  ValidateLRUList({"c", "x", "y", "d", "z"}, 2);
-}
-
-TEST_F(LRUCacheTest, EntriesWithPriority) {
+TEST_F(LRUCacheTest, MidPointInsertion) {
   // Allocate 2 cache entries to high-pri pool.
   NewCache(5, 0.45);
 
@@ -167,15 +131,15 @@ TEST_F(LRUCacheTest, EntriesWithPriority) {
   Insert("a", Cache::Priority::LOW);
   ValidateLRUList({"v", "X", "a", "Y", "Z"}, 2);
 
-  // Low-pri entries will be inserted to head of high-pri pool after lookup.
+  // Low-pri entries will be inserted to head of low-pri pool after lookup.
   ASSERT_TRUE(Lookup("v"));
-  ValidateLRUList({"X", "a", "Y", "Z", "v"}, 2);
+  ValidateLRUList({"X", "a", "v", "Y", "Z"}, 2);
 
   // High-pri entries will be inserted to the head of the list after lookup.
   ASSERT_TRUE(Lookup("X"));
-  ValidateLRUList({"a", "Y", "Z", "v", "X"}, 2);
+  ValidateLRUList({"a", "v", "Y", "Z", "X"}, 2);
   ASSERT_TRUE(Lookup("Z"));
-  ValidateLRUList({"a", "Y", "v", "X", "Z"}, 2);
+  ValidateLRUList({"a", "v", "Y", "X", "Z"}, 2);
 
   Erase("Y");
   ValidateLRUList({"a", "v", "X", "Z"}, 2);
@@ -188,10 +152,10 @@ TEST_F(LRUCacheTest, EntriesWithPriority) {
   Insert("g", Cache::Priority::LOW);
   ValidateLRUList({"d", "e", "f", "g", "Z"}, 1);
   ASSERT_TRUE(Lookup("d"));
-  ValidateLRUList({"e", "f", "g", "Z", "d"}, 2);
+  ValidateLRUList({"e", "f", "g", "d", "Z"}, 1);
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb
 
 int main(int argc, char** argv) {
   ::testing::InitGoogleTest(&argc, argv);

cache/sharded_cache.cc

@@ -7,19 +7,21 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
 
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+
 #include "cache/sharded_cache.h"
 
 #include <string>
 
 #include "util/mutexlock.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 ShardedCache::ShardedCache(size_t capacity, int num_shard_bits,
-                           bool strict_capacity_limit,
-                           std::shared_ptr<MemoryAllocator> allocator)
-    : Cache(std::move(allocator)),
-      num_shard_bits_(num_shard_bits),
+                           bool strict_capacity_limit)
+    : num_shard_bits_(num_shard_bits),
       capacity_(capacity),
       strict_capacity_limit_(strict_capacity_limit),
       last_id_(1) {}
@@ -51,7 +53,7 @@ Status ShardedCache::Insert(const Slice& key, void* value, size_t charge,
       ->Insert(key, hash, value, charge, deleter, handle, priority);
 }
 
-Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* /*stats*/) {
+Cache::Handle* ShardedCache::Lookup(const Slice& key, Statistics* stats) {
   uint32_t hash = HashSlice(key);
   return GetShard(Shard(hash))->Lookup(key, hash);
 }
@@ -61,9 +63,9 @@ bool ShardedCache::Ref(Handle* handle) {
   return GetShard(Shard(hash))->Ref(handle);
 }
 
-bool ShardedCache::Release(Handle* handle, bool force_erase) {
+void ShardedCache::Release(Handle* handle) {
   uint32_t hash = GetHash(handle);
-  return GetShard(Shard(hash))->Release(handle, force_erase);
+  GetShard(Shard(hash))->Release(handle);
 }
 
 void ShardedCache::Erase(const Slice& key) {
@@ -140,9 +142,6 @@ std::string ShardedCache::GetPrintableOptions() const {
              strict_capacity_limit_);
     ret.append(buffer);
   }
-  snprintf(buffer, kBufferSize, "    memory_allocator : %s\n",
-           memory_allocator() ? memory_allocator()->Name() : "None");
-  ret.append(buffer);
   ret.append(GetShard(0)->GetPrintableOptions());
   return ret;
 }
@@ -159,4 +158,4 @@ int GetDefaultCacheShardBits(size_t capacity) {
   return num_shard_bits;
 }
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cache/sharded_cache.h

@@ -16,7 +16,7 @@
 #include "rocksdb/cache.h"
 #include "util/hash.h"
 
-namespace ROCKSDB_NAMESPACE {
+namespace rocksdb {
 
 // Single cache shard interface.
 class CacheShard {
@@ -30,7 +30,7 @@ class CacheShard {
                         Cache::Handle** handle, Cache::Priority priority) = 0;
   virtual Cache::Handle* Lookup(const Slice& key, uint32_t hash) = 0;
   virtual bool Ref(Cache::Handle* handle) = 0;
-  virtual bool Release(Cache::Handle* handle, bool force_erase = false) = 0;
+  virtual void Release(Cache::Handle* handle) = 0;
   virtual void Erase(const Slice& key, uint32_t hash) = 0;
   virtual void SetCapacity(size_t capacity) = 0;
   virtual void SetStrictCapacityLimit(bool strict_capacity_limit) = 0;
@@ -40,13 +40,6 @@ class CacheShard {
                                       bool thread_safe) = 0;
   virtual void EraseUnRefEntries() = 0;
   virtual std::string GetPrintableOptions() const { return ""; }
-  void set_metadata_charge_policy(
-      CacheMetadataChargePolicy metadata_charge_policy) {
-    metadata_charge_policy_ = metadata_charge_policy;
-  }
-
- protected:
-  CacheMetadataChargePolicy metadata_charge_policy_ = kDontChargeCacheMetadata;
 };
 
 // Generic cache interface which shards cache by hash of keys. 2^num_shard_bits
@@ -54,15 +47,13 @@ class CacheShard {
 // Keys are sharded by the highest num_shard_bits bits of hash value.
 class ShardedCache : public Cache {
  public:
-  ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit,
-               std::shared_ptr<MemoryAllocator> memory_allocator = nullptr);
+  ShardedCache(size_t capacity, int num_shard_bits, bool strict_capacity_limit);
   virtual ~ShardedCache() = default;
   virtual const char* Name() const override = 0;
   virtual CacheShard* GetShard(int shard) = 0;
   virtual const CacheShard* GetShard(int shard) const = 0;
   virtual void* Value(Handle* handle) override = 0;
-  virtual size_t GetCharge(Handle* handle) const override = 0;
-
+  virtual size_t GetCharge(Handle* handle) const = 0;
   virtual uint32_t GetHash(Handle* handle) const = 0;
   virtual void DisownData() override = 0;
 
@@ -74,7 +65,7 @@ class ShardedCache : public Cache {
                         Handle** handle, Priority priority) override;
   virtual Handle* Lookup(const Slice& key, Statistics* stats) override;
   virtual bool Ref(Handle* handle) override;
-  virtual bool Release(Handle* handle, bool force_erase = false) override;
+  virtual void Release(Handle* handle) override;
   virtual void Erase(const Slice& key) override;
   virtual uint64_t NewId() override;
   virtual size_t GetCapacity() const override;
@@ -91,7 +82,7 @@ class ShardedCache : public Cache {
 
  private:
   static inline uint32_t HashSlice(const Slice& s) {
-    return static_cast<uint32_t>(GetSliceNPHash64(s));
+    return Hash(s.data(), s.size(), 0);
   }
 
   uint32_t Shard(uint32_t hash) {
@@ -108,4 +99,4 @@ class ShardedCache : public Cache {
 
 extern int GetDefaultCacheShardBits(size_t capacity);
 
-}  // namespace ROCKSDB_NAMESPACE
+}  // namespace rocksdb

cmake/RocksDBConfig.cmake.in

@@ -1,54 +0,0 @@
-@PACKAGE_INIT@
-
-list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
-
-include(CMakeFindDependencyMacro)
-
-set(GFLAGS_USE_TARGET_NAMESPACE @GFLAGS_USE_TARGET_NAMESPACE@)
-
-if(@WITH_JEMALLOC@)
-  find_dependency(JeMalloc)
-endif()
-
-if(@WITH_GFLAGS@)
-  find_dependency(gflags CONFIG)
-  if(NOT gflags_FOUND)
-    find_dependency(gflags)
-  endif()
-endif()
-
-if(@WITH_SNAPPY@)
-  find_dependency(Snappy CONFIG)
-  if(NOT Snappy_FOUND)
-    find_dependency(Snappy)
-  endif()
-endif()
-
-if(@WITH_ZLIB@)
-  find_dependency(ZLIB)
-endif()
-
-if(@WITH_BZ2@)
-  find_dependency(BZip2)
-endif()
-
-if(@WITH_LZ4@)
-  find_dependency(lz4)
-endif()
-
-if(@WITH_ZSTD@)
-  find_dependency(zstd)
-endif()
-
-if(@WITH_NUMA@)
-  find_dependency(NUMA)
-endif()
-
-if(@WITH_TBB@)
-  find_dependency(TBB)
-endif()
-
-find_dependency(Threads)
-
-include("${CMAKE_CURRENT_LIST_DIR}/RocksDBTargets.cmake")
-check_required_components(RocksDB)

cmake/modules/CxxFlags.cmake

@@ -1,7 +0,0 @@
-macro(get_cxx_std_flags FLAGS_VARIABLE)
-  if( CMAKE_CXX_STANDARD_REQUIRED )
-    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION})
-  else()
-    set(${FLAGS_VARIABLE} ${CMAKE_CXX${CMAKE_CXX_STANDARD}_EXTENSION_COMPILE_OPTION})
-  endif()
-endmacro()

cmake/modules/FindJeMalloc.cmake

@@ -1,29 +1,21 @@
 # - Find JeMalloc library
 # Find the native JeMalloc includes and library
 #
-# JeMalloc_INCLUDE_DIRS - where to find jemalloc.h, etc.
-# JeMalloc_LIBRARIES - List of libraries when using jemalloc.
-# JeMalloc_FOUND - True if jemalloc found.
+# JEMALLOC_INCLUDE_DIR - where to find jemalloc.h, etc.
+# JEMALLOC_LIBRARIES - List of libraries when using jemalloc.
+# JEMALLOC_FOUND - True if jemalloc found.
 
-find_path(JeMalloc_INCLUDE_DIRS
+find_path(JEMALLOC_INCLUDE_DIR
   NAMES jemalloc/jemalloc.h
   HINTS ${JEMALLOC_ROOT_DIR}/include)
 
-find_library(JeMalloc_LIBRARIES
+find_library(JEMALLOC_LIBRARIES
   NAMES jemalloc
   HINTS ${JEMALLOC_ROOT_DIR}/lib)
 
 include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(JeMalloc DEFAULT_MSG JeMalloc_LIBRARIES JeMalloc_INCLUDE_DIRS)
+find_package_handle_standard_args(jemalloc DEFAULT_MSG JEMALLOC_LIBRARIES JEMALLOC_INCLUDE_DIR)
 
 mark_as_advanced(
-  JeMalloc_LIBRARIES
-  JeMalloc_INCLUDE_DIRS)
-
-if(JeMalloc_FOUND AND NOT (TARGET JeMalloc::JeMalloc))
-  add_library (JeMalloc::JeMalloc UNKNOWN IMPORTED)
-  set_target_properties(JeMalloc::JeMalloc
-    PROPERTIES
-      IMPORTED_LOCATION ${JeMalloc_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${JeMalloc_INCLUDE_DIRS})
-endif()
+  JEMALLOC_LIBRARIES
+  JEMALLOC_INCLUDE_DIR)

cmake/modules/FindNUMA.cmake

@@ -1,29 +0,0 @@
-# - Find NUMA
-# Find the NUMA library and includes
-#
-# NUMA_INCLUDE_DIRS - where to find numa.h, etc.
-# NUMA_LIBRARIES - List of libraries when using NUMA.
-# NUMA_FOUND - True if NUMA found.
-
-find_path(NUMA_INCLUDE_DIRS
-  NAMES numa.h numaif.h
-  HINTS ${NUMA_ROOT_DIR}/include)
-
-find_library(NUMA_LIBRARIES
-  NAMES numa
-  HINTS ${NUMA_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(NUMA DEFAULT_MSG NUMA_LIBRARIES NUMA_INCLUDE_DIRS)
-
-mark_as_advanced(
-  NUMA_LIBRARIES
-  NUMA_INCLUDE_DIRS)
-
-if(NUMA_FOUND AND NOT (TARGET NUMA::NUMA))
-  add_library (NUMA::NUMA UNKNOWN IMPORTED)
-  set_target_properties(NUMA::NUMA
-    PROPERTIES
-      IMPORTED_LOCATION ${NUMA_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${NUMA_INCLUDE_DIRS})
-endif()

cmake/modules/FindSnappy.cmake

@@ -1,29 +0,0 @@
-# - Find Snappy
-# Find the snappy compression library and includes
-#
-# Snappy_INCLUDE_DIRS - where to find snappy.h, etc.
-# Snappy_LIBRARIES - List of libraries when using snappy.
-# Snappy_FOUND - True if snappy found.
-
-find_path(Snappy_INCLUDE_DIRS
-  NAMES snappy.h
-  HINTS ${snappy_ROOT_DIR}/include)
-
-find_library(Snappy_LIBRARIES
-  NAMES snappy
-  HINTS ${snappy_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(Snappy DEFAULT_MSG Snappy_LIBRARIES Snappy_INCLUDE_DIRS)
-
-mark_as_advanced(
-  Snappy_LIBRARIES
-  Snappy_INCLUDE_DIRS)
-
-if(Snappy_FOUND AND NOT (TARGET Snappy::snappy))
-  add_library (Snappy::snappy UNKNOWN IMPORTED)
-  set_target_properties(Snappy::snappy
-    PROPERTIES
-      IMPORTED_LOCATION ${Snappy_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${Snappy_INCLUDE_DIRS})
-endif()

cmake/modules/FindTBB.cmake

@@ -1,33 +0,0 @@
-# - Find TBB
-# Find the Thread Building Blocks library and includes
-#
-# TBB_INCLUDE_DIRS - where to find tbb.h, etc.
-# TBB_LIBRARIES - List of libraries when using TBB.
-# TBB_FOUND - True if TBB found.
-
-if(NOT DEFINED TBB_ROOT_DIR)
-  set(TBB_ROOT_DIR "$ENV{TBBROOT}")
-endif()
-
-find_path(TBB_INCLUDE_DIRS
-  NAMES tbb/tbb.h
-  HINTS ${TBB_ROOT_DIR}/include)
-
-find_library(TBB_LIBRARIES
-  NAMES tbb
-  HINTS ${TBB_ROOT_DIR}/lib ENV LIBRARY_PATH)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(TBB DEFAULT_MSG TBB_LIBRARIES TBB_INCLUDE_DIRS)
-
-mark_as_advanced(
-  TBB_LIBRARIES
-  TBB_INCLUDE_DIRS)
-
-if(TBB_FOUND AND NOT (TARGET TBB::TBB))
-  add_library (TBB::TBB UNKNOWN IMPORTED)
-  set_target_properties(TBB::TBB
-    PROPERTIES
-      IMPORTED_LOCATION ${TBB_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${TBB_INCLUDE_DIRS})
-endif()

cmake/modules/Findgflags.cmake

@@ -1,29 +0,0 @@
-# - Find gflags library
-# Find the gflags includes and library
-#
-# GFLAGS_INCLUDE_DIR - where to find gflags.h.
-# GFLAGS_LIBRARIES - List of libraries when using gflags.
-# gflags_FOUND - True if gflags found.
-
-find_path(GFLAGS_INCLUDE_DIR
-  NAMES gflags/gflags.h)
-
-find_library(GFLAGS_LIBRARIES
-  NAMES gflags)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(gflags
-  DEFAULT_MSG GFLAGS_LIBRARIES GFLAGS_INCLUDE_DIR)
-
-mark_as_advanced(
-  GFLAGS_LIBRARIES
-  GFLAGS_INCLUDE_DIR)
-
-if(gflags_FOUND AND NOT (TARGET gflags::gflags))
-  add_library(gflags::gflags UNKNOWN IMPORTED)
-  set_target_properties(gflags::gflags
-    PROPERTIES
-      IMPORTED_LOCATION ${GFLAGS_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${GFLAGS_INCLUDE_DIR}
-      IMPORTED_LINK_INTERFACE_LANGUAGES "CXX")
-endif()

cmake/modules/Findlz4.cmake

@@ -1,29 +0,0 @@
-# - Find Lz4
-# Find the lz4 compression library and includes
-#
-# lz4_INCLUDE_DIRS - where to find lz4.h, etc.
-# lz4_LIBRARIES - List of libraries when using lz4.
-# lz4_FOUND - True if lz4 found.
-
-find_path(lz4_INCLUDE_DIRS
-  NAMES lz4.h
-  HINTS ${lz4_ROOT_DIR}/include)
-
-find_library(lz4_LIBRARIES
-  NAMES lz4
-  HINTS ${lz4_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(lz4 DEFAULT_MSG lz4_LIBRARIES lz4_INCLUDE_DIRS)
-
-mark_as_advanced(
-  lz4_LIBRARIES
-  lz4_INCLUDE_DIRS)
-
-if(lz4_FOUND AND NOT (TARGET lz4::lz4))
-  add_library(lz4::lz4 UNKNOWN IMPORTED)
-  set_target_properties(lz4::lz4
-    PROPERTIES
-      IMPORTED_LOCATION ${lz4_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${lz4_INCLUDE_DIRS})
-endif()

cmake/modules/Findsnappy.cmake

@@ -0,0 +1,21 @@
+# - Find Snappy
+# Find the snappy compression library and includes
+#
+# SNAPPY_INCLUDE_DIR - where to find snappy.h, etc.
+# SNAPPY_LIBRARIES - List of libraries when using snappy.
+# SNAPPY_FOUND - True if snappy found.
+
+find_path(SNAPPY_INCLUDE_DIR
+  NAMES snappy.h
+  HINTS ${SNAPPY_ROOT_DIR}/include)
+
+find_library(SNAPPY_LIBRARIES
+  NAMES snappy
+  HINTS ${SNAPPY_ROOT_DIR}/lib)
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(snappy DEFAULT_MSG SNAPPY_LIBRARIES SNAPPY_INCLUDE_DIR)
+
+mark_as_advanced(
+  SNAPPY_LIBRARIES
+  SNAPPY_INCLUDE_DIR)

cmake/modules/Findzstd.cmake

@@ -1,29 +0,0 @@
-# - Find zstd
-# Find the zstd compression library and includes
-#
-# zstd_INCLUDE_DIRS - where to find zstd.h, etc.
-# zstd_LIBRARIES - List of libraries when using zstd.
-# zstd_FOUND - True if zstd found.
-
-find_path(zstd_INCLUDE_DIRS
-  NAMES zstd.h
-  HINTS ${zstd_ROOT_DIR}/include)
-
-find_library(zstd_LIBRARIES
-  NAMES zstd
-  HINTS ${zstd_ROOT_DIR}/lib)
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(zstd DEFAULT_MSG zstd_LIBRARIES zstd_INCLUDE_DIRS)
-
-mark_as_advanced(
-  zstd_LIBRARIES
-  zstd_INCLUDE_DIRS)
-
-if(zstd_FOUND AND NOT (TARGET zstd::zstd))
-  add_library (zstd::zstd UNKNOWN IMPORTED)
-  set_target_properties(zstd::zstd
-    PROPERTIES
-      IMPORTED_LOCATION ${zstd_LIBRARIES}
-      INTERFACE_INCLUDE_DIRECTORIES ${zstd_INCLUDE_DIRS})
-endif()

cmake/modules/ReadVersion.cmake

@@ -1,10 +0,0 @@
-# Read rocksdb version from version.h header file.
-
-function(get_rocksdb_version version_var)
-  file(READ "${CMAKE_CURRENT_SOURCE_DIR}/include/rocksdb/version.h" version_header_file)
-  foreach(component MAJOR MINOR PATCH)
-    string(REGEX MATCH "#define ROCKSDB_${component} ([0-9]+)" _ ${version_header_file})
-    set(ROCKSDB_VERSION_${component} ${CMAKE_MATCH_1})
-  endforeach()
-  set(${version_var} "${ROCKSDB_VERSION_MAJOR}.${ROCKSDB_VERSION_MINOR}.${ROCKSDB_VERSION_PATCH}" PARENT_SCOPE)
-endfunction()

coverage/coverage_test.sh

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+#!/bin/bash
 
 # Exit on error.
 set -e
@@ -12,24 +11,21 @@ fi
 ROOT=".."
 # Fetch right version of gcov
 if [ -d /mnt/gvfs/third-party -a -z "$CXX" ]; then
-  source $ROOT/build_tools/fbcode_config_platform007.sh
+  source $ROOT/build_tools/fbcode_config.sh
   GCOV=$GCC_BASE/bin/gcov
 else
   GCOV=$(which gcov)
 fi
-echo -e "Using $GCOV"
 
 COVERAGE_DIR="$PWD/COVERAGE_REPORT"
 mkdir -p $COVERAGE_DIR
 
 # Find all gcno files to generate the coverage report
 
-PYTHON=${1:-`which python`}
-echo -e "Using $PYTHON"
 GCNO_FILES=`find $ROOT -name "*.gcno"`
 $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
   # Parse the raw gcov report to more human readable form.
-  $PYTHON $ROOT/coverage/parse_gcov_output.py |
+  python $ROOT/coverage/parse_gcov_output.py |
   # Write the output to both stdout and report file.
   tee $COVERAGE_DIR/coverage_report_all.txt &&
 echo -e "Generated coverage report for all files: $COVERAGE_DIR/coverage_report_all.txt\n"
@@ -44,7 +40,7 @@ RECENT_REPORT=$COVERAGE_DIR/coverage_report_recent.txt
 
 echo -e "Recently updated files: $LATEST_FILES\n" > $RECENT_REPORT
 $GCOV --preserve-paths --relative-only --no-output $GCNO_FILES 2>/dev/null |
-  $PYTHON $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
+  python $ROOT/coverage/parse_gcov_output.py -interested-files $LATEST_FILES |
   tee -a $RECENT_REPORT &&
 echo -e "Generated coverage report for recently updated files: $RECENT_REPORT\n"
 

coverage/parse_gcov_output.py

@@ -1,12 +1,9 @@
-#!/usr/bin/env python
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-
-from __future__ import print_function
-
 import optparse
 import re
 import sys
 
+from optparse import OptionParser
+
 # the gcov report follows certain pattern. Each file will have two lines
 # of report, from which we can extract the file name, total lines and coverage
 # percentage.
@@ -50,7 +47,7 @@ def parse_gcov_report(gcov_input):
 def get_option_parser():
     usage = "Parse the gcov output and generate more human-readable code " +\
             "coverage report."
-    parser = optparse.OptionParser(usage)
+    parser = OptionParser(usage)
 
     parser.add_option(
         "--interested-files", "-i",
@@ -75,8 +72,8 @@ def display_file_coverage(per_file_coverage, total_coverage):
     header_template = \
         "%" + str(max_file_name_length) + "s\t%s\t%s"
     separator = "-" * (max_file_name_length + 10 + 20)
-    print(header_template % ("Filename", "Coverage", "Lines"))  # noqa: E999 T25377293 Grandfathered in
-    print(separator)
+    print header_template % ("Filename", "Coverage", "Lines")
+    print separator
 
     # -- Print body
     # template for printing coverage report for each file.
@@ -84,12 +81,12 @@ def display_file_coverage(per_file_coverage, total_coverage):
 
     for fname, coverage_info in per_file_coverage.items():
         coverage, lines = coverage_info
-        print(record_template % (fname, coverage, lines))
+        print record_template % (fname, coverage, lines)
 
     # -- Print footer
     if total_coverage:
-        print(separator)
-        print(record_template % ("Total", total_coverage[0], total_coverage[1]))
+        print separator
+        print record_template % ("Total", total_coverage[0], total_coverage[1])
 
 def report_coverage():
     parser = get_option_parser()
@@ -113,7 +110,7 @@ def report_coverage():
         total_coverage = None
 
     if not len(per_file_coverage):
-        print("Cannot find coverage info for the given files.", file=sys.stderr)
+        print >> sys.stderr, "Cannot find coverage info for the given files."
         return
     display_file_coverage(per_file_coverage, total_coverage)
 

db/arena_wrapped_db_iter.cc

@@ -1,108 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#include "db/arena_wrapped_db_iter.h"
-#include "memory/arena.h"
-#include "rocksdb/env.h"
-#include "rocksdb/iterator.h"
-#include "rocksdb/options.h"
-#include "table/internal_iterator.h"
-#include "table/iterator_wrapper.h"
-#include "util/user_comparator_wrapper.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status ArenaWrappedDBIter::GetProperty(std::string prop_name,
-                                       std::string* prop) {
-  if (prop_name == "rocksdb.iterator.super-version-number") {
-    // First try to pass the value returned from inner iterator.
-    if (!db_iter_->GetProperty(prop_name, prop).ok()) {
-      *prop = ToString(sv_number_);
-    }
-    return Status::OK();
-  }
-  return db_iter_->GetProperty(prop_name, prop);
-}
-
-void ArenaWrappedDBIter::Init(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iteration,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
-  auto mem = arena_.AllocateAligned(sizeof(DBIter));
-  db_iter_ =
-      new (mem) DBIter(env, read_options, cf_options, mutable_cf_options,
-                       cf_options.user_comparator, /* iter */ nullptr, version,
-                       sequence, true, max_sequential_skip_in_iteration,
-                       read_callback, db_impl, cfd, expose_blob_index);
-  sv_number_ = version_number;
-  read_options_ = read_options;
-  allow_refresh_ = allow_refresh;
-}
-
-Status ArenaWrappedDBIter::Refresh() {
-  if (cfd_ == nullptr || db_impl_ == nullptr || !allow_refresh_) {
-    return Status::NotSupported("Creating renew iterator is not allowed.");
-  }
-  assert(db_iter_ != nullptr);
-  // TODO(yiwu): For last_seq_same_as_publish_seq_==false, this is not the
-  // correct behavior. Will be corrected automatically when we take a snapshot
-  // here for the case of WritePreparedTxnDB.
-  uint64_t cur_sv_number = cfd_->GetSuperVersionNumber();
-  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:1");
-  TEST_SYNC_POINT("ArenaWrappedDBIter::Refresh:2");
-  if (sv_number_ != cur_sv_number) {
-    Env* env = db_iter_->env();
-    db_iter_->~DBIter();
-    arena_.~Arena();
-    new (&arena_) Arena();
-
-    SuperVersion* sv = cfd_->GetReferencedSuperVersion(db_impl_);
-    SequenceNumber latest_seq = db_impl_->GetLatestSequenceNumber();
-    if (read_callback_) {
-      read_callback_->Refresh(latest_seq);
-    }
-    Init(env, read_options_, *(cfd_->ioptions()), sv->mutable_cf_options,
-         sv->current, latest_seq,
-         sv->mutable_cf_options.max_sequential_skip_in_iterations,
-         cur_sv_number, read_callback_, db_impl_, cfd_, expose_blob_index_,
-         allow_refresh_);
-
-    InternalIterator* internal_iter = db_impl_->NewInternalIterator(
-        read_options_, cfd_, sv, &arena_, db_iter_->GetRangeDelAggregator(),
-        latest_seq, /* allow_unprepared_value */ true);
-    SetIterUnderDBIter(internal_iter);
-  } else {
-    db_iter_->set_sequence(db_impl_->GetLatestSequenceNumber());
-    db_iter_->set_valid(false);
-  }
-  return Status::OK();
-}
-
-ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback, DBImpl* db_impl,
-    ColumnFamilyData* cfd, bool expose_blob_index, bool allow_refresh) {
-  ArenaWrappedDBIter* iter = new ArenaWrappedDBIter();
-  iter->Init(env, read_options, cf_options, mutable_cf_options, version,
-             sequence, max_sequential_skip_in_iterations, version_number,
-             read_callback, db_impl, cfd, expose_blob_index, allow_refresh);
-  if (db_impl != nullptr && cfd != nullptr && allow_refresh) {
-    iter->StoreRefreshInfo(db_impl, cfd, read_callback, expose_blob_index);
-  }
-
-  return iter;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/arena_wrapped_db_iter.h

@@ -1,115 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-
-#pragma once
-#include <stdint.h>
-#include <string>
-#include "db/db_impl/db_impl.h"
-#include "db/db_iter.h"
-#include "db/dbformat.h"
-#include "db/range_del_aggregator.h"
-#include "memory/arena.h"
-#include "options/cf_options.h"
-#include "rocksdb/db.h"
-#include "rocksdb/iterator.h"
-#include "util/autovector.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Arena;
-class Version;
-
-// A wrapper iterator which wraps DB Iterator and the arena, with which the DB
-// iterator is supposed to be allocated. This class is used as an entry point of
-// a iterator hierarchy whose memory can be allocated inline. In that way,
-// accessing the iterator tree can be more cache friendly. It is also faster
-// to allocate.
-// When using the class's Iterator interface, the behavior is exactly
-// the same as the inner DBIter.
-class ArenaWrappedDBIter : public Iterator {
- public:
-  virtual ~ArenaWrappedDBIter() { db_iter_->~DBIter(); }
-
-  // Get the arena to be used to allocate memory for DBIter to be wrapped,
-  // as well as child iterators in it.
-  virtual Arena* GetArena() { return &arena_; }
-  virtual ReadRangeDelAggregator* GetRangeDelAggregator() {
-    return db_iter_->GetRangeDelAggregator();
-  }
-  const ReadOptions& GetReadOptions() { return read_options_; }
-
-  // Set the internal iterator wrapped inside the DB Iterator. Usually it is
-  // a merging iterator.
-  virtual void SetIterUnderDBIter(InternalIterator* iter) {
-    db_iter_->SetIter(iter);
-  }
-
-  bool Valid() const override { return db_iter_->Valid(); }
-  void SeekToFirst() override { db_iter_->SeekToFirst(); }
-  void SeekToLast() override { db_iter_->SeekToLast(); }
-  // 'target' does not contain timestamp, even if user timestamp feature is
-  // enabled.
-  void Seek(const Slice& target) override { db_iter_->Seek(target); }
-  void SeekForPrev(const Slice& target) override {
-    db_iter_->SeekForPrev(target);
-  }
-  void Next() override { db_iter_->Next(); }
-  void Prev() override { db_iter_->Prev(); }
-  Slice key() const override { return db_iter_->key(); }
-  Slice value() const override { return db_iter_->value(); }
-  Status status() const override { return db_iter_->status(); }
-  Slice timestamp() const override { return db_iter_->timestamp(); }
-  bool IsBlob() const { return db_iter_->IsBlob(); }
-
-  Status GetProperty(std::string prop_name, std::string* prop) override;
-
-  Status Refresh() override;
-
-  void Init(Env* env, const ReadOptions& read_options,
-            const ImmutableCFOptions& cf_options,
-            const MutableCFOptions& mutable_cf_options, const Version* version,
-            const SequenceNumber& sequence,
-            uint64_t max_sequential_skip_in_iterations, uint64_t version_number,
-            ReadCallback* read_callback, DBImpl* db_impl, ColumnFamilyData* cfd,
-            bool expose_blob_index, bool allow_refresh);
-
-  // Store some parameters so we can refresh the iterator at a later point
-  // with these same params
-  void StoreRefreshInfo(DBImpl* db_impl, ColumnFamilyData* cfd,
-                        ReadCallback* read_callback, bool expose_blob_index) {
-    db_impl_ = db_impl;
-    cfd_ = cfd;
-    read_callback_ = read_callback;
-    expose_blob_index_ = expose_blob_index;
-  }
-
- private:
-  DBIter* db_iter_;
-  Arena arena_;
-  uint64_t sv_number_;
-  ColumnFamilyData* cfd_ = nullptr;
-  DBImpl* db_impl_ = nullptr;
-  ReadOptions read_options_;
-  ReadCallback* read_callback_;
-  bool expose_blob_index_ = false;
-  bool allow_refresh_ = true;
-};
-
-// Generate the arena wrapped iterator class.
-// `db_impl` and `cfd` are used for reneweal. If left null, renewal will not
-// be supported.
-extern ArenaWrappedDBIter* NewArenaWrappedDbIterator(
-    Env* env, const ReadOptions& read_options,
-    const ImmutableCFOptions& cf_options,
-    const MutableCFOptions& mutable_cf_options, const Version* version,
-    const SequenceNumber& sequence, uint64_t max_sequential_skip_in_iterations,
-    uint64_t version_number, ReadCallback* read_callback,
-    DBImpl* db_impl = nullptr, ColumnFamilyData* cfd = nullptr,
-    bool expose_blob_index = false, bool allow_refresh = true);
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_constants.h

@@ -1,16 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cstdint>
-
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-constexpr uint64_t kInvalidBlobFileNumber = 0;
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition.cc

@@ -1,154 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_addition.h"
-
-#include <ostream>
-#include <sstream>
-
-#include "logging/event_logger.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Tags for custom fields. Note that these get persisted in the manifest,
-// so existing tags should not be modified.
-enum BlobFileAddition::CustomFieldTags : uint32_t {
-  kEndMarker,
-
-  // Add forward compatible fields here
-
-  /////////////////////////////////////////////////////////////////////
-
-  kForwardIncompatibleMask = 1 << 6,
-
-  // Add forward incompatible fields here
-};
-
-void BlobFileAddition::EncodeTo(std::string* output) const {
-  PutVarint64(output, blob_file_number_);
-  PutVarint64(output, total_blob_count_);
-  PutVarint64(output, total_blob_bytes_);
-  PutLengthPrefixedSlice(output, checksum_method_);
-  PutLengthPrefixedSlice(output, checksum_value_);
-
-  // Encode any custom fields here. The format to use is a Varint32 tag (see
-  // CustomFieldTags above) followed by a length prefixed slice. Unknown custom
-  // fields will be ignored during decoding unless they're in the forward
-  // incompatible range.
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileAddition::EncodeTo::CustomFields", output);
-
-  PutVarint32(output, kEndMarker);
-}
-
-Status BlobFileAddition::DecodeFrom(Slice* input) {
-  constexpr char class_name[] = "BlobFileAddition";
-
-  if (!GetVarint64(input, &blob_file_number_)) {
-    return Status::Corruption(class_name, "Error decoding blob file number");
-  }
-
-  if (!GetVarint64(input, &total_blob_count_)) {
-    return Status::Corruption(class_name, "Error decoding total blob count");
-  }
-
-  if (!GetVarint64(input, &total_blob_bytes_)) {
-    return Status::Corruption(class_name, "Error decoding total blob bytes");
-  }
-
-  Slice checksum_method;
-  if (!GetLengthPrefixedSlice(input, &checksum_method)) {
-    return Status::Corruption(class_name, "Error decoding checksum method");
-  }
-  checksum_method_ = checksum_method.ToString();
-
-  Slice checksum_value;
-  if (!GetLengthPrefixedSlice(input, &checksum_value)) {
-    return Status::Corruption(class_name, "Error decoding checksum value");
-  }
-  checksum_value_ = checksum_value.ToString();
-
-  while (true) {
-    uint32_t custom_field_tag = 0;
-    if (!GetVarint32(input, &custom_field_tag)) {
-      return Status::Corruption(class_name, "Error decoding custom field tag");
-    }
-
-    if (custom_field_tag == kEndMarker) {
-      break;
-    }
-
-    if (custom_field_tag & kForwardIncompatibleMask) {
-      return Status::Corruption(
-          class_name, "Forward incompatible custom field encountered");
-    }
-
-    Slice custom_field_value;
-    if (!GetLengthPrefixedSlice(input, &custom_field_value)) {
-      return Status::Corruption(class_name,
-                                "Error decoding custom field value");
-    }
-  }
-
-  return Status::OK();
-}
-
-std::string BlobFileAddition::DebugString() const {
-  std::ostringstream oss;
-
-  oss << *this;
-
-  return oss.str();
-}
-
-std::string BlobFileAddition::DebugJSON() const {
-  JSONWriter jw;
-
-  jw << *this;
-
-  jw.EndObject();
-
-  return jw.Get();
-}
-
-bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs) {
-  return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() &&
-         lhs.GetTotalBlobCount() == rhs.GetTotalBlobCount() &&
-         lhs.GetTotalBlobBytes() == rhs.GetTotalBlobBytes() &&
-         lhs.GetChecksumMethod() == rhs.GetChecksumMethod() &&
-         lhs.GetChecksumValue() == rhs.GetChecksumValue();
-}
-
-bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs) {
-  return !(lhs == rhs);
-}
-
-std::ostream& operator<<(std::ostream& os,
-                         const BlobFileAddition& blob_file_addition) {
-  os << "blob_file_number: " << blob_file_addition.GetBlobFileNumber()
-     << " total_blob_count: " << blob_file_addition.GetTotalBlobCount()
-     << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes()
-     << " checksum_method: " << blob_file_addition.GetChecksumMethod()
-     << " checksum_value: " << blob_file_addition.GetChecksumValue();
-
-  return os;
-}
-
-JSONWriter& operator<<(JSONWriter& jw,
-                       const BlobFileAddition& blob_file_addition) {
-  jw << "BlobFileNumber" << blob_file_addition.GetBlobFileNumber()
-     << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount()
-     << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes()
-     << "ChecksumMethod" << blob_file_addition.GetChecksumMethod()
-     << "ChecksumValue" << blob_file_addition.GetChecksumValue();
-
-  return jw;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition.h

@@ -1,67 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-#include <cstdint>
-#include <iosfwd>
-#include <string>
-
-#include "db/blob/blob_constants.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class JSONWriter;
-class Slice;
-class Status;
-
-class BlobFileAddition {
- public:
-  BlobFileAddition() = default;
-
-  BlobFileAddition(uint64_t blob_file_number, uint64_t total_blob_count,
-                   uint64_t total_blob_bytes, std::string checksum_method,
-                   std::string checksum_value)
-      : blob_file_number_(blob_file_number),
-        total_blob_count_(total_blob_count),
-        total_blob_bytes_(total_blob_bytes),
-        checksum_method_(std::move(checksum_method)),
-        checksum_value_(std::move(checksum_value)) {
-    assert(checksum_method_.empty() == checksum_value_.empty());
-  }
-
-  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
-  uint64_t GetTotalBlobCount() const { return total_blob_count_; }
-  uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; }
-  const std::string& GetChecksumMethod() const { return checksum_method_; }
-  const std::string& GetChecksumValue() const { return checksum_value_; }
-
-  void EncodeTo(std::string* output) const;
-  Status DecodeFrom(Slice* input);
-
-  std::string DebugString() const;
-  std::string DebugJSON() const;
-
- private:
-  enum CustomFieldTags : uint32_t;
-
-  uint64_t blob_file_number_ = kInvalidBlobFileNumber;
-  uint64_t total_blob_count_ = 0;
-  uint64_t total_blob_bytes_ = 0;
-  std::string checksum_method_;
-  std::string checksum_value_;
-};
-
-bool operator==(const BlobFileAddition& lhs, const BlobFileAddition& rhs);
-bool operator!=(const BlobFileAddition& lhs, const BlobFileAddition& rhs);
-
-std::ostream& operator<<(std::ostream& os,
-                         const BlobFileAddition& blob_file_addition);
-JSONWriter& operator<<(JSONWriter& jw,
-                       const BlobFileAddition& blob_file_addition);
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_addition_test.cc

@@ -1,206 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_addition.h"
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlobFileAdditionTest : public testing::Test {
- public:
-  static void TestEncodeDecode(const BlobFileAddition& blob_file_addition) {
-    std::string encoded;
-    blob_file_addition.EncodeTo(&encoded);
-
-    BlobFileAddition decoded;
-    Slice input(encoded);
-    ASSERT_OK(decoded.DecodeFrom(&input));
-
-    ASSERT_EQ(blob_file_addition, decoded);
-  }
-};
-
-TEST_F(BlobFileAdditionTest, Empty) {
-  BlobFileAddition blob_file_addition;
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), kInvalidBlobFileNumber);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 0);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), 0);
-  ASSERT_TRUE(blob_file_addition.GetChecksumMethod().empty());
-  ASSERT_TRUE(blob_file_addition.GetChecksumValue().empty());
-
-  TestEncodeDecode(blob_file_addition);
-}
-
-TEST_F(BlobFileAdditionTest, NonEmpty) {
-  constexpr uint64_t blob_file_number = 123;
-  constexpr uint64_t total_blob_count = 2;
-  constexpr uint64_t total_blob_bytes = 123456;
-  const std::string checksum_method("SHA1");
-  const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd");
-
-  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
-                                      total_blob_bytes, checksum_method,
-                                      checksum_value);
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), total_blob_count);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(), total_blob_bytes);
-  ASSERT_EQ(blob_file_addition.GetChecksumMethod(), checksum_method);
-  ASSERT_EQ(blob_file_addition.GetChecksumValue(), checksum_value);
-
-  TestEncodeDecode(blob_file_addition);
-}
-
-TEST_F(BlobFileAdditionTest, DecodeErrors) {
-  std::string str;
-  Slice slice(str);
-
-  BlobFileAddition blob_file_addition;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "blob file number"));
-  }
-
-  constexpr uint64_t blob_file_number = 123;
-  PutVarint64(&str, blob_file_number);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "total blob count"));
-  }
-
-  constexpr uint64_t total_blob_count = 4567;
-  PutVarint64(&str, total_blob_count);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "total blob bytes"));
-  }
-
-  constexpr uint64_t total_blob_bytes = 12345678;
-  PutVarint64(&str, total_blob_bytes);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "checksum method"));
-  }
-
-  constexpr char checksum_method[] = "SHA1";
-  PutLengthPrefixedSlice(&str, checksum_method);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "checksum value"));
-  }
-
-  constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd";
-  PutLengthPrefixedSlice(&str, checksum_value);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "custom field tag"));
-  }
-
-  constexpr uint32_t custom_tag = 2;
-  PutVarint32(&str, custom_tag);
-  slice = str;
-
-  {
-    const Status s = blob_file_addition.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "custom field value"));
-  }
-}
-
-TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) {
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) {
-        std::string* output = static_cast<std::string*>(arg);
-
-        constexpr uint32_t forward_compatible_tag = 2;
-        PutVarint32(output, forward_compatible_tag);
-
-        PutLengthPrefixedSlice(output, "deadbeef");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr uint64_t blob_file_number = 678;
-  constexpr uint64_t total_blob_count = 9999;
-  constexpr uint64_t total_blob_bytes = 100000000;
-  const std::string checksum_method("CRC32");
-  const std::string checksum_value("3d87ff57");
-
-  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
-                                      total_blob_bytes, checksum_method,
-                                      checksum_value);
-
-  TestEncodeDecode(blob_file_addition);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) {
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileAddition::EncodeTo::CustomFields", [&](void* arg) {
-        std::string* output = static_cast<std::string*>(arg);
-
-        constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1;
-        PutVarint32(output, forward_incompatible_tag);
-
-        PutLengthPrefixedSlice(output, "foobar");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr uint64_t blob_file_number = 456;
-  constexpr uint64_t total_blob_count = 100;
-  constexpr uint64_t total_blob_bytes = 2000000;
-  const std::string checksum_method("CRC32B");
-  const std::string checksum_value("6dbdf23a");
-
-  BlobFileAddition blob_file_addition(blob_file_number, total_blob_count,
-                                      total_blob_bytes, checksum_method,
-                                      checksum_value);
-
-  std::string encoded;
-  blob_file_addition.EncodeTo(&encoded);
-
-  BlobFileAddition decoded_blob_file_addition;
-  Slice input(encoded);
-  const Status s = decoded_blob_file_addition.DecodeFrom(&input);
-
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible"));
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_builder.cc

@@ -1,350 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_builder.h"
-
-#include <cassert>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_file_completion_callback.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "db/version_set.h"
-#include "file/filename.h"
-#include "file/read_write_util.h"
-#include "file/writable_file_writer.h"
-#include "logging/logging.h"
-#include "options/cf_options.h"
-#include "options/options_helper.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "trace_replay/io_tracer.h"
-#include "util/compression.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobFileBuilder::BlobFileBuilder(
-    VersionSet* versions, FileSystem* fs,
-    const ImmutableCFOptions* immutable_cf_options,
-    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCompletionCallback* blob_callback,
-    std::vector<std::string>* blob_file_paths,
-    std::vector<BlobFileAddition>* blob_file_additions)
-    : BlobFileBuilder([versions]() { return versions->NewFileNumber(); }, fs,
-                      immutable_cf_options, mutable_cf_options, file_options,
-                      job_id, column_family_id, column_family_name, io_priority,
-                      write_hint, io_tracer, blob_callback, blob_file_paths,
-                      blob_file_additions) {}
-
-BlobFileBuilder::BlobFileBuilder(
-    std::function<uint64_t()> file_number_generator, FileSystem* fs,
-    const ImmutableCFOptions* immutable_cf_options,
-    const MutableCFOptions* mutable_cf_options, const FileOptions* file_options,
-    int job_id, uint32_t column_family_id,
-    const std::string& column_family_name, Env::IOPriority io_priority,
-    Env::WriteLifeTimeHint write_hint,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    BlobFileCompletionCallback* blob_callback,
-    std::vector<std::string>* blob_file_paths,
-    std::vector<BlobFileAddition>* blob_file_additions)
-    : file_number_generator_(std::move(file_number_generator)),
-      fs_(fs),
-      immutable_cf_options_(immutable_cf_options),
-      min_blob_size_(mutable_cf_options->min_blob_size),
-      blob_file_size_(mutable_cf_options->blob_file_size),
-      blob_compression_type_(mutable_cf_options->blob_compression_type),
-      file_options_(file_options),
-      job_id_(job_id),
-      column_family_id_(column_family_id),
-      column_family_name_(column_family_name),
-      io_priority_(io_priority),
-      write_hint_(write_hint),
-      io_tracer_(io_tracer),
-      blob_callback_(blob_callback),
-      blob_file_paths_(blob_file_paths),
-      blob_file_additions_(blob_file_additions),
-      blob_count_(0),
-      blob_bytes_(0) {
-  assert(file_number_generator_);
-  assert(fs_);
-  assert(immutable_cf_options_);
-  assert(file_options_);
-  assert(blob_file_paths_);
-  assert(blob_file_paths_->empty());
-  assert(blob_file_additions_);
-  assert(blob_file_additions_->empty());
-}
-
-BlobFileBuilder::~BlobFileBuilder() = default;
-
-Status BlobFileBuilder::Add(const Slice& key, const Slice& value,
-                            std::string* blob_index) {
-  assert(blob_index);
-  assert(blob_index->empty());
-
-  if (value.size() < min_blob_size_) {
-    return Status::OK();
-  }
-
-  {
-    const Status s = OpenBlobFileIfNeeded();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  Slice blob = value;
-  std::string compressed_blob;
-
-  {
-    const Status s = CompressBlobIfNeeded(&blob, &compressed_blob);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  uint64_t blob_file_number = 0;
-  uint64_t blob_offset = 0;
-
-  {
-    const Status s =
-        WriteBlobToFile(key, blob, &blob_file_number, &blob_offset);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  {
-    const Status s = CloseBlobFileIfNeeded();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  BlobIndex::EncodeBlob(blob_index, blob_file_number, blob_offset, blob.size(),
-                        blob_compression_type_);
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::Finish() {
-  if (!IsBlobFileOpen()) {
-    return Status::OK();
-  }
-
-  return CloseBlobFile();
-}
-
-bool BlobFileBuilder::IsBlobFileOpen() const { return !!writer_; }
-
-Status BlobFileBuilder::OpenBlobFileIfNeeded() {
-  if (IsBlobFileOpen()) {
-    return Status::OK();
-  }
-
-  assert(!blob_count_);
-  assert(!blob_bytes_);
-
-  assert(file_number_generator_);
-  const uint64_t blob_file_number = file_number_generator_();
-
-  assert(immutable_cf_options_);
-  assert(!immutable_cf_options_->cf_paths.empty());
-  std::string blob_file_path = BlobFileName(
-      immutable_cf_options_->cf_paths.front().path, blob_file_number);
-
-  std::unique_ptr<FSWritableFile> file;
-
-  {
-    assert(file_options_);
-    Status s = NewWritableFile(fs_, blob_file_path, &file, *file_options_);
-
-    TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile", &s);
-
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  // Note: files get added to blob_file_paths_ right after the open, so they
-  // can be cleaned up upon failure. Contrast this with blob_file_additions_,
-  // which only contains successfully written files.
-  assert(blob_file_paths_);
-  blob_file_paths_->emplace_back(std::move(blob_file_path));
-
-  assert(file);
-  file->SetIOPriority(io_priority_);
-  file->SetWriteLifeTimeHint(write_hint_);
-  FileTypeSet tmp_set = immutable_cf_options_->checksum_handoff_file_types;
-  Statistics* const statistics = immutable_cf_options_->statistics;
-  std::unique_ptr<WritableFileWriter> file_writer(new WritableFileWriter(
-      std::move(file), blob_file_paths_->back(), *file_options_,
-      immutable_cf_options_->clock, io_tracer_, statistics,
-      immutable_cf_options_->listeners,
-      immutable_cf_options_->file_checksum_gen_factory,
-      tmp_set.Contains(FileType::kBlobFile)));
-
-  constexpr bool do_flush = false;
-
-  std::unique_ptr<BlobLogWriter> blob_log_writer(new BlobLogWriter(
-      std::move(file_writer), immutable_cf_options_->clock, statistics,
-      blob_file_number, immutable_cf_options_->use_fsync, do_flush));
-
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-
-  BlobLogHeader header(column_family_id_, blob_compression_type_, has_ttl,
-                       expiration_range);
-
-  {
-    Status s = blob_log_writer->WriteHeader(header);
-
-    TEST_SYNC_POINT_CALLBACK(
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader", &s);
-
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  writer_ = std::move(blob_log_writer);
-
-  assert(IsBlobFileOpen());
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::CompressBlobIfNeeded(
-    Slice* blob, std::string* compressed_blob) const {
-  assert(blob);
-  assert(compressed_blob);
-  assert(compressed_blob->empty());
-
-  if (blob_compression_type_ == kNoCompression) {
-    return Status::OK();
-  }
-
-  CompressionOptions opts;
-  CompressionContext context(blob_compression_type_);
-  constexpr uint64_t sample_for_compression = 0;
-
-  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                       blob_compression_type_, sample_for_compression);
-
-  constexpr uint32_t compression_format_version = 2;
-
-  if (!CompressData(*blob, info, compression_format_version, compressed_blob)) {
-    return Status::Corruption("Error compressing blob");
-  }
-
-  *blob = Slice(*compressed_blob);
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::WriteBlobToFile(const Slice& key, const Slice& blob,
-                                        uint64_t* blob_file_number,
-                                        uint64_t* blob_offset) {
-  assert(IsBlobFileOpen());
-  assert(blob_file_number);
-  assert(blob_offset);
-
-  uint64_t key_offset = 0;
-
-  Status s = writer_->AddRecord(key, blob, &key_offset, blob_offset);
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AddRecord", &s);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  *blob_file_number = writer_->get_log_number();
-
-  ++blob_count_;
-  blob_bytes_ += BlobLogRecord::kHeaderSize + key.size() + blob.size();
-
-  return Status::OK();
-}
-
-Status BlobFileBuilder::CloseBlobFile() {
-  assert(IsBlobFileOpen());
-
-  BlobLogFooter footer;
-  footer.blob_count = blob_count_;
-
-  std::string checksum_method;
-  std::string checksum_value;
-
-  Status s = writer_->AppendFooter(footer, &checksum_method, &checksum_value);
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileBuilder::WriteBlobToFile:AppendFooter", &s);
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  const uint64_t blob_file_number = writer_->get_log_number();
-
-  assert(blob_file_additions_);
-  blob_file_additions_->emplace_back(blob_file_number, blob_count_, blob_bytes_,
-                                     std::move(checksum_method),
-                                     std::move(checksum_value));
-
-  assert(immutable_cf_options_);
-  ROCKS_LOG_INFO(immutable_cf_options_->info_log,
-                 "[%s] [JOB %d] Generated blob file #%" PRIu64 ": %" PRIu64
-                 " total blobs, %" PRIu64 " total bytes",
-                 column_family_name_.c_str(), job_id_, blob_file_number,
-                 blob_count_, blob_bytes_);
-
-  if (blob_callback_) {
-    s = blob_callback_->OnBlobFileCompleted(blob_file_paths_->back());
-  }
-
-  writer_.reset();
-  blob_count_ = 0;
-  blob_bytes_ = 0;
-
-  return s;
-}
-
-Status BlobFileBuilder::CloseBlobFileIfNeeded() {
-  assert(IsBlobFileOpen());
-
-  const WritableFileWriter* const file_writer = writer_->file();
-  assert(file_writer);
-
-  if (file_writer->GetFileSize() < blob_file_size_) {
-    return Status::OK();
-  }
-
-  return CloseBlobFile();
-}
-
-void BlobFileBuilder::Abandon() {
-  if (!IsBlobFileOpen()) {
-    return;
-  }
-
-  if (blob_callback_) {
-    // BlobFileBuilder::Abandon() is called because of error while writing to
-    // Blob files. So we can ignore the below error.
-    blob_callback_->OnBlobFileCompleted(blob_file_paths_->back())
-        .PermitUncheckedError();
-  }
-
-  writer_.reset();
-  blob_count_ = 0;
-  blob_bytes_ = 0;
-}
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder.h

@@ -1,100 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-#pragma once
-
-#include <cinttypes>
-#include <functional>
-#include <memory>
-#include <string>
-#include <vector>
-
-#include "rocksdb/compression_type.h"
-#include "rocksdb/env.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class VersionSet;
-class FileSystem;
-class SystemClock;
-struct ImmutableCFOptions;
-struct MutableCFOptions;
-struct FileOptions;
-class BlobFileAddition;
-class Status;
-class Slice;
-class BlobLogWriter;
-class IOTracer;
-class BlobFileCompletionCallback;
-
-class BlobFileBuilder {
- public:
-  BlobFileBuilder(VersionSet* versions, FileSystem* fs,
-                  const ImmutableCFOptions* immutable_cf_options,
-                  const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
-                  uint32_t column_family_id,
-                  const std::string& column_family_name,
-                  Env::IOPriority io_priority,
-                  Env::WriteLifeTimeHint write_hint,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  BlobFileCompletionCallback* blob_callback,
-                  std::vector<std::string>* blob_file_paths,
-                  std::vector<BlobFileAddition>* blob_file_additions);
-
-  BlobFileBuilder(std::function<uint64_t()> file_number_generator,
-                  FileSystem* fs,
-                  const ImmutableCFOptions* immutable_cf_options,
-                  const MutableCFOptions* mutable_cf_options,
-                  const FileOptions* file_options, int job_id,
-                  uint32_t column_family_id,
-                  const std::string& column_family_name,
-                  Env::IOPriority io_priority,
-                  Env::WriteLifeTimeHint write_hint,
-                  const std::shared_ptr<IOTracer>& io_tracer,
-                  BlobFileCompletionCallback* blob_callback,
-                  std::vector<std::string>* blob_file_paths,
-                  std::vector<BlobFileAddition>* blob_file_additions);
-
-  BlobFileBuilder(const BlobFileBuilder&) = delete;
-  BlobFileBuilder& operator=(const BlobFileBuilder&) = delete;
-
-  ~BlobFileBuilder();
-
-  Status Add(const Slice& key, const Slice& value, std::string* blob_index);
-  Status Finish();
-  void Abandon();
-
- private:
-  bool IsBlobFileOpen() const;
-  Status OpenBlobFileIfNeeded();
-  Status CompressBlobIfNeeded(Slice* blob, std::string* compressed_blob) const;
-  Status WriteBlobToFile(const Slice& key, const Slice& blob,
-                         uint64_t* blob_file_number, uint64_t* blob_offset);
-  Status CloseBlobFile();
-  Status CloseBlobFileIfNeeded();
-
-  std::function<uint64_t()> file_number_generator_;
-  FileSystem* fs_;
-  const ImmutableCFOptions* immutable_cf_options_;
-  uint64_t min_blob_size_;
-  uint64_t blob_file_size_;
-  CompressionType blob_compression_type_;
-  const FileOptions* file_options_;
-  int job_id_;
-  uint32_t column_family_id_;
-  std::string column_family_name_;
-  Env::IOPriority io_priority_;
-  Env::WriteLifeTimeHint write_hint_;
-  std::shared_ptr<IOTracer> io_tracer_;
-  BlobFileCompletionCallback* blob_callback_;
-  std::vector<std::string>* blob_file_paths_;
-  std::vector<BlobFileAddition>* blob_file_additions_;
-  std::unique_ptr<BlobLogWriter> writer_;
-  uint64_t blob_count_;
-  uint64_t blob_bytes_;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_builder_test.cc

@@ -1,673 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_builder.h"
-
-#include <cassert>
-#include <cinttypes>
-#include <string>
-#include <utility>
-#include <vector>
-
-#include "db/blob/blob_file_addition.h"
-#include "db/blob/blob_index.h"
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_sequential_reader.h"
-#include "env/mock_env.h"
-#include "file/filename.h"
-#include "file/random_access_file_reader.h"
-#include "options/cf_options.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_checksum.h"
-#include "rocksdb/options.h"
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/compression.h"
-#include "utilities/fault_injection_env.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class TestFileNumberGenerator {
- public:
-  uint64_t operator()() { return ++next_file_number_; }
-
- private:
-  uint64_t next_file_number_ = 1;
-};
-
-class BlobFileBuilderTest : public testing::Test {
- protected:
-  BlobFileBuilderTest() : mock_env_(Env::Default()) {
-    fs_ = mock_env_.GetFileSystem().get();
-    clock_ = mock_env_.GetSystemClock().get();
-  }
-
-  void VerifyBlobFile(uint64_t blob_file_number,
-                      const std::string& blob_file_path,
-                      uint32_t column_family_id,
-                      CompressionType blob_compression_type,
-                      const std::vector<std::pair<std::string, std::string>>&
-                          expected_key_value_pairs,
-                      const std::vector<std::string>& blob_indexes) {
-    assert(expected_key_value_pairs.size() == blob_indexes.size());
-
-    std::unique_ptr<FSRandomAccessFile> file;
-    constexpr IODebugContext* dbg = nullptr;
-    ASSERT_OK(
-        fs_->NewRandomAccessFile(blob_file_path, file_options_, &file, dbg));
-
-    std::unique_ptr<RandomAccessFileReader> file_reader(
-        new RandomAccessFileReader(std::move(file), blob_file_path, clock_));
-
-    constexpr Statistics* statistics = nullptr;
-    BlobLogSequentialReader blob_log_reader(std::move(file_reader), clock_,
-                                            statistics);
-
-    BlobLogHeader header;
-    ASSERT_OK(blob_log_reader.ReadHeader(&header));
-    ASSERT_EQ(header.version, kVersion1);
-    ASSERT_EQ(header.column_family_id, column_family_id);
-    ASSERT_EQ(header.compression, blob_compression_type);
-    ASSERT_FALSE(header.has_ttl);
-    ASSERT_EQ(header.expiration_range, ExpirationRange());
-
-    for (size_t i = 0; i < expected_key_value_pairs.size(); ++i) {
-      BlobLogRecord record;
-      uint64_t blob_offset = 0;
-
-      ASSERT_OK(blob_log_reader.ReadRecord(
-          &record, BlobLogSequentialReader::kReadHeaderKeyBlob, &blob_offset));
-
-      // Check the contents of the blob file
-      const auto& expected_key_value = expected_key_value_pairs[i];
-      const auto& key = expected_key_value.first;
-      const auto& value = expected_key_value.second;
-
-      ASSERT_EQ(record.key_size, key.size());
-      ASSERT_EQ(record.value_size, value.size());
-      ASSERT_EQ(record.expiration, 0);
-      ASSERT_EQ(record.key, key);
-      ASSERT_EQ(record.value, value);
-
-      // Make sure the blob reference returned by the builder points to the
-      // right place
-      BlobIndex blob_index;
-      ASSERT_OK(blob_index.DecodeFrom(blob_indexes[i]));
-      ASSERT_FALSE(blob_index.IsInlined());
-      ASSERT_FALSE(blob_index.HasTTL());
-      ASSERT_EQ(blob_index.file_number(), blob_file_number);
-      ASSERT_EQ(blob_index.offset(), blob_offset);
-      ASSERT_EQ(blob_index.size(), value.size());
-    }
-
-    BlobLogFooter footer;
-    ASSERT_OK(blob_log_reader.ReadFooter(&footer));
-    ASSERT_EQ(footer.blob_count, expected_key_value_pairs.size());
-    ASSERT_EQ(footer.expiration_range, ExpirationRange());
-  }
-
-  MockEnv mock_env_;
-  FileSystem* fs_;
-  SystemClock* clock_;
-  FileOptions file_options_;
-};
-
-TEST_F(BlobFileBuilderTest, BuildAndCheckOneFile) {
-  // Build a single blob file
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 4;
-  constexpr size_t value_offset = 1234;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileBuilderTest_BuildAndCheckOneFile"),
-      0);
-  options.enable_blob_files = true;
-  options.env = &mock_env_;
-
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
-      number_of_blobs);
-  std::vector<std::string> blob_indexes(number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    auto& expected_key_value = expected_key_value_pairs[i];
-
-    auto& key = expected_key_value.first;
-    key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    auto& value = expected_key_value.second;
-    value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    auto& blob_index = blob_indexes[i];
-
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_FALSE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(blob_file_path,
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), number_of_blobs);
-  ASSERT_EQ(
-      blob_file_addition.GetTotalBlobBytes(),
-      number_of_blobs * (BlobLogRecord::kHeaderSize + key_size + value_size));
-
-  // Verify the contents of the new blob file as well as the blob references
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kNoCompression, expected_key_value_pairs, blob_indexes);
-}
-
-TEST_F(BlobFileBuilderTest, BuildAndCheckMultipleFiles) {
-  // Build multiple blob files: file size limit is set to the size of a single
-  // value, so each blob ends up in a file of its own
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 10;
-  constexpr size_t value_offset = 1234567890;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileBuilderTest_BuildAndCheckMultipleFiles"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_file_size = value_size;
-  options.env = &mock_env_;
-
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs(
-      number_of_blobs);
-  std::vector<std::string> blob_indexes(number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    auto& expected_key_value = expected_key_value_pairs[i];
-
-    auto& key = expected_key_value.first;
-    key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    auto& value = expected_key_value.second;
-    value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    auto& blob_index = blob_indexes[i];
-
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_FALSE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  ASSERT_EQ(blob_file_paths.size(), number_of_blobs);
-  ASSERT_EQ(blob_file_additions.size(), number_of_blobs);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    const uint64_t blob_file_number = i + 2;
-
-    ASSERT_EQ(blob_file_paths[i],
-              BlobFileName(immutable_cf_options.cf_paths.front().path,
-                           blob_file_number));
-
-    const auto& blob_file_addition = blob_file_additions[i];
-
-    ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-    ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-    ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-              BlobLogRecord::kHeaderSize + key_size + value_size);
-  }
-
-  // Verify the contents of the new blob files as well as the blob references
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    std::vector<std::pair<std::string, std::string>> expected_key_value_pair{
-        expected_key_value_pairs[i]};
-    std::vector<std::string> blob_index{blob_indexes[i]};
-
-    VerifyBlobFile(i + 2, blob_file_paths[i], column_family_id, kNoCompression,
-                   expected_key_value_pair, blob_index);
-  }
-}
-
-TEST_F(BlobFileBuilderTest, InlinedValues) {
-  // All values are below the min_blob_size threshold; no blob files get written
-  constexpr size_t number_of_blobs = 10;
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 10;
-  constexpr size_t value_offset = 1234567890;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_InlinedValues"),
-      0);
-  options.enable_blob_files = true;
-  options.min_blob_size = 1024;
-  options.env = &mock_env_;
-
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  for (size_t i = 0; i < number_of_blobs; ++i) {
-    const std::string key = std::to_string(i);
-    assert(key.size() == key_size);
-
-    const std::string value = std::to_string(i + value_offset);
-    assert(value.size() == value_size);
-
-    std::string blob_index;
-    ASSERT_OK(builder.Add(key, value, &blob_index));
-    ASSERT_TRUE(blob_index.empty());
-  }
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  ASSERT_TRUE(blob_file_paths.empty());
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-TEST_F(BlobFileBuilderTest, Compression) {
-  // Build a blob file with a compressed blob
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  constexpr size_t key_size = 1;
-  constexpr size_t value_size = 100;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_Compression"), 0);
-  options.enable_blob_files = true;
-  options.blob_compression_type = kSnappyCompression;
-  options.env = &mock_env_;
-
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  const std::string key("1");
-  const std::string uncompressed_value(value_size, 'x');
-
-  std::string blob_index;
-
-  ASSERT_OK(builder.Add(key, uncompressed_value, &blob_index));
-  ASSERT_FALSE(blob_index.empty());
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(blob_file_path,
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-
-  CompressionOptions opts;
-  CompressionContext context(kSnappyCompression);
-  constexpr uint64_t sample_for_compression = 0;
-
-  CompressionInfo info(opts, context, CompressionDict::GetEmptyDict(),
-                       kSnappyCompression, sample_for_compression);
-
-  std::string compressed_value;
-  ASSERT_TRUE(Snappy_Compress(info, uncompressed_value.data(),
-                              uncompressed_value.size(), &compressed_value));
-
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-            BlobLogRecord::kHeaderSize + key_size + compressed_value.size());
-
-  // Verify the contents of the new blob file as well as the blob reference
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
-      {key, compressed_value}};
-  std::vector<std::string> blob_indexes{blob_index};
-
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kSnappyCompression, expected_key_value_pairs, blob_indexes);
-}
-
-TEST_F(BlobFileBuilderTest, CompressionError) {
-  // Simulate an error during compression
-  if (!Snappy_Supported()) {
-    return;
-  }
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_CompressionError"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_compression_type = kSnappyCompression;
-  options.env = &mock_env_;
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  SyncPoint::GetInstance()->SetCallBack("CompressData:TamperWithReturnValue",
-                                        [](void* arg) {
-                                          bool* ret = static_cast<bool*>(arg);
-                                          *ret = false;
-                                        });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr char key[] = "1";
-  constexpr char value[] = "deadbeef";
-
-  std::string blob_index;
-
-  ASSERT_TRUE(builder.Add(key, value, &blob_index).IsCorruption());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-  ASSERT_EQ(blob_file_paths[0],
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
-
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-TEST_F(BlobFileBuilderTest, Checksum) {
-  // Build a blob file with checksum
-
-  class DummyFileChecksumGenerator : public FileChecksumGenerator {
-   public:
-    void Update(const char* /* data */, size_t /* n */) override {}
-
-    void Finalize() override {}
-
-    std::string GetChecksum() const override { return std::string("dummy"); }
-
-    const char* Name() const override { return "DummyFileChecksum"; }
-  };
-
-  class DummyFileChecksumGenFactory : public FileChecksumGenFactory {
-   public:
-    std::unique_ptr<FileChecksumGenerator> CreateFileChecksumGenerator(
-        const FileChecksumGenContext& /* context */) override {
-      return std::unique_ptr<FileChecksumGenerator>(
-          new DummyFileChecksumGenerator);
-    }
-
-    const char* Name() const override { return "DummyFileChecksumGenFactory"; }
-  };
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderTest_Checksum"), 0);
-  options.enable_blob_files = true;
-  options.file_checksum_gen_factory =
-      std::make_shared<DummyFileChecksumGenFactory>();
-  options.env = &mock_env_;
-
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  const std::string key("1");
-  const std::string value("deadbeef");
-
-  std::string blob_index;
-
-  ASSERT_OK(builder.Add(key, value, &blob_index));
-  ASSERT_FALSE(blob_index.empty());
-
-  ASSERT_OK(builder.Finish());
-
-  // Check the metadata generated
-  constexpr uint64_t blob_file_number = 2;
-
-  ASSERT_EQ(blob_file_paths.size(), 1);
-
-  const std::string& blob_file_path = blob_file_paths[0];
-
-  ASSERT_EQ(blob_file_path,
-            BlobFileName(immutable_cf_options.cf_paths.front().path,
-                         blob_file_number));
-
-  ASSERT_EQ(blob_file_additions.size(), 1);
-
-  const auto& blob_file_addition = blob_file_additions[0];
-
-  ASSERT_EQ(blob_file_addition.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobCount(), 1);
-  ASSERT_EQ(blob_file_addition.GetTotalBlobBytes(),
-            BlobLogRecord::kHeaderSize + key.size() + value.size());
-  ASSERT_EQ(blob_file_addition.GetChecksumMethod(), "DummyFileChecksum");
-  ASSERT_EQ(blob_file_addition.GetChecksumValue(), "dummy");
-
-  // Verify the contents of the new blob file as well as the blob reference
-  std::vector<std::pair<std::string, std::string>> expected_key_value_pairs{
-      {key, value}};
-  std::vector<std::string> blob_indexes{blob_index};
-
-  VerifyBlobFile(blob_file_number, blob_file_path, column_family_id,
-                 kNoCompression, expected_key_value_pairs, blob_indexes);
-}
-
-class BlobFileBuilderIOErrorTest
-    : public testing::Test,
-      public testing::WithParamInterface<std::string> {
- protected:
-  BlobFileBuilderIOErrorTest()
-      : mock_env_(Env::Default()),
-        fs_(mock_env_.GetFileSystem().get()),
-        sync_point_(GetParam()) {}
-
-  MockEnv mock_env_;
-  FileSystem* fs_;
-  FileOptions file_options_;
-  std::string sync_point_;
-};
-
-INSTANTIATE_TEST_CASE_P(
-    BlobFileBuilderTest, BlobFileBuilderIOErrorTest,
-    ::testing::ValuesIn(std::vector<std::string>{
-        "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile",
-        "BlobFileBuilder::OpenBlobFileIfNeeded:WriteHeader",
-        "BlobFileBuilder::WriteBlobToFile:AddRecord",
-        "BlobFileBuilder::WriteBlobToFile:AppendFooter"}));
-
-TEST_P(BlobFileBuilderIOErrorTest, IOError) {
-  // Simulate an I/O error during the specified step of Add()
-  // Note: blob_file_size will be set to value_size in order for the first blob
-  // to trigger close
-  constexpr size_t value_size = 8;
-
-  Options options;
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileBuilderIOErrorTest_IOError"),
-      0);
-  options.enable_blob_files = true;
-  options.blob_file_size = value_size;
-  options.env = &mock_env_;
-
-  ImmutableCFOptions immutable_cf_options(options);
-  MutableCFOptions mutable_cf_options(options);
-
-  constexpr int job_id = 1;
-  constexpr uint32_t column_family_id = 123;
-  constexpr char column_family_name[] = "foobar";
-  constexpr Env::IOPriority io_priority = Env::IO_HIGH;
-  constexpr Env::WriteLifeTimeHint write_hint = Env::WLTH_MEDIUM;
-
-  std::vector<std::string> blob_file_paths;
-  std::vector<BlobFileAddition> blob_file_additions;
-
-  BlobFileBuilder builder(TestFileNumberGenerator(), fs_, &immutable_cf_options,
-                          &mutable_cf_options, &file_options_, job_id,
-                          column_family_id, column_family_name, io_priority,
-                          write_hint, nullptr /*IOTracer*/,
-                          nullptr /*BlobFileCompletionCallback*/,
-                          &blob_file_paths, &blob_file_additions);
-
-  SyncPoint::GetInstance()->SetCallBack(sync_point_, [this](void* arg) {
-    Status* const s = static_cast<Status*>(arg);
-    assert(s);
-
-    (*s) = Status::IOError(sync_point_);
-  });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr char key[] = "1";
-  constexpr char value[] = "deadbeef";
-
-  std::string blob_index;
-
-  ASSERT_TRUE(builder.Add(key, value, &blob_index).IsIOError());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-
-  if (sync_point_ == "BlobFileBuilder::OpenBlobFileIfNeeded:NewWritableFile") {
-    ASSERT_TRUE(blob_file_paths.empty());
-  } else {
-    constexpr uint64_t blob_file_number = 2;
-
-    ASSERT_EQ(blob_file_paths.size(), 1);
-    ASSERT_EQ(blob_file_paths[0],
-              BlobFileName(immutable_cf_options.cf_paths.front().path,
-                           blob_file_number));
-  }
-
-  ASSERT_TRUE(blob_file_additions.empty());
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_cache.cc

@@ -1,102 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_cache.h"
-
-#include <cassert>
-#include <memory>
-
-#include "db/blob/blob_file_reader.h"
-#include "options/cf_options.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/slice.h"
-#include "test_util/sync_point.h"
-#include "trace_replay/io_tracer.h"
-#include "util/hash.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-BlobFileCache::BlobFileCache(Cache* cache,
-                             const ImmutableCFOptions* immutable_cf_options,
-                             const FileOptions* file_options,
-                             uint32_t column_family_id,
-                             HistogramImpl* blob_file_read_hist,
-                             const std::shared_ptr<IOTracer>& io_tracer)
-    : cache_(cache),
-      mutex_(kNumberOfMutexStripes, kGetSliceNPHash64UnseededFnPtr),
-      immutable_cf_options_(immutable_cf_options),
-      file_options_(file_options),
-      column_family_id_(column_family_id),
-      blob_file_read_hist_(blob_file_read_hist),
-      io_tracer_(io_tracer) {
-  assert(cache_);
-  assert(immutable_cf_options_);
-  assert(file_options_);
-}
-
-Status BlobFileCache::GetBlobFileReader(
-    uint64_t blob_file_number,
-    CacheHandleGuard<BlobFileReader>* blob_file_reader) {
-  assert(blob_file_reader);
-  assert(blob_file_reader->IsEmpty());
-
-  const Slice key = GetSlice(&blob_file_number);
-
-  assert(cache_);
-
-  Cache::Handle* handle = cache_->Lookup(key);
-  if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-    return Status::OK();
-  }
-
-  TEST_SYNC_POINT("BlobFileCache::GetBlobFileReader:DoubleCheck");
-
-  // Check again while holding mutex
-  MutexLock lock(mutex_.get(key));
-
-  handle = cache_->Lookup(key);
-  if (handle) {
-    *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-    return Status::OK();
-  }
-
-  assert(immutable_cf_options_);
-  Statistics* const statistics = immutable_cf_options_->statistics;
-
-  RecordTick(statistics, NO_FILE_OPENS);
-
-  std::unique_ptr<BlobFileReader> reader;
-
-  {
-    assert(file_options_);
-    const Status s = BlobFileReader::Create(
-        *immutable_cf_options_, *file_options_, column_family_id_,
-        blob_file_read_hist_, blob_file_number, io_tracer_, &reader);
-    if (!s.ok()) {
-      RecordTick(statistics, NO_FILE_ERRORS);
-      return s;
-    }
-  }
-
-  {
-    constexpr size_t charge = 1;
-
-    const Status s = cache_->Insert(key, reader.get(), charge,
-                                    &DeleteCacheEntry<BlobFileReader>, &handle);
-    if (!s.ok()) {
-      RecordTick(statistics, NO_FILE_ERRORS);
-      return s;
-    }
-  }
-
-  reader.release();
-
-  *blob_file_reader = CacheHandleGuard<BlobFileReader>(cache_, handle);
-
-  return Status::OK();
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_cache.h

@@ -1,52 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cinttypes>
-
-#include "cache/cache_helpers.h"
-#include "rocksdb/rocksdb_namespace.h"
-#include "util/mutexlock.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class Cache;
-struct ImmutableCFOptions;
-struct FileOptions;
-class HistogramImpl;
-class Status;
-class BlobFileReader;
-class Slice;
-class IOTracer;
-
-class BlobFileCache {
- public:
-  BlobFileCache(Cache* cache, const ImmutableCFOptions* immutable_cf_options,
-                const FileOptions* file_options, uint32_t column_family_id,
-                HistogramImpl* blob_file_read_hist,
-                const std::shared_ptr<IOTracer>& io_tracer);
-
-  BlobFileCache(const BlobFileCache&) = delete;
-  BlobFileCache& operator=(const BlobFileCache&) = delete;
-
-  Status GetBlobFileReader(uint64_t blob_file_number,
-                           CacheHandleGuard<BlobFileReader>* blob_file_reader);
-
- private:
-  Cache* cache_;
-  // Note: mutex_ below is used to guard against multiple threads racing to open
-  // the same file.
-  Striped<port::Mutex, Slice> mutex_;
-  const ImmutableCFOptions* immutable_cf_options_;
-  const FileOptions* file_options_;
-  uint32_t column_family_id_;
-  HistogramImpl* blob_file_read_hist_;
-  std::shared_ptr<IOTracer> io_tracer_;
-
-  static constexpr size_t kNumberOfMutexStripes = 1 << 7;
-};
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_cache_test.cc

@@ -1,268 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_cache.h"
-
-#include <cassert>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "db/blob/blob_log_writer.h"
-#include "env/mock_env.h"
-#include "file/filename.h"
-#include "file/read_write_util.h"
-#include "file/writable_file_writer.h"
-#include "options/cf_options.h"
-#include "rocksdb/cache.h"
-#include "rocksdb/env.h"
-#include "rocksdb/file_system.h"
-#include "rocksdb/options.h"
-#include "rocksdb/statistics.h"
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-namespace {
-
-// Creates a test blob file with a single blob in it.
-void WriteBlobFile(uint32_t column_family_id,
-                   const ImmutableCFOptions& immutable_cf_options,
-                   uint64_t blob_file_number) {
-  assert(!immutable_cf_options.cf_paths.empty());
-
-  const std::string blob_file_path = BlobFileName(
-      immutable_cf_options.cf_paths.front().path, blob_file_number);
-
-  std::unique_ptr<FSWritableFile> file;
-  ASSERT_OK(NewWritableFile(immutable_cf_options.fs, blob_file_path, &file,
-                            FileOptions()));
-
-  std::unique_ptr<WritableFileWriter> file_writer(
-      new WritableFileWriter(std::move(file), blob_file_path, FileOptions(),
-                             immutable_cf_options.clock));
-
-  constexpr Statistics* statistics = nullptr;
-  constexpr bool use_fsync = false;
-  constexpr bool do_flush = false;
-
-  BlobLogWriter blob_log_writer(std::move(file_writer),
-                                immutable_cf_options.clock, statistics,
-                                blob_file_number, use_fsync, do_flush);
-
-  constexpr bool has_ttl = false;
-  constexpr ExpirationRange expiration_range;
-
-  BlobLogHeader header(column_family_id, kNoCompression, has_ttl,
-                       expiration_range);
-
-  ASSERT_OK(blob_log_writer.WriteHeader(header));
-
-  constexpr char key[] = "key";
-  constexpr char blob[] = "blob";
-
-  std::string compressed_blob;
-
-  uint64_t key_offset = 0;
-  uint64_t blob_offset = 0;
-
-  ASSERT_OK(blob_log_writer.AddRecord(key, blob, &key_offset, &blob_offset));
-
-  BlobLogFooter footer;
-  footer.blob_count = 1;
-  footer.expiration_range = expiration_range;
-
-  std::string checksum_method;
-  std::string checksum_value;
-
-  ASSERT_OK(
-      blob_log_writer.AppendFooter(footer, &checksum_method, &checksum_value));
-}
-
-}  // anonymous namespace
-
-class BlobFileCacheTest : public testing::Test {
- protected:
-  BlobFileCacheTest() : mock_env_(Env::Default()) {}
-
-  MockEnv mock_env_;
-};
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_, "BlobFileCacheTest_GetBlobFileReader"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr uint32_t column_family_id = 1;
-  ImmutableCFOptions immutable_cf_options(options);
-  constexpr uint64_t blob_file_number = 123;
-
-  WriteBlobFile(column_family_id, immutable_cf_options, blob_file_number);
-
-  constexpr size_t capacity = 10;
-  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
-
-  FileOptions file_options;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  // First try: reader should be opened and put in cache
-  CacheHandleGuard<BlobFileReader> first;
-
-  ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &first));
-  ASSERT_NE(first.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-
-  // Second try: reader should be served from cache
-  CacheHandleGuard<BlobFileReader> second;
-
-  ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &second));
-  ASSERT_NE(second.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-
-  ASSERT_EQ(first.GetValue(), second.GetValue());
-}
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader_Race) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileCacheTest_GetBlobFileReader_Race"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr uint32_t column_family_id = 1;
-  ImmutableCFOptions immutable_cf_options(options);
-  constexpr uint64_t blob_file_number = 123;
-
-  WriteBlobFile(column_family_id, immutable_cf_options, blob_file_number);
-
-  constexpr size_t capacity = 10;
-  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
-
-  FileOptions file_options;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  CacheHandleGuard<BlobFileReader> first;
-  CacheHandleGuard<BlobFileReader> second;
-
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileCache::GetBlobFileReader:DoubleCheck", [&](void* /* arg */) {
-        // Disabling sync points to prevent infinite recursion
-        SyncPoint::GetInstance()->DisableProcessing();
-
-        ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &second));
-        ASSERT_NE(second.GetValue(), nullptr);
-        ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-        ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  ASSERT_OK(blob_file_cache.GetBlobFileReader(blob_file_number, &first));
-  ASSERT_NE(first.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 0);
-
-  ASSERT_EQ(first.GetValue(), second.GetValue());
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader_IOError) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileCacheTest_GetBlobFileReader_IOError"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr size_t capacity = 10;
-  std::shared_ptr<Cache> backing_cache = NewLRUCache(capacity);
-
-  ImmutableCFOptions immutable_cf_options(options);
-  FileOptions file_options;
-  constexpr uint32_t column_family_id = 1;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  // Note: there is no blob file with the below number
-  constexpr uint64_t blob_file_number = 123;
-
-  CacheHandleGuard<BlobFileReader> reader;
-
-  ASSERT_TRUE(
-      blob_file_cache.GetBlobFileReader(blob_file_number, &reader).IsIOError());
-  ASSERT_EQ(reader.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1);
-}
-
-TEST_F(BlobFileCacheTest, GetBlobFileReader_CacheFull) {
-  Options options;
-  options.env = &mock_env_;
-  options.statistics = CreateDBStatistics();
-  options.cf_paths.emplace_back(
-      test::PerThreadDBPath(&mock_env_,
-                            "BlobFileCacheTest_GetBlobFileReader_CacheFull"),
-      0);
-  options.enable_blob_files = true;
-
-  constexpr uint32_t column_family_id = 1;
-  ImmutableCFOptions immutable_cf_options(options);
-  constexpr uint64_t blob_file_number = 123;
-
-  WriteBlobFile(column_family_id, immutable_cf_options, blob_file_number);
-
-  constexpr size_t capacity = 0;
-  constexpr int num_shard_bits = -1;  // determined automatically
-  constexpr bool strict_capacity_limit = true;
-  std::shared_ptr<Cache> backing_cache =
-      NewLRUCache(capacity, num_shard_bits, strict_capacity_limit);
-
-  FileOptions file_options;
-  constexpr HistogramImpl* blob_file_read_hist = nullptr;
-
-  BlobFileCache blob_file_cache(backing_cache.get(), &immutable_cf_options,
-                                &file_options, column_family_id,
-                                blob_file_read_hist, nullptr /*IOTracer*/);
-
-  // Insert into cache should fail since it has zero capacity and
-  // strict_capacity_limit is set
-  CacheHandleGuard<BlobFileReader> reader;
-
-  ASSERT_TRUE(blob_file_cache.GetBlobFileReader(blob_file_number, &reader)
-                  .IsIncomplete());
-  ASSERT_EQ(reader.GetValue(), nullptr);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_OPENS), 1);
-  ASSERT_EQ(options.statistics->getTickerCount(NO_FILE_ERRORS), 1);
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_completion_callback.h

@@ -1,53 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-//
-// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file. See the AUTHORS file for names of contributors.
-#pragma once
-
-#include "db/error_handler.h"
-#include "file/sst_file_manager_impl.h"
-#include "rocksdb/status.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlobFileCompletionCallback {
- public:
-  BlobFileCompletionCallback(SstFileManager* sst_file_manager,
-                             InstrumentedMutex* mutex,
-                             ErrorHandler* error_handler)
-      : sst_file_manager_(sst_file_manager),
-        mutex_(mutex),
-        error_handler_(error_handler) {}
-
-  Status OnBlobFileCompleted(const std::string& file_name) {
-    Status s;
-
-#ifndef ROCKSDB_LITE
-    auto sfm = static_cast<SstFileManagerImpl*>(sst_file_manager_);
-    if (sfm) {
-      // Report new blob files to SstFileManagerImpl
-      s = sfm->OnAddFile(file_name);
-      if (sfm->IsMaxAllowedSpaceReached()) {
-        s = Status::SpaceLimit("Max allowed space was reached");
-        TEST_SYNC_POINT(
-            "BlobFileCompletionCallback::CallBack::MaxAllowedSpaceReached");
-        InstrumentedMutexLock l(mutex_);
-        error_handler_->SetBGError(s, BackgroundErrorReason::kFlush);
-      }
-    }
-#else
-    (void)file_name;
-#endif  // ROCKSDB_LITE
-    return s;
-  }
-
- private:
-  SstFileManager* sst_file_manager_;
-  InstrumentedMutex* mutex_;
-  ErrorHandler* error_handler_;
-};
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_garbage.cc

@@ -1,134 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_garbage.h"
-
-#include <ostream>
-#include <sstream>
-
-#include "logging/event_logger.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// Tags for custom fields. Note that these get persisted in the manifest,
-// so existing tags should not be modified.
-enum BlobFileGarbage::CustomFieldTags : uint32_t {
-  kEndMarker,
-
-  // Add forward compatible fields here
-
-  /////////////////////////////////////////////////////////////////////
-
-  kForwardIncompatibleMask = 1 << 6,
-
-  // Add forward incompatible fields here
-};
-
-void BlobFileGarbage::EncodeTo(std::string* output) const {
-  PutVarint64(output, blob_file_number_);
-  PutVarint64(output, garbage_blob_count_);
-  PutVarint64(output, garbage_blob_bytes_);
-
-  // Encode any custom fields here. The format to use is a Varint32 tag (see
-  // CustomFieldTags above) followed by a length prefixed slice. Unknown custom
-  // fields will be ignored during decoding unless they're in the forward
-  // incompatible range.
-
-  TEST_SYNC_POINT_CALLBACK("BlobFileGarbage::EncodeTo::CustomFields", output);
-
-  PutVarint32(output, kEndMarker);
-}
-
-Status BlobFileGarbage::DecodeFrom(Slice* input) {
-  constexpr char class_name[] = "BlobFileGarbage";
-
-  if (!GetVarint64(input, &blob_file_number_)) {
-    return Status::Corruption(class_name, "Error decoding blob file number");
-  }
-
-  if (!GetVarint64(input, &garbage_blob_count_)) {
-    return Status::Corruption(class_name, "Error decoding garbage blob count");
-  }
-
-  if (!GetVarint64(input, &garbage_blob_bytes_)) {
-    return Status::Corruption(class_name, "Error decoding garbage blob bytes");
-  }
-
-  while (true) {
-    uint32_t custom_field_tag = 0;
-    if (!GetVarint32(input, &custom_field_tag)) {
-      return Status::Corruption(class_name, "Error decoding custom field tag");
-    }
-
-    if (custom_field_tag == kEndMarker) {
-      break;
-    }
-
-    if (custom_field_tag & kForwardIncompatibleMask) {
-      return Status::Corruption(
-          class_name, "Forward incompatible custom field encountered");
-    }
-
-    Slice custom_field_value;
-    if (!GetLengthPrefixedSlice(input, &custom_field_value)) {
-      return Status::Corruption(class_name,
-                                "Error decoding custom field value");
-    }
-  }
-
-  return Status::OK();
-}
-
-std::string BlobFileGarbage::DebugString() const {
-  std::ostringstream oss;
-
-  oss << *this;
-
-  return oss.str();
-}
-
-std::string BlobFileGarbage::DebugJSON() const {
-  JSONWriter jw;
-
-  jw << *this;
-
-  jw.EndObject();
-
-  return jw.Get();
-}
-
-bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) {
-  return lhs.GetBlobFileNumber() == rhs.GetBlobFileNumber() &&
-         lhs.GetGarbageBlobCount() == rhs.GetGarbageBlobCount() &&
-         lhs.GetGarbageBlobBytes() == rhs.GetGarbageBlobBytes();
-}
-
-bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs) {
-  return !(lhs == rhs);
-}
-
-std::ostream& operator<<(std::ostream& os,
-                         const BlobFileGarbage& blob_file_garbage) {
-  os << "blob_file_number: " << blob_file_garbage.GetBlobFileNumber()
-     << " garbage_blob_count: " << blob_file_garbage.GetGarbageBlobCount()
-     << " garbage_blob_bytes: " << blob_file_garbage.GetGarbageBlobBytes();
-
-  return os;
-}
-
-JSONWriter& operator<<(JSONWriter& jw,
-                       const BlobFileGarbage& blob_file_garbage) {
-  jw << "BlobFileNumber" << blob_file_garbage.GetBlobFileNumber()
-     << "GarbageBlobCount" << blob_file_garbage.GetGarbageBlobCount()
-     << "GarbageBlobBytes" << blob_file_garbage.GetGarbageBlobBytes();
-
-  return jw;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_garbage.h

@@ -1,57 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cstdint>
-#include <iosfwd>
-#include <string>
-
-#include "db/blob/blob_constants.h"
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class JSONWriter;
-class Slice;
-class Status;
-
-class BlobFileGarbage {
- public:
-  BlobFileGarbage() = default;
-
-  BlobFileGarbage(uint64_t blob_file_number, uint64_t garbage_blob_count,
-                  uint64_t garbage_blob_bytes)
-      : blob_file_number_(blob_file_number),
-        garbage_blob_count_(garbage_blob_count),
-        garbage_blob_bytes_(garbage_blob_bytes) {}
-
-  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
-  uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; }
-  uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; }
-
-  void EncodeTo(std::string* output) const;
-  Status DecodeFrom(Slice* input);
-
-  std::string DebugString() const;
-  std::string DebugJSON() const;
-
- private:
-  enum CustomFieldTags : uint32_t;
-
-  uint64_t blob_file_number_ = kInvalidBlobFileNumber;
-  uint64_t garbage_blob_count_ = 0;
-  uint64_t garbage_blob_bytes_ = 0;
-};
-
-bool operator==(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs);
-bool operator!=(const BlobFileGarbage& lhs, const BlobFileGarbage& rhs);
-
-std::ostream& operator<<(std::ostream& os,
-                         const BlobFileGarbage& blob_file_garbage);
-JSONWriter& operator<<(JSONWriter& jw,
-                       const BlobFileGarbage& blob_file_garbage);
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_garbage_test.cc

@@ -1,173 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_garbage.h"
-
-#include <cstdint>
-#include <cstring>
-#include <string>
-
-#include "test_util/sync_point.h"
-#include "test_util/testharness.h"
-#include "util/coding.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-class BlobFileGarbageTest : public testing::Test {
- public:
-  static void TestEncodeDecode(const BlobFileGarbage& blob_file_garbage) {
-    std::string encoded;
-    blob_file_garbage.EncodeTo(&encoded);
-
-    BlobFileGarbage decoded;
-    Slice input(encoded);
-    ASSERT_OK(decoded.DecodeFrom(&input));
-
-    ASSERT_EQ(blob_file_garbage, decoded);
-  }
-};
-
-TEST_F(BlobFileGarbageTest, Empty) {
-  BlobFileGarbage blob_file_garbage;
-
-  ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), kInvalidBlobFileNumber);
-  ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), 0);
-  ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), 0);
-
-  TestEncodeDecode(blob_file_garbage);
-}
-
-TEST_F(BlobFileGarbageTest, NonEmpty) {
-  constexpr uint64_t blob_file_number = 123;
-  constexpr uint64_t garbage_blob_count = 1;
-  constexpr uint64_t garbage_blob_bytes = 9876;
-
-  BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count,
-                                    garbage_blob_bytes);
-
-  ASSERT_EQ(blob_file_garbage.GetBlobFileNumber(), blob_file_number);
-  ASSERT_EQ(blob_file_garbage.GetGarbageBlobCount(), garbage_blob_count);
-  ASSERT_EQ(blob_file_garbage.GetGarbageBlobBytes(), garbage_blob_bytes);
-
-  TestEncodeDecode(blob_file_garbage);
-}
-
-TEST_F(BlobFileGarbageTest, DecodeErrors) {
-  std::string str;
-  Slice slice(str);
-
-  BlobFileGarbage blob_file_garbage;
-
-  {
-    const Status s = blob_file_garbage.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "blob file number"));
-  }
-
-  constexpr uint64_t blob_file_number = 123;
-  PutVarint64(&str, blob_file_number);
-  slice = str;
-
-  {
-    const Status s = blob_file_garbage.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "garbage blob count"));
-  }
-
-  constexpr uint64_t garbage_blob_count = 4567;
-  PutVarint64(&str, garbage_blob_count);
-  slice = str;
-
-  {
-    const Status s = blob_file_garbage.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "garbage blob bytes"));
-  }
-
-  constexpr uint64_t garbage_blob_bytes = 12345678;
-  PutVarint64(&str, garbage_blob_bytes);
-  slice = str;
-
-  {
-    const Status s = blob_file_garbage.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "custom field tag"));
-  }
-
-  constexpr uint32_t custom_tag = 2;
-  PutVarint32(&str, custom_tag);
-  slice = str;
-
-  {
-    const Status s = blob_file_garbage.DecodeFrom(&slice);
-    ASSERT_TRUE(s.IsCorruption());
-    ASSERT_TRUE(std::strstr(s.getState(), "custom field value"));
-  }
-}
-
-TEST_F(BlobFileGarbageTest, ForwardCompatibleCustomField) {
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) {
-        std::string* output = static_cast<std::string*>(arg);
-
-        constexpr uint32_t forward_compatible_tag = 2;
-        PutVarint32(output, forward_compatible_tag);
-
-        PutLengthPrefixedSlice(output, "deadbeef");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr uint64_t blob_file_number = 678;
-  constexpr uint64_t garbage_blob_count = 9999;
-  constexpr uint64_t garbage_blob_bytes = 100000000;
-
-  BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count,
-                                    garbage_blob_bytes);
-
-  TestEncodeDecode(blob_file_garbage);
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-TEST_F(BlobFileGarbageTest, ForwardIncompatibleCustomField) {
-  SyncPoint::GetInstance()->SetCallBack(
-      "BlobFileGarbage::EncodeTo::CustomFields", [&](void* arg) {
-        std::string* output = static_cast<std::string*>(arg);
-
-        constexpr uint32_t forward_incompatible_tag = (1 << 6) + 1;
-        PutVarint32(output, forward_incompatible_tag);
-
-        PutLengthPrefixedSlice(output, "foobar");
-      });
-  SyncPoint::GetInstance()->EnableProcessing();
-
-  constexpr uint64_t blob_file_number = 456;
-  constexpr uint64_t garbage_blob_count = 100;
-  constexpr uint64_t garbage_blob_bytes = 2000000;
-
-  BlobFileGarbage blob_file_garbage(blob_file_number, garbage_blob_count,
-                                    garbage_blob_bytes);
-
-  std::string encoded;
-  blob_file_garbage.EncodeTo(&encoded);
-
-  BlobFileGarbage decoded_blob_file_addition;
-  Slice input(encoded);
-  const Status s = decoded_blob_file_addition.DecodeFrom(&input);
-
-  ASSERT_TRUE(s.IsCorruption());
-  ASSERT_TRUE(std::strstr(s.getState(), "Forward incompatible"));
-
-  SyncPoint::GetInstance()->DisableProcessing();
-  SyncPoint::GetInstance()->ClearAllCallBacks();
-}
-
-}  // namespace ROCKSDB_NAMESPACE
-
-int main(int argc, char** argv) {
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}

db/blob/blob_file_meta.cc

@@ -1,55 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_meta.h"
-
-#include <ostream>
-#include <sstream>
-
-namespace ROCKSDB_NAMESPACE {
-
-std::string SharedBlobFileMetaData::DebugString() const {
-  std::ostringstream oss;
-  oss << (*this);
-
-  return oss.str();
-}
-
-std::ostream& operator<<(std::ostream& os,
-                         const SharedBlobFileMetaData& shared_meta) {
-  os << "blob_file_number: " << shared_meta.GetBlobFileNumber()
-     << " total_blob_count: " << shared_meta.GetTotalBlobCount()
-     << " total_blob_bytes: " << shared_meta.GetTotalBlobBytes()
-     << " checksum_method: " << shared_meta.GetChecksumMethod()
-     << " checksum_value: " << shared_meta.GetChecksumValue();
-
-  return os;
-}
-
-std::string BlobFileMetaData::DebugString() const {
-  std::ostringstream oss;
-  oss << (*this);
-
-  return oss.str();
-}
-
-std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta) {
-  const auto& shared_meta = meta.GetSharedMeta();
-  assert(shared_meta);
-  os << (*shared_meta);
-
-  os << " linked_ssts: {";
-  for (uint64_t file_number : meta.GetLinkedSsts()) {
-    os << ' ' << file_number;
-  }
-  os << " }";
-
-  os << " garbage_blob_count: " << meta.GetGarbageBlobCount()
-     << " garbage_blob_bytes: " << meta.GetGarbageBlobBytes();
-
-  return os;
-}
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_meta.h

@@ -1,164 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#pragma once
-
-#include <cassert>
-#include <iosfwd>
-#include <memory>
-#include <string>
-#include <unordered_set>
-
-#include "rocksdb/rocksdb_namespace.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-// SharedBlobFileMetaData represents the immutable part of blob files' metadata,
-// like the blob file number, total number and size of blobs, or checksum
-// method and value. There is supposed to be one object of this class per blob
-// file (shared across all versions that include the blob file in question);
-// hence, the type is neither copyable nor movable. A blob file can be marked
-// obsolete when the corresponding SharedBlobFileMetaData object is destroyed.
-
-class SharedBlobFileMetaData {
- public:
-  static std::shared_ptr<SharedBlobFileMetaData> Create(
-      uint64_t blob_file_number, uint64_t total_blob_count,
-      uint64_t total_blob_bytes, std::string checksum_method,
-      std::string checksum_value) {
-    return std::shared_ptr<SharedBlobFileMetaData>(new SharedBlobFileMetaData(
-        blob_file_number, total_blob_count, total_blob_bytes,
-        std::move(checksum_method), std::move(checksum_value)));
-  }
-
-  template <typename Deleter>
-  static std::shared_ptr<SharedBlobFileMetaData> Create(
-      uint64_t blob_file_number, uint64_t total_blob_count,
-      uint64_t total_blob_bytes, std::string checksum_method,
-      std::string checksum_value, Deleter deleter) {
-    return std::shared_ptr<SharedBlobFileMetaData>(
-        new SharedBlobFileMetaData(blob_file_number, total_blob_count,
-                                   total_blob_bytes, std::move(checksum_method),
-                                   std::move(checksum_value)),
-        deleter);
-  }
-
-  SharedBlobFileMetaData(const SharedBlobFileMetaData&) = delete;
-  SharedBlobFileMetaData& operator=(const SharedBlobFileMetaData&) = delete;
-
-  SharedBlobFileMetaData(SharedBlobFileMetaData&&) = delete;
-  SharedBlobFileMetaData& operator=(SharedBlobFileMetaData&&) = delete;
-
-  uint64_t GetBlobFileNumber() const { return blob_file_number_; }
-  uint64_t GetTotalBlobCount() const { return total_blob_count_; }
-  uint64_t GetTotalBlobBytes() const { return total_blob_bytes_; }
-  const std::string& GetChecksumMethod() const { return checksum_method_; }
-  const std::string& GetChecksumValue() const { return checksum_value_; }
-
-  std::string DebugString() const;
-
- private:
-  SharedBlobFileMetaData(uint64_t blob_file_number, uint64_t total_blob_count,
-                         uint64_t total_blob_bytes, std::string checksum_method,
-                         std::string checksum_value)
-      : blob_file_number_(blob_file_number),
-        total_blob_count_(total_blob_count),
-        total_blob_bytes_(total_blob_bytes),
-        checksum_method_(std::move(checksum_method)),
-        checksum_value_(std::move(checksum_value)) {
-    assert(checksum_method_.empty() == checksum_value_.empty());
-  }
-
-  uint64_t blob_file_number_;
-  uint64_t total_blob_count_;
-  uint64_t total_blob_bytes_;
-  std::string checksum_method_;
-  std::string checksum_value_;
-};
-
-std::ostream& operator<<(std::ostream& os,
-                         const SharedBlobFileMetaData& shared_meta);
-
-// BlobFileMetaData contains the part of the metadata for blob files that can
-// vary across versions, like the amount of garbage in the blob file. In
-// addition, BlobFileMetaData objects point to and share the ownership of the
-// SharedBlobFileMetaData object for the corresponding blob file. Similarly to
-// SharedBlobFileMetaData, BlobFileMetaData are not copyable or movable. They
-// are meant to be jointly owned by the versions in which the blob file has the
-// same (immutable *and* mutable) state.
-
-class BlobFileMetaData {
- public:
-  using LinkedSsts = std::unordered_set<uint64_t>;
-
-  static std::shared_ptr<BlobFileMetaData> Create(
-      std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-      LinkedSsts linked_ssts, uint64_t garbage_blob_count,
-      uint64_t garbage_blob_bytes) {
-    return std::shared_ptr<BlobFileMetaData>(
-        new BlobFileMetaData(std::move(shared_meta), std::move(linked_ssts),
-                             garbage_blob_count, garbage_blob_bytes));
-  }
-
-  BlobFileMetaData(const BlobFileMetaData&) = delete;
-  BlobFileMetaData& operator=(const BlobFileMetaData&) = delete;
-
-  BlobFileMetaData(BlobFileMetaData&&) = delete;
-  BlobFileMetaData& operator=(BlobFileMetaData&&) = delete;
-
-  const std::shared_ptr<SharedBlobFileMetaData>& GetSharedMeta() const {
-    return shared_meta_;
-  }
-
-  uint64_t GetBlobFileNumber() const {
-    assert(shared_meta_);
-    return shared_meta_->GetBlobFileNumber();
-  }
-  uint64_t GetTotalBlobCount() const {
-    assert(shared_meta_);
-    return shared_meta_->GetTotalBlobCount();
-  }
-  uint64_t GetTotalBlobBytes() const {
-    assert(shared_meta_);
-    return shared_meta_->GetTotalBlobBytes();
-  }
-  const std::string& GetChecksumMethod() const {
-    assert(shared_meta_);
-    return shared_meta_->GetChecksumMethod();
-  }
-  const std::string& GetChecksumValue() const {
-    assert(shared_meta_);
-    return shared_meta_->GetChecksumValue();
-  }
-
-  const LinkedSsts& GetLinkedSsts() const { return linked_ssts_; }
-
-  uint64_t GetGarbageBlobCount() const { return garbage_blob_count_; }
-  uint64_t GetGarbageBlobBytes() const { return garbage_blob_bytes_; }
-
-  std::string DebugString() const;
-
- private:
-  BlobFileMetaData(std::shared_ptr<SharedBlobFileMetaData> shared_meta,
-                   LinkedSsts linked_ssts, uint64_t garbage_blob_count,
-                   uint64_t garbage_blob_bytes)
-      : shared_meta_(std::move(shared_meta)),
-        linked_ssts_(std::move(linked_ssts)),
-        garbage_blob_count_(garbage_blob_count),
-        garbage_blob_bytes_(garbage_blob_bytes) {
-    assert(shared_meta_);
-    assert(garbage_blob_count_ <= shared_meta_->GetTotalBlobCount());
-    assert(garbage_blob_bytes_ <= shared_meta_->GetTotalBlobBytes());
-  }
-
-  std::shared_ptr<SharedBlobFileMetaData> shared_meta_;
-  LinkedSsts linked_ssts_;
-  uint64_t garbage_blob_count_;
-  uint64_t garbage_blob_bytes_;
-};
-
-std::ostream& operator<<(std::ostream& os, const BlobFileMetaData& meta);
-
-}  // namespace ROCKSDB_NAMESPACE

db/blob/blob_file_reader.cc

@@ -1,429 +0,0 @@
-//  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-//  This source code is licensed under both the GPLv2 (found in the
-//  COPYING file in the root directory) and Apache 2.0 License
-//  (found in the LICENSE.Apache file in the root directory).
-
-#include "db/blob/blob_file_reader.h"
-
-#include <cassert>
-#include <string>
-
-#include "db/blob/blob_log_format.h"
-#include "file/filename.h"
-#include "options/cf_options.h"
-#include "rocksdb/file_system.h"
-#include "rocksdb/slice.h"
-#include "rocksdb/status.h"
-#include "test_util/sync_point.h"
-#include "util/compression.h"
-#include "util/crc32c.h"
-
-namespace ROCKSDB_NAMESPACE {
-
-Status BlobFileReader::Create(
-    const ImmutableCFOptions& immutable_cf_options,
-    const FileOptions& file_options, uint32_t column_family_id,
-    HistogramImpl* blob_file_read_hist, uint64_t blob_file_number,
-    const std::shared_ptr<IOTracer>& io_tracer,
-    std::unique_ptr<BlobFileReader>* blob_file_reader) {
-  assert(blob_file_reader);
-  assert(!*blob_file_reader);
-
-  uint64_t file_size = 0;
-  std::unique_ptr<RandomAccessFileReader> file_reader;
-
-  {
-    const Status s =
-        OpenFile(immutable_cf_options, file_options, blob_file_read_hist,
-                 blob_file_number, io_tracer, &file_size, &file_reader);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  assert(file_reader);
-
-  CompressionType compression_type = kNoCompression;
-
-  {
-    const Status s =
-        ReadHeader(file_reader.get(), column_family_id, &compression_type);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  {
-    const Status s = ReadFooter(file_size, file_reader.get());
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  blob_file_reader->reset(
-      new BlobFileReader(std::move(file_reader), file_size, compression_type));
-
-  return Status::OK();
-}
-
-Status BlobFileReader::OpenFile(
-    const ImmutableCFOptions& immutable_cf_options,
-    const FileOptions& file_opts, HistogramImpl* blob_file_read_hist,
-    uint64_t blob_file_number, const std::shared_ptr<IOTracer>& io_tracer,
-    uint64_t* file_size, std::unique_ptr<RandomAccessFileReader>* file_reader) {
-  assert(file_size);
-  assert(file_reader);
-
-  const auto& cf_paths = immutable_cf_options.cf_paths;
-  assert(!cf_paths.empty());
-
-  const std::string blob_file_path =
-      BlobFileName(cf_paths.front().path, blob_file_number);
-
-  FileSystem* const fs = immutable_cf_options.fs;
-  assert(fs);
-
-  constexpr IODebugContext* dbg = nullptr;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::OpenFile:GetFileSize");
-
-    const Status s =
-        fs->GetFileSize(blob_file_path, IOOptions(), file_size, dbg);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (*file_size < BlobLogHeader::kSize + BlobLogFooter::kSize) {
-    return Status::Corruption("Malformed blob file");
-  }
-
-  std::unique_ptr<FSRandomAccessFile> file;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::OpenFile:NewRandomAccessFile");
-
-    const Status s =
-        fs->NewRandomAccessFile(blob_file_path, file_opts, &file, dbg);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  assert(file);
-
-  if (immutable_cf_options.advise_random_on_open) {
-    file->Hint(FSRandomAccessFile::kRandom);
-  }
-
-  file_reader->reset(new RandomAccessFileReader(
-      std::move(file), blob_file_path, immutable_cf_options.clock, io_tracer,
-      immutable_cf_options.statistics, BLOB_DB_BLOB_FILE_READ_MICROS,
-      blob_file_read_hist, immutable_cf_options.rate_limiter,
-      immutable_cf_options.listeners));
-
-  return Status::OK();
-}
-
-Status BlobFileReader::ReadHeader(const RandomAccessFileReader* file_reader,
-                                  uint32_t column_family_id,
-                                  CompressionType* compression_type) {
-  assert(file_reader);
-  assert(compression_type);
-
-  Slice header_slice;
-  Buffer buf;
-  AlignedBuf aligned_buf;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::ReadHeader:ReadFromFile");
-
-    constexpr uint64_t read_offset = 0;
-    constexpr size_t read_size = BlobLogHeader::kSize;
-
-    const Status s = ReadFromFile(file_reader, read_offset, read_size,
-                                  &header_slice, &buf, &aligned_buf);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::ReadHeader:TamperWithResult",
-                             &header_slice);
-  }
-
-  BlobLogHeader header;
-
-  {
-    const Status s = header.DecodeFrom(header_slice);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  constexpr ExpirationRange no_expiration_range;
-
-  if (header.has_ttl || header.expiration_range != no_expiration_range) {
-    return Status::Corruption("Unexpected TTL blob file");
-  }
-
-  if (header.column_family_id != column_family_id) {
-    return Status::Corruption("Column family ID mismatch");
-  }
-
-  *compression_type = header.compression;
-
-  return Status::OK();
-}
-
-Status BlobFileReader::ReadFooter(uint64_t file_size,
-                                  const RandomAccessFileReader* file_reader) {
-  assert(file_size >= BlobLogHeader::kSize + BlobLogFooter::kSize);
-  assert(file_reader);
-
-  Slice footer_slice;
-  Buffer buf;
-  AlignedBuf aligned_buf;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::ReadFooter:ReadFromFile");
-
-    const uint64_t read_offset = file_size - BlobLogFooter::kSize;
-    constexpr size_t read_size = BlobLogFooter::kSize;
-
-    const Status s = ReadFromFile(file_reader, read_offset, read_size,
-                                  &footer_slice, &buf, &aligned_buf);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::ReadFooter:TamperWithResult",
-                             &footer_slice);
-  }
-
-  BlobLogFooter footer;
-
-  {
-    const Status s = footer.DecodeFrom(footer_slice);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  constexpr ExpirationRange no_expiration_range;
-
-  if (footer.expiration_range != no_expiration_range) {
-    return Status::Corruption("Unexpected TTL blob file");
-  }
-
-  return Status::OK();
-}
-
-Status BlobFileReader::ReadFromFile(const RandomAccessFileReader* file_reader,
-                                    uint64_t read_offset, size_t read_size,
-                                    Slice* slice, Buffer* buf,
-                                    AlignedBuf* aligned_buf) {
-  assert(slice);
-  assert(buf);
-  assert(aligned_buf);
-
-  assert(file_reader);
-
-  Status s;
-
-  if (file_reader->use_direct_io()) {
-    constexpr char* scratch = nullptr;
-
-    s = file_reader->Read(IOOptions(), read_offset, read_size, slice, scratch,
-                          aligned_buf);
-  } else {
-    buf->reset(new char[read_size]);
-    constexpr AlignedBuf* aligned_scratch = nullptr;
-
-    s = file_reader->Read(IOOptions(), read_offset, read_size, slice,
-                          buf->get(), aligned_scratch);
-  }
-
-  if (!s.ok()) {
-    return s;
-  }
-
-  if (slice->size() != read_size) {
-    return Status::Corruption("Failed to read data from blob file");
-  }
-
-  return Status::OK();
-}
-
-BlobFileReader::BlobFileReader(
-    std::unique_ptr<RandomAccessFileReader>&& file_reader, uint64_t file_size,
-    CompressionType compression_type)
-    : file_reader_(std::move(file_reader)),
-      file_size_(file_size),
-      compression_type_(compression_type) {
-  assert(file_reader_);
-}
-
-BlobFileReader::~BlobFileReader() = default;
-
-Status BlobFileReader::GetBlob(const ReadOptions& read_options,
-                               const Slice& user_key, uint64_t offset,
-                               uint64_t value_size,
-                               CompressionType compression_type,
-                               PinnableSlice* value,
-                               uint64_t* bytes_read) const {
-  assert(value);
-
-  const uint64_t key_size = user_key.size();
-
-  if (!IsValidBlobOffset(offset, key_size, value_size, file_size_)) {
-    return Status::Corruption("Invalid blob offset");
-  }
-
-  if (compression_type != compression_type_) {
-    return Status::Corruption("Compression type mismatch when reading blob");
-  }
-
-  // Note: if verify_checksum is set, we read the entire blob record to be able
-  // to perform the verification; otherwise, we just read the blob itself. Since
-  // the offset in BlobIndex actually points to the blob value, we need to make
-  // an adjustment in the former case.
-  const uint64_t adjustment =
-      read_options.verify_checksums
-          ? BlobLogRecord::CalculateAdjustmentForRecordHeader(key_size)
-          : 0;
-  assert(offset >= adjustment);
-
-  const uint64_t record_offset = offset - adjustment;
-  const uint64_t record_size = value_size + adjustment;
-
-  Slice record_slice;
-  Buffer buf;
-  AlignedBuf aligned_buf;
-
-  {
-    TEST_SYNC_POINT("BlobFileReader::GetBlob:ReadFromFile");
-
-    const Status s = ReadFromFile(file_reader_.get(), record_offset,
-                                  static_cast<size_t>(record_size),
-                                  &record_slice, &buf, &aligned_buf);
-    if (!s.ok()) {
-      return s;
-    }
-
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::GetBlob:TamperWithResult",
-                             &record_slice);
-  }
-
-  if (read_options.verify_checksums) {
-    const Status s = VerifyBlob(record_slice, user_key, value_size);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  const Slice value_slice(record_slice.data() + adjustment, value_size);
-
-  {
-    const Status s =
-        UncompressBlobIfNeeded(value_slice, compression_type, value);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (bytes_read) {
-    *bytes_read = record_size;
-  }
-
-  return Status::OK();
-}
-
-Status BlobFileReader::VerifyBlob(const Slice& record_slice,
-                                  const Slice& user_key, uint64_t value_size) {
-  BlobLogRecord record;
-
-  const Slice header_slice(record_slice.data(), BlobLogRecord::kHeaderSize);
-
-  {
-    const Status s = record.DecodeHeaderFrom(header_slice);
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  if (record.key_size != user_key.size()) {
-    return Status::Corruption("Key size mismatch when reading blob");
-  }
-
-  if (record.value_size != value_size) {
-    return Status::Corruption("Value size mismatch when reading blob");
-  }
-
-  record.key =
-      Slice(record_slice.data() + BlobLogRecord::kHeaderSize, record.key_size);
-  if (record.key != user_key) {
-    return Status::Corruption("Key mismatch when reading blob");
-  }
-
-  record.value = Slice(record.key.data() + record.key_size, value_size);
-
-  {
-    TEST_SYNC_POINT_CALLBACK("BlobFileReader::VerifyBlob:CheckBlobCRC",
-                             &record);
-
-    const Status s = record.CheckBlobCRC();
-    if (!s.ok()) {
-      return s;
-    }
-  }
-
-  return Status::OK();
-}
-
-Status BlobFileReader::UncompressBlobIfNeeded(const Slice& value_slice,
-                                              CompressionType compression_type,
-                                              PinnableSlice* value) {
-  assert(value);
-
-  if (compression_type == kNoCompression) {
-    SaveValue(value_slice, value);
-
-    return Status::OK();
-  }
-
-  UncompressionContext context(compression_type);
-  UncompressionInfo info(context, UncompressionDict::GetEmptyDict(),
-                         compression_type);
-
-  size_t uncompressed_size = 0;
-  constexpr uint32_t compression_format_version = 2;
-  constexpr MemoryAllocator* allocator = nullptr;
-
-  CacheAllocationPtr output =
-      UncompressData(info, value_slice.data(), value_slice.size(),
-                     &uncompressed_size, compression_format_version, allocator);
-
-  TEST_SYNC_POINT_CALLBACK(
-      "BlobFileReader::UncompressBlobIfNeeded:TamperWithResult", &output);
-
-  if (!output) {
-    return Status::Corruption("Unable to uncompress blob");
-  }
-
-  SaveValue(Slice(output.get(), uncompressed_size), value);
-
-  return Status::OK();
-}
-
-void BlobFileReader::SaveValue(const Slice& src, PinnableSlice* dst) {
-  assert(dst);
-
-  if (dst->IsPinned()) {
-    dst->Reset();
-  }
-
-  dst->PinSelf(src);
-}
-
-}  // namespace ROCKSDB_NAMESPACE