Fix an issue with unnecessary capture in lambda expressions

Closes https://github.com/facebook/rocksdb/issues/3900

.clang-tidy

@@ -1,86 +0,0 @@
-# When making changes, verify the output of:
-#   clang-tidy -list-checks
----
-Checks: "-*,\
-  bugprone-argument-comment,\
-  bugprone-dangling-handle,\
-  bugprone-fold-init-type,\
-  bugprone-forward-declaration-namespace,\
-  bugprone-forwarding-reference-overload,\
-  bugprone-shadow,\
-  bugprone-sizeof-*,\
-  bugprone-string-constructor,\
-  bugprone-undefined-memory-manipulation,\
-  bugprone-unused-return-value,\
-  bugprone-use-after-move,\
-  cert-env33-c,\
-  cert-err58-cpp,\
-  cert-msc30-c,\
-  cert-msc50-cpp,\
-  clang-analyzer-*,\
-  clang-diagnostic-*,\
-  -clang-diagnostic-missing-designated-field-initializers,\
-  concurrency-mt-unsafe,\
-  cppcoreguidelines-avoid-non-const-global-variables,\
-  cppcoreguidelines-missing-std-forward,\
-  cppcoreguidelines-pro-type-member-init,\
-  cppcoreguidelines-special-member-functions,\
-  cppcoreguidelines-virtual-class-destructor,\
-  google-build-using-namespace,\
-  google-explicit-constructor,\
-  google-readability-avoid-underscore-in-googletest-name,\
-  misc-definitions-in-headers,\
-  misc-redundant-expression,\
-  modernize-make-shared,\
-  modernize-use-emplace,\
-  modernize-use-noexcept,\
-  modernize-use-override,\
-  modernize-use-using,\
-  performance-faster-string-find,\
-  performance-for-range-copy,\
-  performance-implicit-conversion-in-loop,\
-  performance-inefficient-algorithm,\
-  performance-inefficient-string-concatenation,\
-  performance-inefficient-vector-operation,\
-  performance-move-const-arg,\
-  performance-move-constructor-init,\
-  performance-no-automatic-move,\
-  performance-no-int-to-ptr,\
-  performance-noexcept-move-constructor,\
-  performance-noexcept-swap,\
-  performance-trivially-destructible,\
-  performance-type-promotion-in-math-fn,\
-  performance-unnecessary-copy-initialization,\
-  performance-unnecessary-value-param,\
-  readability-braces-around-statements,\
-  readability-duplicate-include,\
-  readability-isolate-declaration,\
-  readability-operators-representation,\
-  readability-redundant-string-init"
-
-WarningsAsErrors: "bugprone-use-after-move"
-
-CheckOptions:
-- key: bugprone-easily-swappable-parameters.MinimumLength
-  value: 4
-- key: cppcoreguidelines-avoid-non-const-global-variables.AllowThreadLocal
-  value: true
-- key: cppcoreguidelines-special-member-functions.AllowSoleDefaultDtor
-  value: true
-- key: cppcoreguidelines-special-member-functions.AllowImplicitlyDeletedCopyOrMove
-  value: true
-- key: modernize-use-using.IgnoreExternC
-  value: true
-- key: performance-move-const-arg.CheckTriviallyCopyableMove
-  value: false
-- key: performance-unnecessary-value-param.AllowedTypes
-  value: '[Pp]ointer$;[Pp]tr$;[Rr]ef(erence)?$'
-- key: performance-unnecessary-copy-initialization.AllowedTypes
-  value: '[Pp]ointer$;[Pp]tr$;[Rr]ef(erence)?$'
-- key: readability-operators-representation.BinaryOperators
-  value: '&&;&=;&;|;~;!;!=;||;|=;^;^='
-- key: readability-redundant-string-init.StringNames
-  value: '::std::basic_string'
-- key: readability-named-parameter.InsertPlainNamesInForwardDecls
-  value: true
-...

.github/actions/build-folly/action.yml

@@ -1,26 +0,0 @@
-name: build-folly
-description: Build folly and dependencies (skipped if cache hit)
-inputs:
-  cache-hit:
-    description: Whether the folly cache was hit
-    required: true
-runs:
-  using: composite
-  steps:
-  - name: Build folly and dependencies
-    if: ${{ inputs.cache-hit != 'true' }}
-    run: |
-      clean_path=()
-      IFS=: read -ra path_entries <<< "$PATH"
-      for entry in "${path_entries[@]}"; do
-        if [[ "$entry" != "/usr/lib/ccache" ]]; then
-          clean_path+=("$entry")
-        fi
-      done
-      export PATH="$(IFS=:; echo "${clean_path[*]}")"
-      make build_folly
-    shell: bash
-  - name: Skip folly build (using cached version)
-    if: ${{ inputs.cache-hit == 'true' }}
-    run: echo "Folly build skipped - using cached version"
-    shell: bash

.github/actions/build-for-benchmarks/action.yml

@@ -1,8 +0,0 @@
-name: build-for-benchmarks
-runs:
-  using: composite
-  steps:
-  - uses: "./.github/actions/pre-steps"
-  - name: Linux build for benchmarks
-    run: make V=1 J=8 -j8 release
-    shell: bash

.github/actions/cache-folly/action.yml

@@ -1,33 +0,0 @@
-name: cache-folly
-description: Cache folly build to speed up CI
-outputs:
-  cache-hit:
-    description: Whether the cache was hit
-    value: ${{ steps.cache-folly-build.outputs.cache-hit }}
-runs:
-  using: composite
-  steps:
-  - name: Extract FOLLY_MK_HASH
-    id: extract-folly-hash
-    shell: bash
-    run: |
-      FOLLY_MK_HASH=$(md5sum folly.mk | cut -d' ' -f1)
-      echo "hash=$FOLLY_MK_HASH" >> $GITHUB_OUTPUT
-  - name: Extract FOLLY_INSTALL_DIR
-    id: extract-folly-install-dir
-    shell: bash
-    run: |
-      FOLLY_INSTALL_DIR=$(cd third-party/folly && python3 build/fbcode_builder/getdeps.py show-inst-dir)
-      echo "dir=$(echo $FOLLY_INSTALL_DIR | sed 's|installed/folly|installed|')" >> $GITHUB_OUTPUT
-  - name: Cache folly build
-    id: cache-folly-build
-    uses: actions/cache@v4
-    with:
-      # Cache the folly build directory
-      path: ${{ steps.extract-folly-install-dir.outputs.dir }}
-      # Key is based on:
-      # - OS and architecture
-      # - The docker image, which may not always be specified/known
-      # - Hash of folly.mk, which includes the folly repository commit hash
-      # NOTE: this is still only intended for DEBUG folly builds
-      key: folly-build-${{ runner.os }}-${{ runner.arch }}-${{ github.job_container.image }}-${{ steps.extract-folly-hash.outputs.hash }}-${{ env.PORTABLE == '1' && 'portable' || 'native' }}

.github/actions/cache-getdeps-downloads/action.yml

@@ -1,21 +0,0 @@
-name: cache-getdeps-downloads
-description: Cache getdeps downloads to avoid unreliable mirrors and speed up builds
-outputs:
-  cache-hit:
-    description: Whether the cache was hit
-    value: ${{ steps.cache-downloads.outputs.cache-hit }}
-runs:
-  using: composite
-  steps:
-  - name: Cache getdeps downloads
-    id: cache-downloads
-    uses: actions/cache@v4
-    with:
-      # Use a fixed path that we control - folly.mk will sync with getdeps downloads dir
-      path: /tmp/rocksdb-getdeps-cache
-      # Use a rolling cache key - the cache accumulates downloads over time
-      # The key includes a weekly timestamp to ensure periodic refresh
-      key: getdeps-downloads-${{ runner.os }}-${{ runner.arch }}-week-${{ github.run_id }}
-      restore-keys: |
-        getdeps-downloads-${{ runner.os }}-${{ runner.arch }}-week-
-        getdeps-downloads-${{ runner.os }}-${{ runner.arch }}-

.github/actions/increase-max-open-files-on-macos/action.yml

@@ -1,10 +0,0 @@
-name: increase-max-open-files-on-macos
-runs:
-  using: composite
-  steps:
-  - name: Increase max open files
-    run: |-
-      sudo sysctl -w kern.maxfiles=1048576
-      sudo sysctl -w kern.maxfilesperproc=1048576
-      sudo launchctl limit maxfiles 1048576
-    shell: bash

.github/actions/install-gflags-on-macos/action.yml

@@ -1,7 +0,0 @@
-name: install-gflags-on-macos
-runs:
-  using: composite
-  steps:
-  - name: Install gflags on macos
-    run: HOMEBREW_NO_AUTO_UPDATE=1 brew install gflags
-    shell: bash

.github/actions/install-gflags/action.yml

@@ -1,7 +0,0 @@
-name: install-gflags
-runs:
-  using: composite
-  steps:
-  - name: Install gflags
-    run: sudo apt-get update -y && sudo apt-get install -y libgflags-dev
-    shell: bash

.github/actions/install-jdk8-on-macos/action.yml

@@ -1,9 +0,0 @@
-name: install-jdk8-on-macos
-runs:
-  using: composite
-  steps:
-  - name: Install JDK 8 on macos
-    run: |-
-      HOMEBREW_NO_AUTO_UPDATE=1 brew tap bell-sw/liberica
-      HOMEBREW_NO_AUTO_UPDATE=1 brew install --cask liberica-jdk8
-    shell: bash

.github/actions/install-maven/action.yml

@@ -1,11 +0,0 @@
-name: install-maven
-runs:
-  using: composite
-  steps:
-  - name: Install Maven
-    run: |
-      wget --no-check-certificate https://archive.apache.org/dist/maven/maven-3/3.9.11/binaries/apache-maven-3.9.11-bin.tar.gz
-      tar zxf apache-maven-3.9.11-bin.tar.gz
-      echo "export M2_HOME=$(pwd)/apache-maven-3.9.11" >> $GITHUB_ENV
-      echo "$(pwd)/apache-maven-3.9.11/bin" >> $GITHUB_PATH
-    shell: bash

.github/actions/perform-benchmarks/action.yml

@@ -1,22 +0,0 @@
-name: perform-benchmarks
-runs:
-  using: composite
-  steps:
-  - name: Test low-variance benchmarks
-    run: "./tools/benchmark_ci.py --db_dir ${{ runner.temp }}/rocksdb-benchmark-datadir --output_dir ${{ runner.temp }}/benchmark-results --num_keys 20000000"
-    env:
-      LD_LIBRARY_PATH: "/usr/local/lib"
-      DURATION_RO: 300
-      DURATION_RW: 500
-      NUM_THREADS: 1
-      MAX_BACKGROUND_JOBS: 4
-      CI_TESTS_ONLY: 'true'
-      WRITE_BUFFER_SIZE_MB: 16
-      TARGET_FILE_SIZE_BASE_MB: 16
-      MAX_BYTES_FOR_LEVEL_BASE_MB: 64
-      COMPRESSION_TYPE: none
-      CACHE_INDEX_AND_FILTER_BLOCKS: 1
-      MIN_LEVEL_TO_COMPRESS: 3
-      CACHE_SIZE_MB: 10240
-      MB_WRITE_PER_SEC: 2
-    shell: bash

.github/actions/post-benchmarks/action.yml

@@ -1,17 +0,0 @@
-name: post-benchmarks
-runs:
-  using: composite
-  steps:
-  - name: Upload Benchmark Results artifact
-    uses: actions/upload-artifact@v4.0.0
-    with:
-      name: benchmark-results
-      path: "${{ runner.temp }}/benchmark-results/**"
-      if-no-files-found: error
-  - name: Send benchmark report to visualisation
-    run: |-
-      set +e
-      set +o pipefail
-      ./build_tools/benchmark_log_tool.py --tsvfile ${{ runner.temp }}/benchmark-results/report.tsv --esdocument https://search-rocksdb-bench-k2izhptfeap2hjfxteolsgsynm.us-west-2.es.amazonaws.com/bench_test3_rix/_doc
-      true
-    shell: bash

.github/actions/post-steps/action.yml

@@ -1,38 +0,0 @@
-name: post-steps
-description: Steps that are taken after a RocksDB job
-inputs:
-  artifact-prefix:
-    description: Prefix to append to the name of artifacts that are uploaded
-    required: true
-    default: "${{ github.job }}"
-runs:
-  using: composite
-  steps:
-  - name: Upload Test Results artifact
-    uses: actions/upload-artifact@v4.0.0
-    with:
-      name: "${{ inputs.artifact-prefix }}-test-results"
-      path: "${{ runner.temp }}/test-results/**"
-  - name: Upload DB LOG file artifact
-    uses: actions/upload-artifact@v4.0.0
-    with:
-      name: "${{ inputs.artifact-prefix }}-db-log-file"
-      path: LOG
-  - name: Copy Test Logs (on Failure)
-    if: ${{ failure() }}
-    run: |
-      mkdir -p ${{ runner.temp }}/failure-test-logs
-      cp -r t/* ${{ runner.temp }}/failure-test-logs
-    shell: bash
-  - name: Upload Test Logs (on Failure) artifact
-    uses: actions/upload-artifact@v4.0.0
-    with:
-      name: "${{ inputs.artifact-prefix }}-failure-test-logs"
-      path: ${{ runner.temp }}/failure-test-logs/**
-      if-no-files-found: ignore
-  - name: Upload Core Dumps artifact
-    uses: actions/upload-artifact@v4.0.0
-    with:
-      name: "${{ inputs.artifact-prefix }}-core-dumps"
-      path: "core.*"
-      if-no-files-found: ignore

.github/actions/pre-steps-macos/action.yml

@@ -1,5 +0,0 @@
-name: pre-steps-macos
-runs:
-  using: composite
-  steps:
-  - uses: "./.github/actions/pre-steps"

.github/actions/pre-steps/action.yml

@@ -1,21 +0,0 @@
-name: pre-steps
-runs:
-  using: composite
-  steps:
-  - name: Install lld linker for faster builds
-    run: apt-get update -y && apt-get install -y lld 2>/dev/null || true
-    shell: bash
-  - name: Setup Environment Variables
-    run: |-
-      echo "GTEST_THROW_ON_FAILURE=0" >> "$GITHUB_ENV"
-      echo "GTEST_OUTPUT=\"xml:${{ runner.temp }}/test-results/\"" >> "$GITHUB_ENV"
-      echo "SKIP_FORMAT_BUCK_CHECKS=1" >> "$GITHUB_ENV"
-      echo "GTEST_COLOR=1" >> "$GITHUB_ENV"
-      echo "CTEST_OUTPUT_ON_FAILURE=1" >> "$GITHUB_ENV"
-      echo "CTEST_TEST_TIMEOUT=300" >> "$GITHUB_ENV"
-      echo "ZLIB_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zlib" >> "$GITHUB_ENV"
-      echo "BZIP2_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/bzip2" >> "$GITHUB_ENV"
-      echo "SNAPPY_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/snappy" >> "$GITHUB_ENV"
-      echo "LZ4_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/lz4" >> "$GITHUB_ENV"
-      echo "ZSTD_DOWNLOAD_BASE=https://rocksdb-deps.s3.us-west-2.amazonaws.com/pkgs/zstd" >> "$GITHUB_ENV"
-    shell: bash

.github/actions/setup-ccache/action.yml

@@ -1,54 +0,0 @@
-name: setup-ccache
-description: Setup ccache for faster C++ compilation caching
-inputs:
-  cache-key-prefix:
-    description: Unique prefix for the cache key (e.g., 'build-linux')
-    required: true
-  portable:
-    description: Set PORTABLE=1 to disable -march=native (set to "false" for jobs linking pre-built Folly)
-    required: false
-    default: "true"
-runs:
-  using: composite
-  steps:
-    - name: Set ccache environment variables
-      run: |
-        echo "CCACHE_DIR=${{ github.workspace }}/.ccache" >> $GITHUB_ENV
-        echo "CCACHE_BASEDIR=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "CCACHE_NOHASHDIR=true" >> $GITHUB_ENV
-        echo "CCACHE_COMPILERCHECK=content" >> $GITHUB_ENV
-        echo "CCACHE_SLOPPINESS=clang_index_store,file_stat_matches,include_file_ctime,include_file_mtime,ivfsoverlay,pch_defines,modules,system_headers,time_macros" >> $GITHUB_ENV
-        echo "CCACHE_MAXSIZE=4G" >> $GITHUB_ENV
-        if [ "${{ inputs.portable }}" = "true" ]; then
-          echo "PORTABLE=1" >> $GITHUB_ENV
-        fi
-      shell: bash
-    - name: Restore ccache
-      uses: actions/cache@v4
-      with:
-        path: ${{ github.workspace }}/.ccache
-        key: ccache-${{ inputs.cache-key-prefix }}-${{ github.head_ref || github.ref }}-${{ github.sha }}
-        restore-keys: |-
-          ccache-${{ inputs.cache-key-prefix }}-${{ github.head_ref || github.ref }}-
-          ccache-${{ inputs.cache-key-prefix }}-refs/heads/main-
-    - name: Install ccache
-      run: |
-        if [ "$RUNNER_OS" = "macOS" ]; then
-          which ccache || brew install ccache
-        else
-          which ccache || (apt-get update && apt-get install -y ccache)
-        fi
-      shell: bash
-    - name: Add ccache to PATH
-      run: |
-        if [ "$RUNNER_OS" = "macOS" ]; then
-          echo "$(brew --prefix ccache)/libexec" >> $GITHUB_PATH
-        else
-          echo "/usr/lib/ccache" >> $GITHUB_PATH
-        fi
-      shell: bash
-    - name: Zero ccache stats and set build marker
-      run: |
-        ccache -z
-        touch "$CCACHE_DIR/.build_marker"
-      shell: bash

.github/actions/setup-folly/action.yml

@@ -1,11 +0,0 @@
-name: setup-folly
-runs:
-  using: composite
-  steps:
-  - name: Checkout folly sources
-    run: |
-      make checkout_folly
-    shell: bash
-  - name: Install patchelf and libaio
-    run: apt-get update -y && apt-get install -y patchelf libaio-dev
-    shell: bash

.github/actions/setup-upstream/action.yml

@@ -1,20 +0,0 @@
-name: build-folly
-runs:
-  using: composite
-  steps:
-  - name: Fix repo ownership
-    # Needed in some cases, as safe.directory setting doesn't take effect
-    # under env -i
-    run: chown `whoami` . || true
-    shell: bash
-  - name: Set upstream
-    run: git remote add upstream https://github.com/facebook/rocksdb.git
-    shell: bash
-  - name: Fetch upstream
-    run: git fetch upstream
-    shell: bash
-  - name: Git status
-    # NOTE: some old branch builds under check_format_compatible.sh invoke
-    # git under env -i
-    run: git status && git remote -v && env -i git branch
-    shell: bash

.github/actions/teardown-ccache/action.yml

@@ -1,15 +0,0 @@
-name: teardown-ccache
-description: Trim stale ccache entries and print stats
-runs:
-  using: composite
-  steps:
-    - name: Trim and print ccache stats
-      run: |
-        if [ -z "$CCACHE_DIR" ]; then
-          echo "teardown-ccache: CCACHE_DIR not set, skipping (setup-ccache may not have run)"
-          exit 0
-        fi
-        .github/scripts/ccache-trim.sh || true
-        ccache -s || echo "teardown-ccache: ccache not found, skipping stats"
-      if: always()
-      shell: bash

.github/actions/windows-build-steps/action.yml

@@ -1,91 +0,0 @@
-name: windows-build-steps
-inputs:
-  suite-run:
-    description: Comma-separated list of test suites to run (empty to skip C++ tests)
-    required: false
-    default: arena_test,db_basic_test,db_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
-  run-java:
-    description: Whether to run Java tests
-    required: false
-    default: "true"
-runs:
-  using: composite
-  steps:
-  - name: Add msbuild to PATH
-    uses: microsoft/setup-msbuild@v1.3.1
-  - name: ccache
-    uses: hendrikmuhs/ccache-action@v1.2
-    with:
-      max-size: "10GB"
-      key: ccache-windows-${{ github.workflow }}
-      restore-keys: |
-        ccache-windows-
-  - name: Configure ccache
-    shell: pwsh
-    run: |
-      ccache --set-config=base_dir=C:\a\rocksdb\rocksdb
-      ccache --set-config=hash_dir=false
-      ccache --set-config=compiler_check=content
-  - name: Custom steps
-    env:
-      THIRDPARTY_HOME: ${{ github.workspace }}/thirdparty
-      CMAKE_HOME: C:/Program Files/CMake
-      CMAKE_BIN: C:/Program Files/CMake/bin/cmake.exe
-      CTEST_BIN: C:/Program Files/CMake/bin/ctest.exe
-      JAVA_HOME: C:/Program Files/BellSoft/LibericaJDK-8
-      SNAPPY_HOME: ${{ github.workspace }}/thirdparty/snappy-1.2.2
-      SNAPPY_INCLUDE: ${{ github.workspace }}/thirdparty/snappy-1.2.2;${{ github.workspace }}/thirdparty/snappy-1.2.2/build
-      SNAPPY_LIB_DEBUG: ${{ github.workspace }}/thirdparty/snappy-1.2.2/build/Debug/snappy.lib
-    run: |-
-      # NOTE: if ... Exit $LASTEXITCODE lines needed to exit and report failure
-      echo ===================== Install Dependencies =====================
-      choco install liberica8jdk -y
-      if(!$?) { Exit $LASTEXITCODE }
-      mkdir $Env:THIRDPARTY_HOME
-      cd $Env:THIRDPARTY_HOME
-      echo "Building Snappy dependency..."
-      curl -Lo snappy-1.2.2.zip https://github.com/google/snappy/archive/refs/tags/1.2.2.zip
-      if(!$?) { Exit $LASTEXITCODE }
-      unzip -q snappy-1.2.2.zip
-      if(!$?) { Exit $LASTEXITCODE }
-      cd snappy-1.2.2
-      mkdir build
-      cd build
-      & cmake -G "$Env:CMAKE_GENERATOR" .. -DSNAPPY_BUILD_TESTS=OFF -DSNAPPY_BUILD_BENCHMARKS=OFF
-      if(!$?) { Exit $LASTEXITCODE }
-      msbuild Snappy.sln -maxCpuCount -property:Configuration=Debug -property:Platform=x64
-      if(!$?) { Exit $LASTEXITCODE }
-      echo ======================== Build RocksDB =========================
-      cd ${{ github.workspace }}
-      $env:Path = $env:JAVA_HOME + ";" + $env:Path
-      mkdir build
-      cd build
-      & cmake -G "$Env:CMAKE_GENERATOR" -DCMAKE_BUILD_TYPE=Debug -DWIN_CI=1 -DPORTABLE="$Env:CMAKE_PORTABLE" -DSNAPPY=1 -DXPRESS=1 -DJNI=1 ..
-      if(!$?) { Exit $LASTEXITCODE }
-      cd ..
-      echo "Building with VS version: $Env:CMAKE_GENERATOR"
-      # use more parallel processes than the number of processes available, as most of the compile command would be cache hit
-      msbuild build/rocksdb.sln /m:32 /p:LinkIncremental=false -property:Configuration=Debug -property:Platform=x64
-      if(!$?) { Exit $LASTEXITCODE }
-      echo ========================= Test RocksDB =========================
-      $suiteRun = "${{ inputs.suite-run }}"
-      if ($suiteRun -ne "") {
-        $suiteArray = $suiteRun -split ','
-        build_tools\run_ci_db_test.ps1 -SuiteRun $suiteArray -Concurrency 16
-        if(!$?) { Exit $LASTEXITCODE }
-      } else {
-        echo "Skipping C++ tests (suite-run is empty)"
-      }
-      if ("${{ inputs.run-java }}" -eq "true") {
-        echo ======================== Test RocksJava ========================
-        cd build\java
-        & ctest -C Debug -j 16
-        if(!$?) { Exit $LASTEXITCODE }
-      } else {
-        echo "Skipping Java tests"
-      }
-    shell: pwsh
-  - name: Show ccache stats
-    shell: pwsh
-    run: |
-      ccache --show-stats -v

.github/scripts/build-ai-review-comment.js

@@ -1,27 +0,0 @@
-// Shared markdown builder for AI review comment bodies.
-//
-// Usage:
-//   const build = require('./build-ai-review-comment.js');
-//   return build({ icon, headerTitle, triggerLine, responseBody, footerLines
-//   });
-
-module.exports = function buildAiReviewComment(
-    {icon, headerTitle, triggerLine, responseBody, footerLines}) {
-  return [
-    `## ${icon} ${headerTitle}`,
-    '',
-    triggerLine,
-    '',
-    '---',
-    '',
-    responseBody,
-    '',
-    '---',
-    '',
-    '<details>',
-    '<summary>ℹ️ About this response</summary>',
-    '',
-    ...footerLines,
-    '</details>',
-  ].join('\n');
-};

.github/scripts/ccache-trim.sh

@@ -1,39 +0,0 @@
-#!/bin/bash
-# Trim ccache to keep only entries accessed during the current build.
-#
-# Usage: 
-#   1. Before build: touch "$CCACHE_DIR/.build_marker"
-#   2. Run build (ccache updates mtime on hits, creates new files for misses)
-#   3. After build: .github/scripts/ccache-trim.sh
-#
-# This removes cache files not accessed during the build (stale entries from
-# previous commits). Only intended for CI where each run builds one commit.
-# Do NOT use on local builds where multiple worktrees may share the cache.
-
-set -e
-
-CCACHE_DIR="${CCACHE_DIR:?CCACHE_DIR must be set}"
-MARKER="$CCACHE_DIR/.build_marker"
-
-if [ ! -f "$MARKER" ]; then
-    echo "ccache-trim: No .build_marker found, skipping (was the marker created before build?)"
-    exit 0
-fi
-
-# Count files before cleanup
-before=$(find "$CCACHE_DIR" -type f \( -name '*R' -o -name '*M' \) | wc -l)
-
-# Delete cache files (results and manifests) older than the marker
-find "$CCACHE_DIR" -type f \( -name '*R' -o -name '*M' \) ! -newer "$MARKER" -delete
-
-# Clean up empty directories
-find "$CCACHE_DIR" -mindepth 2 -type d -empty -delete 2>/dev/null || true
-
-# Recalculate size counters
-ccache -c 2>/dev/null || true
-
-after=$(find "$CCACHE_DIR" -type f \( -name '*R' -o -name '*M' \) | wc -l)
-echo "ccache-trim: $before -> $after cache files (removed $((before - after)) stale entries)"
-
-# Clean up marker
-rm -f "$MARKER"

.github/scripts/compute-test-shard.sh

@@ -1,32 +0,0 @@
-#!/bin/bash
-# Compute test shard for parallel CI execution.
-# Distributes tests round-robin across N shards for balanced load.
-# Outputs ROCKSDBTESTS_SUBSET — the list of test binaries for this shard.
-# The Makefile uses this to build and run only the assigned tests.
-#
-# Usage: compute-test-shard.sh <shard_index> <num_shards>
-
-set -euo pipefail
-
-shard=${1:?Usage: compute-test-shard.sh <shard_index> <num_shards>}
-nshards=${2:?Usage: compute-test-shard.sh <shard_index> <num_shards>}
-
-# Get sorted test list (db_test first since it's the heaviest, then alpha)
-make -s list_all_tests 2>/dev/null | tr ' ' '\n' | grep '_test$' | sort -u > /tmp/sorted.txt
-(echo db_test; grep -v '^db_test$' /tmp/sorted.txt) > /tmp/all_tests.txt
-
-total=$(wc -l < /tmp/all_tests.txt)
-
-# Round-robin: assign test i to shard (i % nshards).
-# This spreads heavy tests (which are scattered alphabetically) evenly.
-awk -v s="$shard" -v n="$nshards" 'NR > 0 && (NR - 1) % n == s' /tmp/all_tests.txt > /tmp/include.txt
-
-included=$(wc -l < /tmp/include.txt)
-first=$(head -1 /tmp/include.txt)
-last=$(tail -1 /tmp/include.txt)
-
-# Output space-separated list for ROCKSDBTESTS_SUBSET
-subset=$(tr '\n' ' ' < /tmp/include.txt)
-echo "subset=${subset}" >> "$GITHUB_OUTPUT"
-
-echo "Shard $shard/$nshards: $included tests (round-robin), first=$first last=$last (total $total)"

.github/scripts/parse-claude-review.js

@@ -1,118 +0,0 @@
-// Parse Claude Code execution log and produce a markdown review comment.
-//
-// Usage from actions/github-script:
-//   const parse = require('./.github/scripts/parse-claude-review.js');
-//   const markdown = parse({ executionFile, conclusion, meta });
-//
-// Parameters:
-//   executionFile - path to the JSON execution log from claude-code-base-action
-//   conclusion    - 'success' or 'failure' from the action output
-//   meta          - { trigger, autoMode, headSha, reviewer, isQuery, isPartial
-//   }
-
-const fs = require('fs');
-const buildComment = require('./build-ai-review-comment.js');
-
-module.exports = function parseClaude({executionFile, conclusion, meta}) {
-  function getTriggerLine() {
-    if (meta.trigger !== 'auto') {
-      return `*Requested by @${meta.reviewer}*`;
-    }
-    const shortSha = meta.headSha ? meta.headSha.substring(0, 7) : 'unknown';
-    if (meta.autoMode === 'early') {
-      return `*Auto-triggered after CI reached the early-review threshold — reviewing commit ${
-          shortSha}*`;
-    }
-    return `*Auto-triggered after CI passed — reviewing commit ${shortSha}*`;
-  }
-
-  let responseBody = '';
-
-  try {
-    const executionLog = JSON.parse(fs.readFileSync(executionFile, 'utf8'));
-
-    if (!Array.isArray(executionLog)) {
-      throw new Error('Expected array format from claude-code-base-action');
-    }
-
-    const resultMessage = executionLog.find(m => m.type === 'result');
-
-    // Helper: extract the last substantial assistant text from the log.
-    // Used as a fallback when Claude ran out of turns and the recovery
-    // session also failed to produce a result.
-    function getLastAssistantText(log, minLength = 200) {
-      for (let i = log.length - 1; i >= 0; i--) {
-        const m = log[i];
-        if (m.type !== 'assistant') continue;
-        const content = m.message && m.message.content;
-        if (!Array.isArray(content)) continue;
-        for (let j = content.length - 1; j >= 0; j--) {
-          if (content[j].type === 'text' && content[j].text &&
-              content[j].text.trim().length >= minLength) {
-            const text = content[j].text.trim();
-            // Truncate to avoid enormous PR comments
-            return text.length > 50000 ? text.substring(0, 50000) +
-                    '\n\n*[Truncated — full output in execution log artifact]*' :
-                                         text;
-          }
-        }
-      }
-      return null;
-    }
-
-    if (!resultMessage) {
-      responseBody = '⚠️ No result message found in execution log.';
-    } else if (resultMessage.subtype === 'success' && resultMessage.result) {
-      responseBody = resultMessage.result;
-    } else if (resultMessage.is_error || resultMessage.subtype === 'error') {
-      const errorInfo =
-          resultMessage.result || resultMessage.error || 'Unknown error';
-      responseBody = `❌ **Claude encountered an error:**\n\n${errorInfo}`;
-    } else if (resultMessage.subtype === 'error_max_turns') {
-      // The workflow runs a recovery session when this happens, so this
-      // branch is typically only hit if recovery wasn't attempted (e.g.,
-      // no findings file was written). Extract what we can.
-      const partial = getLastAssistantText(executionLog);
-      if (partial) {
-        responseBody =
-            `⚠️ **Review incomplete — Claude hit the turn limit.**\n\nBelow is the last partial output. You can request a fresh review with \`/claude-review\`.\n\n---\n\n${
-                partial}`;
-      } else {
-        responseBody =
-            '⚠️ **Review incomplete — Claude hit the turn limit before producing output.** You can request a fresh review with `/claude-review`.';
-      }
-    } else if (resultMessage.result) {
-      responseBody = `⚠️ **Completed with status: ${
-          resultMessage.subtype}**\n\n${resultMessage.result}`;
-    } else {
-      responseBody = '⚠️ Claude completed but produced no output.';
-    }
-  } catch (error) {
-    responseBody = `❌ Error parsing Claude response: ${error.message}`;
-  }
-
-  const isPartial = !!meta.isPartial;
-  const icon = isPartial ? '🟡' : (conclusion === 'success' ? '✅' : '⚠️');
-  const headerTitle = meta.isQuery ? 'Claude Response' : 'Claude Code Review';
-  const triggerLine = getTriggerLine();
-
-  return buildComment({
-    icon,
-    headerTitle,
-    triggerLine,
-    responseBody,
-    footerLines: [
-      'Generated by [Claude Code](https://github.com/anthropics/claude-code-action).',
-      'Review methodology: `claude_md/code_review.md`',
-      '',
-      '**Limitations:**',
-      '- Claude may miss context from files not in the diff',
-      '- Large PRs may be truncated',
-      '- Always apply human judgment to AI suggestions',
-      '',
-      '**Commands:**',
-      '- `/claude-review [context]` — Request a code review',
-      '- `/claude-query <question>` — Ask about the PR or codebase',
-    ],
-  });
-};

.github/scripts/parse-codex-review.js

@@ -1,98 +0,0 @@
-// Parse Codex review artifacts and produce a markdown review comment.
-//
-// Usage from actions/github-script:
-//   const parse = require('./.github/scripts/parse-codex-review.js');
-//   const markdown = parse({ responseFile, recoveryFile, findingsFile, logFile,
-//   exitCode, meta });
-//
-// Parameters:
-//   responseFile - path to Codex final response output
-//   recoveryFile - path to recovery output formatted from review-findings.md
-//   findingsFile - path to incremental findings file written during review
-//   logFile      - path to Codex stdout/stderr log
-//   exitCode     - Codex process exit code
-//   meta         - { trigger, autoMode, headSha, reviewer, isQuery, isPartial }
-
-const fs = require('fs');
-const buildComment = require('./build-ai-review-comment.js');
-
-module.exports = function parseCodex(
-    {responseFile, recoveryFile, findingsFile, logFile, exitCode, meta}) {
-  function getTriggerLine() {
-    if (meta.trigger !== 'auto') {
-      return `*Requested by @${meta.reviewer}*`;
-    }
-    const shortSha = meta.headSha ? meta.headSha.substring(0, 7) : 'unknown';
-    if (meta.autoMode === 'early') {
-      return `*Auto-triggered after CI reached the early-review threshold — reviewing commit ${
-          shortSha}*`;
-    }
-    return `*Auto-triggered after CI passed — reviewing commit ${shortSha}*`;
-  }
-
-  function readIfPresent(path) {
-    if (!path || !fs.existsSync(path)) {
-      return '';
-    }
-    const text = fs.readFileSync(path, 'utf8').trim();
-    return text;
-  }
-
-  function tailFile(path, maxChars = 12000) {
-    const text = readIfPresent(path);
-    if (!text) {
-      return '';
-    }
-    if (text.length <= maxChars) {
-      return text;
-    }
-    return text.slice(text.length - maxChars);
-  }
-
-  let responseBody = '';
-  const recovered = readIfPresent(recoveryFile);
-  const direct = readIfPresent(responseFile);
-  const findings = readIfPresent(findingsFile);
-
-  if (recovered) {
-    responseBody = recovered;
-  } else if (direct) {
-    responseBody = direct;
-  } else if (findings) {
-    responseBody =
-        '⚠️ **Review incomplete — Codex did not produce a final response.**\n\n' +
-        'Below are the incremental findings recovered from `review-findings.md`.\n\n---\n\n' +
-        findings;
-  } else {
-    const logTail = tailFile(logFile);
-    responseBody = logTail ?
-        `❌ **Codex review failed before producing findings.**\n\n\`\`\`\n${
-            logTail}\n\`\`\`` :
-        '❌ Codex review failed before producing any output.';
-  }
-
-  const isPartial = !!meta.isPartial;
-  const code = Number.parseInt(exitCode || '1', 10);
-  const icon = isPartial ? '🟡' : (code === 0 ? '✅' : '⚠️');
-  const triggerLine = getTriggerLine();
-
-  return buildComment({
-    icon,
-    headerTitle: meta.isQuery ? 'Codex Response' : 'Codex Code Review',
-    triggerLine,
-    responseBody,
-    footerLines: [
-      'Generated by Codex CLI.',
-      'Review methodology: `claude_md/code_review.md`',
-      '',
-      '**Limitations:**',
-      '- Codex may miss context from files not in the diff',
-      '- Large PRs may be truncated',
-      '- Always apply human judgment to AI suggestions',
-      '',
-      '**Commands:**',
-      '- `/codex-review [context]` — Request a code review',
-      '- `/codex-query <question>` — Ask about the PR or codebase',
-    ],
-  });
-};

.github/scripts/post-pr-comment.js

@@ -1,204 +0,0 @@
-// Shared PR comment posting utility.
-// Used by both clang-tidy-comment.yml and claude-review-comment.yml.
-//
-// Usage from actions/github-script:
-//   const post = require('./.github/scripts/post-pr-comment.js');
-//   await post({ github, context, core, prNumber, body, marker });
-//
-// Parameters:
-//   github   - octokit instance from actions/github-script
-//   context  - GitHub Actions context
-//   core     - @actions/core for logging
-//   prNumber - PR number to comment on
-//   body     - comment body (markdown string)
-//   marker   - HTML comment marker for dedup (e.g. '<!-- claude-review -->')
-//              If an existing comment with this marker is found, it is updated.
-//              If not found, a new comment is created.
-//   legacyMarkers - optional list of legacy markers that should be migrated by
-//              being considered part of the same comment family.
-//   prunePrefix - optional marker prefix whose older comments should be
-//                 superseded. If obsoleteTitle is set, older comments are
-//                 collapsed instead of deleted.
-//   preserveLatest - optional count of active comments to keep when
-//                    prunePrefix is set.
-//   obsoleteMarker - optional HTML marker used to detect already-obsolete
-//                    comments.
-//   obsoleteTitle - optional heading to use when collapsing superseded
-//                   comments into a details block.
-
-module.exports = async function postPrComment({
-  github,
-  context,
-  core,
-  prNumber,
-  body,
-  marker,
-  legacyMarkers = [],
-  prunePrefix = '',
-  preserveLatest = 0,
-  obsoleteMarker = '',
-  obsoleteTitle = '',
-}) {
-  if (!prNumber || !body) {
-    core.warning('Missing prNumber or body; skipping comment.');
-    return;
-  }
-
-  const owner = context.repo.owner;
-  const repo = context.repo.repo;
-
-  // Ensure marker is embedded in the body
-  const markedBody = body.includes(marker) ? body : `${marker}\n${body}`;
-
-  async function listComments() {
-    return await github.paginate(github.rest.issues.listComments, {
-      owner,
-      repo,
-      issue_number: prNumber,
-      per_page: 100,
-    });
-  }
-
-  function commentActivityTime(comment) {
-    const timestamp =
-        Date.parse(comment.updated_at || comment.created_at || '');
-    return Number.isNaN(timestamp) ? 0 : timestamp;
-  }
-
-  function commentSortDescending(left, right) {
-    const timeDelta = commentActivityTime(right) - commentActivityTime(left);
-    if (timeDelta !== 0) {
-      return timeDelta;
-    }
-    return Number(right.id || 0) - Number(left.id || 0);
-  }
-
-  function isObsoleteComment(comment) {
-    return !!obsoleteMarker && typeof comment.body === 'string' &&
-        comment.body.includes(obsoleteMarker);
-  }
-
-  function buildObsoleteBody(comment) {
-    const originalBody =
-        typeof comment.body === 'string' && comment.body.trim() ?
-        comment.body :
-        '*No original review body preserved.*';
-    const title = obsoleteTitle || 'AI Review - OBSOLETE';
-    return `${obsoleteMarker}\n## ${
-        title}\n\n<details>\n<summary>Superseded by a newer AI review. Expand to see the original review.</summary>\n\n${
-        originalBody}\n\n</details>`;
-  }
-
-  async function deleteCommentIfPresent(comment, reason) {
-    try {
-      await github.rest.issues.deleteComment({
-        owner,
-        repo,
-        comment_id: comment.id,
-      });
-      core.info(`${reason} ${comment.id}`);
-    } catch (error) {
-      if (error.status === 404) {
-        core.info(`Comment ${comment.id} was already deleted by another run.`);
-        return;
-      }
-      throw error;
-    }
-  }
-
-  async function supersedeCommentIfPresent(comment, reason) {
-    if (!obsoleteTitle) {
-      await deleteCommentIfPresent(comment, reason);
-      return;
-    }
-    if (isObsoleteComment(comment)) {
-      return;
-    }
-    try {
-      await github.rest.issues.updateComment({
-        owner,
-        repo,
-        comment_id: comment.id,
-        body: buildObsoleteBody(comment),
-      });
-      core.info(`${reason} ${comment.id}`);
-    } catch (error) {
-      if (error.status === 404) {
-        core.info(`Comment ${comment.id} was already deleted by another run.`);
-        return;
-      }
-      throw error;
-    }
-  }
-
-  let comments = await listComments();
-  const existing = comments.find(
-      comment =>
-          typeof comment.body === 'string' && comment.body.includes(marker));
-  let currentCommentId = null;
-
-  if (existing) {
-    try {
-      const response = await github.rest.issues.updateComment({
-        owner,
-        repo,
-        comment_id: existing.id,
-        body: markedBody,
-      });
-      currentCommentId = existing.id;
-      core.info(`Updated existing comment ${existing.id}`);
-    } catch (error) {
-      if (error.status !== 404) {
-        throw error;
-      }
-      core.info(
-          `Comment ${existing.id} disappeared before update; ` +
-          'creating a fresh comment instead.');
-    }
-  }
-
-  if (!currentCommentId) {
-    const response = await github.rest.issues.createComment({
-      owner,
-      repo,
-      issue_number: prNumber,
-      body: markedBody,
-    });
-    currentCommentId = response.data.id;
-    core.info('Created new PR comment');
-  }
-
-  if (prunePrefix || legacyMarkers.length > 0) {
-    comments = await listComments();
-    const relatedComments = comments.filter(
-        comment => comment.id !== currentCommentId &&
-            typeof comment.body === 'string' &&
-            ((prunePrefix && comment.body.includes(prunePrefix)) ||
-             legacyMarkers.some(
-                 legacyMarker => comment.body.includes(legacyMarker))));
-    const activeRelatedComments =
-        comments
-            .filter(
-                comment => typeof comment.body === 'string' &&
-                    !isObsoleteComment(comment) &&
-                    (comment.id === currentCommentId ||
-                     relatedComments.some(
-                         relatedComment => relatedComment.id === comment.id)))
-            .sort(commentSortDescending);
-
-    const keep =
-        new Set(activeRelatedComments.slice(0, Math.max(preserveLatest, 1))
-                    .map(comment => comment.id));
-
-    const supersedeCandidates = obsoleteTitle ?
-        activeRelatedComments.filter(comment => !keep.has(comment.id)) :
-        relatedComments.filter(comment => !keep.has(comment.id));
-
-    for (const comment of supersedeCandidates) {
-      if (keep.has(comment.id)) {
-        continue;
-      }
-      await supersedeCommentIfPresent(comment, 'Superseded old AI review');
-    }
-  }
-};

.github/scripts/post-pr-comment.test.js

@@ -1,347 +0,0 @@
-const test = require('node:test');
-const assert = require('node:assert/strict');
-
-const postPrComment = require('./post-pr-comment.js');
-
-const OBSOLETE_MARKER = '<!-- claude-review-obsolete -->';
-const OBSOLETE_TITLE = 'Claude Code Review - OBSOLETE';
-
-function makeComment(id, body, createdAt, updatedAt) {
-  return {
-    id,
-    body,
-    created_at: createdAt,
-    updated_at: updatedAt || createdAt,
-  };
-}
-
-function createHarness(initialComments, options = {}) {
-  let comments = initialComments.map(comment => ({...comment}));
-  let nextCommentId = options.nextCommentId || 1000;
-  let paginateCount = 0;
-
-  const calls = {
-    update: [],
-    create: [],
-    delete: [],
-  };
-
-  const github = {
-    paginate: async () => {
-      paginateCount++;
-      if (options.onPaginate) {
-        const updated = options.onPaginate({
-          paginateCount,
-          comments: comments.map(comment => ({...comment})),
-        });
-        if (updated) {
-          comments = updated.map(comment => ({...comment}));
-        }
-      }
-      return comments.map(comment => ({...comment}));
-    },
-    rest: {
-      issues: {
-        listComments: () => {
-          throw new Error('listComments should only be used through paginate');
-        },
-        updateComment: async ({comment_id, body}) => {
-          calls.update.push({comment_id, body});
-          const error =
-              options.updateErrors && options.updateErrors[comment_id];
-          if (error) {
-            throw error;
-          }
-          const index =
-              comments.findIndex(comment => comment.id === comment_id);
-          if (index === -1) {
-            const notFound = new Error(`Comment ${comment_id} not found`);
-            notFound.status = 404;
-            throw notFound;
-          }
-          const updated = {
-            ...comments[index],
-            body,
-            updated_at: options.updateTimestamp || '2026-04-24T00:00:00Z',
-          };
-          comments[index] = updated;
-          return {data: {...updated}};
-        },
-        createComment: async ({issue_number, body}) => {
-          calls.create.push({issue_number, body});
-          const created = makeComment(
-              nextCommentId++, body,
-              options.createTimestamp || '2026-04-24T00:00:00Z');
-          comments.push(created);
-          return {data: {id: created.id}};
-        },
-        deleteComment: async ({comment_id}) => {
-          calls.delete.push({comment_id});
-          const error =
-              options.deleteErrors && options.deleteErrors[comment_id];
-          if (error) {
-            throw error;
-          }
-          comments = comments.filter(comment => comment.id !== comment_id);
-        },
-      },
-    },
-  };
-
-  return {
-    github,
-    calls,
-    getComments: () => comments.map(comment => ({...comment})),
-  };
-}
-
-function createCore() {
-  return {
-    info: () => {},
-    warning: () => {},
-  };
-}
-
-function escapeRegExp(value) {
-  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-}
-
-function assertObsoleteComment(comment, originalBody) {
-  assert.ok(comment);
-  assert.match(comment.body, new RegExp(escapeRegExp(OBSOLETE_MARKER)));
-  assert.match(comment.body, new RegExp(`## ${escapeRegExp(OBSOLETE_TITLE)}`));
-  assert.match(comment.body, /<details>/);
-  assert.match(comment.body, /Superseded by a newer AI review/);
-  assert.match(comment.body, new RegExp(escapeRegExp(originalBody)));
-}
-
-const context = {
-  repo: {
-    owner: 'facebook',
-    repo: 'rocksdb',
-  },
-};
-
-test(
-    'creates a fresh review comment and supersedes legacy comments',
-    async () => {
-      const harness = createHarness(
-          [
-            makeComment(
-                1, '<!-- claude-review-auto -->\nold legacy', 'not-a-date'),
-            makeComment(
-                2, '<!-- claude-review-auto -->\nnew legacy',
-                '2026-04-21T00:00:00Z'),
-          ],
-          {
-            updateTimestamp: '2026-04-24T00:00:00Z',
-          });
-
-      await postPrComment({
-        github: harness.github,
-        context,
-        core: createCore(),
-        prNumber: 14659,
-        body: 'review body',
-        marker: '<!-- claude-review-auto-run-500 -->',
-        legacyMarkers: ['<!-- claude-review-auto -->'],
-        prunePrefix: '<!-- claude-review-auto-',
-        preserveLatest: 1,
-        obsoleteMarker: OBSOLETE_MARKER,
-        obsoleteTitle: OBSOLETE_TITLE,
-      });
-
-      assert.equal(harness.calls.create.length, 1);
-      assert.deepEqual(
-          harness.calls.update.map(call => call.comment_id), [2, 1]);
-      assert.equal(harness.calls.delete.length, 0);
-
-      const comments = harness.getComments();
-      assert.equal(comments.length, 3);
-      assert.match(
-          comments.find(comment => comment.id === 1000).body,
-          /<!-- claude-review-auto-run-500 -->/);
-      assertObsoleteComment(
-          comments.find(comment => comment.id === 2),
-          '<!-- claude-review-auto -->\nnew legacy');
-      assertObsoleteComment(
-          comments.find(comment => comment.id === 1),
-          '<!-- claude-review-auto -->\nold legacy');
-    });
-
-test(
-    'updates an exact-match comment and supersedes leftover legacy comments',
-    async () => {
-      const harness = createHarness(
-          [
-            makeComment(
-                3, '<!-- claude-review-auto-abcdef0 -->\ncurrent body',
-                '2026-04-24T00:00:00Z'),
-            makeComment(
-                4, '<!-- claude-review-auto -->\nlegacy body',
-                '2026-04-20T00:00:00Z'),
-          ],
-          {
-            updateTimestamp: '2026-04-24T01:00:00Z',
-          });
-
-      await postPrComment({
-        github: harness.github,
-        context,
-        core: createCore(),
-        prNumber: 14659,
-        body: 'refreshed body',
-        marker: '<!-- claude-review-auto-abcdef0 -->',
-        legacyMarkers: ['<!-- claude-review-auto -->'],
-        prunePrefix: '<!-- claude-review-auto-',
-        preserveLatest: 1,
-        obsoleteMarker: OBSOLETE_MARKER,
-        obsoleteTitle: OBSOLETE_TITLE,
-      });
-
-      assert.deepEqual(
-          harness.calls.update.map(call => call.comment_id), [3, 4]);
-      assert.equal(harness.calls.create.length, 0);
-      assert.equal(harness.calls.delete.length, 0);
-
-      const comments = harness.getComments();
-      assert.match(
-          comments.find(comment => comment.id === 3).body,
-          /<!-- claude-review-auto-abcdef0 -->\nrefreshed body/);
-      assertObsoleteComment(
-          comments.find(comment => comment.id === 4),
-          '<!-- claude-review-auto -->\nlegacy body');
-    });
-
-test(
-    'creates a new comment if the target disappears before update',
-    async () => {
-      const missing = new Error('gone');
-      missing.status = 404;
-      const harness = createHarness(
-          [
-            makeComment(
-                10, '<!-- claude-review-auto-abcdef0 -->\nold body',
-                '2026-04-20T00:00:00Z'),
-          ],
-          {
-            updateErrors: {
-              10: missing,
-            },
-          });
-
-      await postPrComment({
-        github: harness.github,
-        context,
-        core: createCore(),
-        prNumber: 14659,
-        body: 'replacement body',
-        marker: '<!-- claude-review-auto-abcdef0 -->',
-      });
-
-      assert.equal(harness.calls.update.length, 1);
-      assert.equal(harness.calls.create.length, 1);
-      assert.match(
-          harness.calls.create[0].body, /<!-- claude-review-auto-abcdef0 -->/);
-    });
-
-test(
-    'ignores 404 when superseding a comment already deleted by another run',
-    async () => {
-      const missing = new Error('gone');
-      missing.status = 404;
-      const harness = createHarness(
-          [
-            makeComment(
-                20, '<!-- claude-review-auto-oldest -->\noldest',
-                '2026-04-20T00:00:00Z'),
-            makeComment(
-                21, '<!-- claude-review-auto-newer -->\nnewer',
-                '2026-04-21T00:00:00Z'),
-          ],
-          {
-            nextCommentId: 30,
-            createTimestamp: '2026-04-24T00:00:00Z',
-            deleteErrors: {
-              20: missing,
-            },
-          });
-
-      await postPrComment({
-        github: harness.github,
-        context,
-        core: createCore(),
-        prNumber: 14659,
-        body: 'fresh body',
-        marker: '<!-- claude-review-auto-latest -->',
-        prunePrefix: '<!-- claude-review-auto-',
-        preserveLatest: 1,
-        obsoleteMarker: OBSOLETE_MARKER,
-        obsoleteTitle: OBSOLETE_TITLE,
-      });
-
-      assert.equal(harness.calls.create.length, 1);
-      assert.equal(harness.calls.delete.length, 0);
-      assert.deepEqual(
-          harness.calls.update.map(call => call.comment_id), [21, 20]);
-
-      const comments = harness.getComments();
-      assertObsoleteComment(
-          comments.find(comment => comment.id === 21),
-          '<!-- claude-review-auto-newer -->\nnewer');
-    });
-
-test(
-    'supersedes the current review when a newer concurrent one appears',
-    async () => {
-      const harness = createHarness(
-          [
-            makeComment(
-                40, '<!-- claude-review-auto-current -->\ncurrent',
-                '2026-04-20T00:00:00Z'),
-            makeComment(
-                41, '<!-- claude-review-auto-older -->\nolder',
-                '2026-04-19T00:00:00Z'),
-          ],
-          {
-            updateTimestamp: '2026-04-24T00:00:00Z',
-            onPaginate: ({paginateCount, comments}) => {
-              if (paginateCount !== 2) {
-                return comments;
-              }
-              return comments.concat([
-                makeComment(
-                    42, '<!-- claude-review-auto-newer -->\nnewer',
-                    '2026-04-25T00:00:00Z'),
-              ]);
-            },
-          });
-
-      await postPrComment({
-        github: harness.github,
-        context,
-        core: createCore(),
-        prNumber: 14659,
-        body: 'updated current body',
-        marker: '<!-- claude-review-auto-current -->',
-        prunePrefix: '<!-- claude-review-auto-',
-        preserveLatest: 1,
-        obsoleteMarker: OBSOLETE_MARKER,
-        obsoleteTitle: OBSOLETE_TITLE,
-      });
-
-      assert.deepEqual(
-          harness.calls.update.map(call => call.comment_id), [40, 40, 41]);
-      assert.equal(harness.calls.delete.length, 0);
-
-      const comments = harness.getComments();
-      assert.match(
-          comments.find(comment => comment.id === 42).body,
-          /<!-- claude-review-auto-newer -->\nnewer/);
-      assertObsoleteComment(
-          comments.find(comment => comment.id === 40),
-          '<!-- claude-review-auto-current -->\nupdated current body');
-      assertObsoleteComment(
-          comments.find(comment => comment.id === 41),
-          '<!-- claude-review-auto-older -->\nolder');
-    });

.github/workflows/ai-review-comment.yml

@@ -1,185 +0,0 @@
-# Shared comment-posting workflow for AI reviews.
-
-name: AI Review Comment
-
-on:
-  workflow_call:
-    inputs:
-      provider:
-        description: Provider key, for example "claude" or "codex"
-        required: true
-        type: string
-      result_artifact_name:
-        description: Artifact name produced by the analysis workflow
-        required: true
-        type: string
-      comment_file:
-        description: Markdown comment file produced by the analysis workflow
-        required: true
-        type: string
-
-permissions:
-  pull-requests: write
-  issues: write
-
-jobs:
-  comment:
-    if: github.event.workflow_run.conclusion == 'success'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout scripts
-        uses: actions/checkout@v4
-        with:
-          sparse-checkout: .github/scripts
-          sparse-checkout-cone-mode: true
-
-      - name: Download review artifact
-        id: download
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.result_artifact_name }}
-          run-id: ${{ github.event.workflow_run.id }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-
-      - name: Post or update PR comment
-        if: steps.download.outcome == 'success'
-        env:
-          PROVIDER: ${{ inputs.provider }}
-          COMMENT_FILE: ${{ inputs.comment_file }}
-          RUN_ID: ${{ github.event.workflow_run.id }}
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const fs = require('fs');
-            if (!fs.existsSync(process.env.COMMENT_FILE) ||
-                !fs.existsSync('pr_number.txt')) {
-              core.info('No review results found; skipping.');
-              return;
-            }
-
-            const post = require('./.github/scripts/post-pr-comment.js');
-            const provider = process.env.PROVIDER;
-            const providerTitle = provider === 'codex' ? 'Codex' : 'Claude';
-            const body = fs.readFileSync(process.env.COMMENT_FILE, 'utf8');
-            const prNumber = parseInt(
-              fs.readFileSync('pr_number.txt', 'utf8').trim(),
-              10
-            );
-            const trigger = fs.existsSync('trigger_type.txt') ?
-              fs.readFileSync('trigger_type.txt', 'utf8').trim() :
-              'auto';
-            const headSha = fs.existsSync('head_sha.txt') ?
-              fs.readFileSync('head_sha.txt', 'utf8').trim() :
-              '';
-            const shortSha = headSha ? headSha.substring(0, 7) : '';
-            const runId = process.env.RUN_ID;
-
-            let marker = '';
-            let legacyMarkers = [];
-            let prunePrefix = '';
-            let preserveLatest = 0;
-            let obsoleteMarker = '';
-            let obsoleteTitle = '';
-
-            if (trigger === 'auto') {
-              if (!shortSha) {
-                core.warning(
-                  'Missing head_sha.txt for auto review; skipping comment.'
-                );
-                return;
-              }
-              marker = `<!-- ${provider}-review-auto-run-${runId} -->`;
-              legacyMarkers = [`<!-- ${provider}-review-auto -->`];
-              prunePrefix = `<!-- ${provider}-review-auto-`;
-              preserveLatest = 1;
-              obsoleteMarker = `<!-- ${provider}-review-obsolete -->`;
-              obsoleteTitle = `${providerTitle} Code Review - OBSOLETE`;
-            } else {
-              marker = `<!-- ${provider}-review-manual-run-${runId} -->`;
-            }
-
-            await post({
-              github,
-              context,
-              core,
-              prNumber,
-              body,
-              marker,
-              legacyMarkers,
-              prunePrefix,
-              preserveLatest,
-              obsoleteMarker,
-              obsoleteTitle,
-            });
-
-      - name: Add reaction to trigger comment
-        if: steps.download.outcome == 'success'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const fs = require('fs');
-            if (!fs.existsSync('trigger_type.txt')) return;
-            const trigger = fs.readFileSync('trigger_type.txt', 'utf8').trim();
-            if (trigger !== 'manual') return;
-            if (!fs.existsSync('comment_id.txt')) return;
-            const commentId = parseInt(
-              fs.readFileSync('comment_id.txt', 'utf8').trim(),
-              10
-            );
-            if (!commentId) return;
-
-            await github.rest.reactions.createForIssueComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              comment_id: commentId,
-              content: 'rocket'
-            });
-
-  failure-notice:
-    if: github.event.workflow_run.conclusion == 'failure'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download artifact for PR number
-        id: download
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ inputs.result_artifact_name }}
-          run-id: ${{ github.event.workflow_run.id }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-
-      - name: React with failure
-        if: steps.download.outcome == 'success'
-        uses: actions/github-script@v7
-        with:
-          script: |
-            const fs = require('fs');
-            if (!fs.existsSync('comment_id.txt')) return;
-            const commentId = parseInt(
-              fs.readFileSync('comment_id.txt', 'utf8').trim(),
-              10
-            );
-            if (!commentId) return;
-
-            await github.rest.reactions.createForIssueComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              comment_id: commentId,
-              content: 'confused'
-            });
-
-  unauthorized-notice:
-    runs-on: ubuntu-latest
-    if: >-
-      github.event.workflow_run.event == 'issue_comment' &&
-      github.event.workflow_run.conclusion == 'skipped'
-    steps:
-      - name: Log skipped unauthorized request
-        uses: actions/github-script@v7
-        with:
-          script: |
-            core.info(
-              'Analysis workflow was skipped, which usually means the ' +
-              'issue_comment requester was not in the authorized list.'
-            );

.github/workflows/benchmark-linux.yml

@@ -1,15 +0,0 @@
-name: facebook/rocksdb/benchmark-linux
-on: workflow_dispatch
-permissions: {}
-  # FIXME: Disabled temporarily
-  # schedule:
-  # - cron: 7 */2 * * *  # At minute 7 past every 2nd hour
-jobs:
-  benchmark-linux:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: ubuntu-latest # FIXME: change this back to self-hosted when ready
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/build-for-benchmarks"
-    - uses: "./.github/actions/perform-benchmarks"
-    - uses: "./.github/actions/post-benchmarks"

.github/workflows/clang-tidy-comment.yml

@@ -1,51 +0,0 @@
-name: Post clang-tidy PR comment
-on:
-  workflow_run:
-    workflows: ["clang-tidy"]
-    types: [completed]
-
-permissions:
-  pull-requests: write
-
-jobs:
-  comment:
-    if: github.event.workflow_run.event == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Checkout scripts
-      uses: actions/checkout@v4
-      with:
-        sparse-checkout: .github/scripts
-        sparse-checkout-cone-mode: true
-
-    - name: Download clang-tidy results
-      id: download
-      uses: actions/download-artifact@v4.1.3
-      with:
-        name: clang-tidy-result
-        run-id: ${{ github.event.workflow_run.id }}
-        github-token: ${{ secrets.GITHUB_TOKEN }}
-      continue-on-error: true
-
-    - name: Post or update PR comment
-      if: steps.download.outcome == 'success'
-      uses: actions/github-script@v7
-      with:
-        script: |
-          const fs = require('fs');
-          if (!fs.existsSync('clang-tidy-comment.md') || !fs.existsSync('pr_number.txt')) {
-            core.info('No clang-tidy results found; skipping.');
-            return;
-          }
-          const body = fs.readFileSync('clang-tidy-comment.md', 'utf8');
-          const prNumber = parseInt(fs.readFileSync('pr_number.txt', 'utf8').trim());
-          const post = require('./.github/scripts/post-pr-comment.js');
-
-          await post({
-            github,
-            context,
-            core,
-            prNumber,
-            body,
-            marker: '<!-- clang-tidy-bot -->',
-          });

.github/workflows/clang-tidy.yml

@@ -1,74 +0,0 @@
-name: clang-tidy
-on:
-  push:
-  pull_request:
-
-permissions: {}
-
-jobs:
-  clang-tidy:
-    if: github.repository_owner == 'facebook'
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-    steps:
-    - uses: actions/checkout@v4.1.0
-      with:
-        fetch-depth: 2
-    - name: Mark workspace as safe for git
-      run: git config --global --add safe.directory $GITHUB_WORKSPACE
-    - name: Determine diff base
-      id: diff-base
-      run: |
-        if [ "${{ github.event_name }}" = "pull_request" ]; then
-          BASE="${{ github.event.pull_request.base.sha }}"
-          git fetch --depth=1 origin "$BASE"
-        else
-          BASE="${{ github.event.before }}"
-          if echo "$BASE" | grep -q '^0\{40\}$'; then
-            echo "skip=true" >> "$GITHUB_OUTPUT"
-            echo "New branch push; skipping clang-tidy."
-            exit 0
-          fi
-        fi
-        echo "ref=$BASE" >> "$GITHUB_OUTPUT"
-        echo "skip=false" >> "$GITHUB_OUTPUT"
-    - name: Install clang-tidy
-      if: steps.diff-base.outputs.skip != 'true'
-      run: apt-get update && apt-get install -y clang-tidy-21 && ln -sf /usr/bin/clang-tidy-21 /usr/local/bin/clang-tidy
-    - name: Generate compile_commands.json
-      if: steps.diff-base.outputs.skip != 'true'
-      run: |
-        mkdir build && cd build
-        cmake -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
-              -DCMAKE_C_COMPILER=clang-21 \
-              -DCMAKE_CXX_COMPILER=clang++-21 ..
-        cd ..
-        ln -sf build/compile_commands.json compile_commands.json
-    - name: Run clang-tidy on changed files
-      id: clang-tidy
-      if: steps.diff-base.outputs.skip != 'true'
-      run: |
-        python3 tools/run_clang_tidy.py \
-          -j 4 \
-          --diff-base ${{ steps.diff-base.outputs.ref }} \
-          --github-annotations \
-          --github-step-summary \
-          --comment-output clang-tidy-comment.md
-      continue-on-error: true
-    - name: Save PR number
-      if: github.event_name == 'pull_request' && always()
-      run: echo "${{ github.event.pull_request.number }}" > pr_number.txt
-    - name: Upload clang-tidy results
-      if: always()
-      uses: actions/upload-artifact@v4.0.0
-      with:
-        name: clang-tidy-result
-        path: |
-          clang-tidy-comment.md
-          pr_number.txt
-        if-no-files-found: ignore
-    - name: Fail if clang-tidy found issues
-      if: steps.clang-tidy.outcome == 'failure'
-      run: exit 1

.github/workflows/claude-review-comment.yml

@@ -1,25 +0,0 @@
-# Claude Code Review — Comment Posting Workflow
-#
-# Thin wrapper around .github/workflows/ai-review-comment.yml so the provider
-# specific workflow name stays stable while the shared implementation lives in
-# one reusable workflow.
-
-name: Post Claude Review Comment
-
-on:
-  workflow_run:
-    workflows: ["Claude Code Review"]
-    types: [completed]
-
-permissions:
-  pull-requests: write
-  issues: write
-
-jobs:
-  review-comment:
-    uses: ./.github/workflows/ai-review-comment.yml
-    with:
-      provider: claude
-      result_artifact_name: claude-review-result
-      comment_file: claude-review-comment.md
-    secrets: inherit

.github/workflows/claude-review.yml

@@ -1,69 +0,0 @@
-# Claude Code Review — Analysis Workflow
-#
-# Thin wrapper around .github/workflows/ai-review-analysis.yml so provider
-# specific triggers and workflow_dispatch inputs stay stable while the shared
-# implementation lives in one reusable workflow.
-
-name: Claude Code Review
-
-on:
-  workflow_run:
-    workflows: ["facebook/rocksdb/pr-jobs"]
-    types: [completed]
-
-  # The early pull_request_target path is limited to same-repo PRs by the job
-  # condition below. Fork PRs skip this path and rely on workflow_run instead.
-  pull_request_target:
-    types: [opened, reopened, synchronize]
-
-  issue_comment:
-    types: [created]
-
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: PR number to review
-        required: true
-        type: number
-      model:
-        description: Claude model to use (defaults to latest Opus)
-        required: false
-        type: choice
-        options:
-          - claude-opus-4-6
-          - claude-sonnet-4-6
-        default: claude-opus-4-6
-      thinking_budget:
-        description: Override MAX_THINKING_TOKENS (blank = auto-classify, capped at 24000)
-        required: false
-        type: string
-        default: ""
-
-permissions:
-  contents: read
-  # The shared analysis workflow polls check state, inspects prior runs, and
-  # reads PR metadata. Keep those permissions read-only in the caller.
-  actions: read
-  checks: read
-  pull-requests: read
-
-jobs:
-  review:
-    if: >-
-      github.event_name != 'pull_request_target' ||
-      github.event.pull_request.head.repo.full_name == github.repository
-    uses: ./.github/workflows/ai-review-analysis.yml
-    with:
-      provider: claude
-      display_name: Claude
-      review_command: /claude-review
-      query_command: /claude-query
-      result_artifact_name: claude-review-result
-      comment_file: claude-review-comment.md
-      default_model: claude-opus-4-6
-      selected_model: ${{ github.event_name == 'workflow_dispatch' && inputs.model || '' }}
-      classifier_model: claude-sonnet-4-6
-      recovery_model: claude-sonnet-4-6
-      thinking_budget: ${{ github.event_name == 'workflow_dispatch' && inputs.thinking_budget || '' }}
-      dispatch_pr_number: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || '' }}
-    secrets: inherit

.github/workflows/codex-review-comment.yml

@@ -1,25 +0,0 @@
-# Codex Code Review — Comment Posting Workflow
-#
-# Thin wrapper around .github/workflows/ai-review-comment.yml so the provider
-# specific workflow name stays stable while the shared implementation lives in
-# one reusable workflow.
-
-name: Post Codex Review Comment
-
-on:
-  workflow_run:
-    workflows: ["Codex Code Review"]
-    types: [completed]
-
-permissions:
-  pull-requests: write
-  issues: write
-
-jobs:
-  review-comment:
-    uses: ./.github/workflows/ai-review-comment.yml
-    with:
-      provider: codex
-      result_artifact_name: codex-review-result
-      comment_file: codex-review-comment.md
-    secrets: inherit

.github/workflows/codex-review.yml

@@ -1,68 +0,0 @@
-# Codex Code Review — Analysis Workflow
-#
-# Thin wrapper around .github/workflows/ai-review-analysis.yml so provider
-# specific triggers and workflow_dispatch inputs stay stable while the shared
-# implementation lives in one reusable workflow.
-
-name: Codex Code Review
-
-on:
-  workflow_run:
-    workflows: ["facebook/rocksdb/pr-jobs"]
-    types: [completed]
-
-  # The early pull_request_target path is limited to same-repo PRs by the job
-  # condition below. Fork PRs skip this path and rely on workflow_run instead.
-  pull_request_target:
-    types: [opened, reopened, synchronize]
-
-  issue_comment:
-    types: [created]
-
-  workflow_dispatch:
-    inputs:
-      pr_number:
-        description: PR number to review
-        required: true
-        type: number
-      model:
-        description: Codex model to use
-        required: false
-        type: choice
-        options:
-          - gpt-5.5
-          - gpt-5.3-codex
-          - gpt-5.2-codex
-        default: gpt-5.5
-      thinking_budget:
-        description: Override MAX_THINKING_TOKENS (blank = auto-classify)
-        required: false
-        type: string
-        default: ""
-
-permissions:
-  contents: read
-  # The shared analysis workflow polls check state, inspects prior runs, and
-  # reads PR metadata. Keep those permissions read-only in the caller.
-  actions: read
-  checks: read
-  pull-requests: read
-
-jobs:
-  review:
-    if: >-
-      github.event_name != 'pull_request_target' ||
-      github.event.pull_request.head.repo.full_name == github.repository
-    uses: ./.github/workflows/ai-review-analysis.yml
-    with:
-      provider: codex
-      display_name: Codex
-      review_command: /codex-review
-      query_command: /codex-query
-      result_artifact_name: codex-review-result
-      comment_file: codex-review-comment.md
-      default_model: gpt-5.5
-      selected_model: ${{ github.event_name == 'workflow_dispatch' && inputs.model || '' }}
-      thinking_budget: ${{ github.event_name == 'workflow_dispatch' && inputs.thinking_budget || '' }}
-      dispatch_pr_number: ${{ github.event_name == 'workflow_dispatch' && inputs.pr_number || '' }}
-    secrets: inherit

.github/workflows/nightly-candidate.yml

@@ -1,19 +0,0 @@
-name: facebook/rocksdb/nightly
-on: workflow_dispatch
-permissions: {}
-jobs:
-  # These jobs would be in nightly but are failing or otherwise broken for
-  # some reason.
-  build-linux-arm-test-full:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: arm64large
-    container:
-      image: ubuntu-2004:202111-02
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/install-gflags"
-    - run: make V=1 J=4 -j4 check
-    - uses: "./.github/actions/post-steps"

.github/workflows/nightly.yml

@@ -1,190 +0,0 @@
-name: facebook/rocksdb/nightly
-on:
-  schedule:
-  - cron: 0 9 * * *
-  workflow_dispatch:
-permissions: {}
-jobs:
-  build-format-compatible:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-      with:
-        fetch-depth: 0 # Need full repo history
-        fetch-tags: true
-    - uses: "./.github/actions/setup-upstream"
-    - uses: "./.github/actions/pre-steps"
-    - name: test
-      run: |-
-        export TEST_TMPDIR=/dev/shm/rocksdb
-        rm -rf /dev/shm/rocksdb
-        mkdir /dev/shm/rocksdb
-        git config --global --add safe.directory /__w/rocksdb/rocksdb
-        tools/check_format_compatible.sh
-    - uses: "./.github/actions/post-steps"
-  build-linux-non-shm:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    env:
-      TEST_TMPDIR: "/tmp/rocksdb_test_tmp"
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: make V=1 -j32 check
-    - uses: "./.github/actions/post-steps"
-  build-linux-clang-21-asan-ubsan-with-folly:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    env:
-      CC: clang-21
-      CXX: clang++-21
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - name: Build folly and dependencies
-      run: |
-        clean_path=()
-        IFS=: read -ra path_entries <<< "$PATH"
-        for entry in "${path_entries[@]}"; do
-          if [[ "$entry" != "/usr/lib/ccache" ]]; then
-            clean_path+=("$entry")
-          fi
-        done
-        export PATH="$(IFS=:; echo "${clean_path[*]}")"
-        ccache_bin="$(command -v ccache || true)"
-        if [[ -n "$ccache_bin" && -x "$ccache_bin" ]]; then
-          mv "$ccache_bin" "${ccache_bin}.disabled"
-        fi
-        export CC=gcc
-        export CXX=g++
-        export USE_CCACHE=0
-        export CCACHE_DISABLE=1
-        make build_folly
-      shell: bash
-    - run: LIB_MODE=static USE_CLANG=1 USE_FOLLY=1 COMPILE_WITH_UBSAN=1 COMPILE_WITH_ASAN=1 make -j32 check
-    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-with-folly:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - uses: "./.github/actions/cache-folly"
-      id: cache-folly
-    - uses: "./.github/actions/build-folly"
-      with:
-        cache-hit: ${{ steps.cache-folly.outputs.cache-hit }}
-    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make VERBOSE=1 -j20 && ctest -j20)"
-    - uses: "./.github/actions/post-steps"
-  build-linux-release-with-folly:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - run: "DEBUG_LEVEL=0 make -j20 build_folly"
-    - run: "USE_FOLLY=1 LIB_MODE=static DEBUG_LEVEL=0 V=1 make -j20 release"
-    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 -DCMAKE_BUILD_TYPE=Release .. && make VERBOSE=1 -j20 && ctest -j20)"
-    - uses: "./.github/actions/post-steps"
-  build-windows-vs2022-avx2:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on: windows-2022
-    env:
-      CMAKE_GENERATOR: Visual Studio 17 2022
-      CMAKE_PORTABLE: AVX2
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/windows-build-steps"
-  build-linux-arm-test-full:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu-arm
-    steps:
-      - uses: actions/checkout@v4.1.0
-      - uses: "./.github/actions/pre-steps"
-      - run: sudo apt-get update && sudo apt-get install -y build-essential libgflags-dev
-      - run: make V=1 J=4 -j4 check
-      - uses: "./.github/actions/post-steps"
-  build-linux-arm-crashtest:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu-arm
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: sudo apt-get update && sudo apt-get install -y build-essential libgflags-dev libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-    - run: sudo mount -o remount,size=16G /dev/shm
-    - run: sudo dd bs=1048576 count=4096 if=/dev/zero of=/swapfile && sudo chmod 600 /swapfile && sudo mkswap /swapfile && sudo swapon /swapfile
-    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=1800 --max_key=2500000' blackbox_crash_test_with_atomic_flush
-    - run: rm -rf /dev/shm/rocksdb.*
-    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=1800 --max_key=2500000' blackbox_crash_test_with_multiops_wc_txn
-    - uses: "./.github/actions/post-steps"
-  build-examples:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - name: Build examples
-      run: make V=1 -j4 static_lib && cd examples && make V=1 -j4
-    - uses: "./.github/actions/post-steps"
-  build-fuzzers:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - name: Build rocksdb lib
-      run: CC=clang-21 CXX=clang++-21 USE_CLANG=1 make -j4 static_lib
-    - name: Build fuzzers
-      run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
-    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-with-folly-lite:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 -DCMAKE_CXX_FLAGS=-DGLOG_USE_GLOG_EXPORT .. && make VERBOSE=1 -j20 && ctest -j20)"
-    - uses: "./.github/actions/post-steps"

.github/workflows/pr-jobs-candidate.yml

@@ -1,47 +0,0 @@
-name: facebook/rocksdb/pr-jobs-candidate
-on: workflow_dispatch
-permissions: {}
-jobs:
-  # These jobs would be in pr-jobs but are failing or otherwise broken for
-  # some reason.
-  # =========================== ARM Jobs ============================ #
-  build-linux-arm:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: arm64large # GitHub hosted ARM runners do not yet exist
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/install-gflags"
-    - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
-    - uses: "./.github/actions/post-steps"
-  build-linux-arm-cmake-no_test_run:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: arm64large # GitHub hosted ARM runners do not yet exist
-    env:
-      JAVA_HOME: "/usr/lib/jvm/java-8-openjdk-arm64"
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/install-gflags"
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        echo 'export PATH=$JAVA_HOME/bin:$PATH' >> $BASH_ENV
-        which java && java -version
-        which javac && javac -version
-    - name: Build with cmake
-      run: |-
-        mkdir build
-        cd build
-        cmake -DCMAKE_BUILD_TYPE=Release -DWITH_TESTS=0 -DWITH_GFLAGS=1 -DWITH_BENCHMARK_TOOLS=0 -DWITH_TOOLS=0 -DWITH_CORE_TOOLS=1 ..
-        make -j4
-    - name: Build Java with cmake
-      run: |-
-        rm -rf build
-        mkdir build
-        cd build
-        cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=1 ..
-        make -j4 rocksdb rocksdbjni
-    - uses: "./.github/actions/post-steps"

.github/workflows/pr-jobs.yml

@@ -1,727 +0,0 @@
-name: facebook/rocksdb/pr-jobs
-on: [push, pull_request]
-permissions: {}
-env:
-  # Set to a job name to run only that job (on any repo), or leave empty for
-  # normal behavior (all jobs on facebook repo only).
-  ONLY_JOB: ''
-jobs:
-  config:
-    runs-on: ubuntu-latest
-    outputs:
-      only_job: ${{ steps.set.outputs.only_job }}
-    steps:
-    - id: set
-      run: echo "only_job=$ONLY_JOB" >> "$GITHUB_OUTPUT"
-  # NOTE: multiple workflows would be recommended, but the current GHA UI in
-  # PRs doesn't make it clear when there's an overall error with a workflow,
-  # making it easy to overlook something broken. Grouping everything into one
-  # workflow minimizes the problem because it will be suspicious if there are
-  # no GHA results.
-  #
-  # The if: ${{ github.repository_owner == 'facebook' }} lines prevent the
-  # jobs from attempting to run on repo forks, because of a few problems:
-  # * runs-on labels are repository (owner) specific, so the job might wait
-  # for days waiting for a runner that simply isn't available.
-  # * Pushes to branches on forks for pull requests (the normal process) would
-  # run the workflow jobs twice: once in the pull-from fork and once for the PR
-  # destination repo. This is wasteful and dumb.
-  # * It is not known how to avoid copy-pasting the line to each job,
-  # increasing the risk of misconfiguration, especially on forks that might
-  # want to run with this GHA setup.
-  #
-  # SELECTIVE JOB EXECUTION: Set the ONLY_JOB env var at the top of this file
-  # to a job name (e.g. "build-linux-clang-tidy") to run only that job,
-  # bypassing the repository owner check. Leave it empty for normal behavior.
-  #
-  # DEBUGGING WITH SSH: Temporarily add this as a job step, either before the
-  # step of interest without the "if:" line or after the failing step with the
-  # "if:" line. Then use ssh command printed in CI output.
-  #  - name: Setup tmate session # TEMPORARY!
-  #    if: ${{ failure() }}
-  #    uses: mxschmitt/action-tmate@v3
-  #    with:
-  #      limit-access-to-actor: true
-
-  # ======================== Fast Initial Checks ====================== #
-  check-format-and-targets:
-    if: needs.config.outputs.only_job == 'check-format-and-targets' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: ubuntu-24.04
-    steps:
-    - uses: actions/checkout@v4.1.0
-      with:
-        fetch-depth: 0 # Need full checkout to determine merge base
-        fetch-tags: true
-    - uses: "./.github/actions/setup-upstream"
-    - name: Setup Python
-      uses: actions/setup-python@v5
-    - name: Install Dependencies
-      run: python -m pip install --upgrade pip
-    - name: Install argparse
-      run: pip install argparse
-    - name: Install clang-format
-      run: |
-        pip install https://files.pythonhosted.org/packages/fb/ac/3c04772acc0257f5730e83adb542b2603c1a62d1315010ab593a980af404/clang_format-21.1.2-py2.py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl
-        clang-format --version
-    - name: Download clang-format-diff.py
-      run: wget https://rocksdb-deps.s3.us-west-2.amazonaws.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
-    - name: Check format
-      run: VERBOSE_CHECK=1 make check-format
-    - name: Compare buckify output
-      run: make check-buck-targets
-    - name: Simple source code checks
-      run: make check-sources
-    - name: Validate GitHub Actions YAML
-      run: make check-workflow-yaml
-    - name: Sanity check check_format_compatible.sh
-      run: |-
-        export TEST_TMPDIR=/dev/shm/rocksdb
-        rm -rf /dev/shm/rocksdb
-        mkdir /dev/shm/rocksdb
-        git reset --hard
-        git config --global --add safe.directory /__w/rocksdb/rocksdb
-        SANITY_CHECK=1 LONG_TEST=1 tools/check_format_compatible.sh
-  # ========================= Linux With Tests ======================== #
-  build-linux:
-    if: needs.config.outputs.only_job == 'build-linux' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: build-linux
-    - run: make V=1 J=32 -j32 check
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-mingw:
-    if: needs.config.outputs.only_job == 'build-linux-cmake-mingw' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: cmake-mingw
-    - run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
-    - name: Build cmake-mingw
-      run: |-
-        export PATH=$JAVA_HOME/bin:$PATH
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-        mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-make-with-folly:
-    if: needs.config.outputs.only_job == 'build-linux-make-with-folly' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 32-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: make-with-folly
-    - uses: "./.github/actions/cache-folly"
-      id: cache-folly
-    - uses: "./.github/actions/build-folly"
-      with:
-        cache-hit: ${{ steps.cache-folly.outputs.cache-hit }}
-    - run: USE_FOLLY=1 LIB_MODE=static V=1 make -j64 check
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-make-with-folly-lite-no-test:
-    if: needs.config.outputs.only_job == 'build-linux-make-with-folly-lite-no-test' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: make-folly-lite
-    - run: USE_FOLLY_LITE=1 EXTRA_CXXFLAGS=-DGLOG_USE_GLOG_EXPORT V=1 make -j32 all
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-with-folly-coroutines:
-    if: needs.config.outputs.only_job == 'build-linux-cmake-with-folly-coroutines' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 32-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/cache-getdeps-downloads"
-    - uses: "./.github/actions/setup-folly"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: cmake-folly-coroutines
-    - uses: "./.github/actions/cache-folly"
-      id: cache-folly
-    - uses: "./.github/actions/build-folly"
-      with:
-        cache-hit: ${{ steps.cache-folly.outputs.cache-hit }}
-    - run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 -DPORTABLE=ON .. && make VERBOSE=1 -j64 && ctest -j64)"
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-cmake-with-benchmark-no-thread-status:
-    if: needs.config.outputs.only_job == 'build-linux-cmake-with-benchmark-no-thread-status' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [0, 1, 2, 3]
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: cmake-benchmark
-    - name: Build
-      run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 -DPORTABLE=ON -DCMAKE_CXX_FLAGS=-DNROCKSDB_THREAD_STATUS .. && make VERBOSE=1 -j20
-    - name: Test shard ${{ matrix.shard }} of 4
-      run: cd build && ctest -j20 -I ${{ matrix.shard }},,4
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-      with:
-        artifact-prefix: "${{ github.job }}-shard${{ matrix.shard }}"
-  build-linux-encrypted_env-no_compression:
-    if: needs.config.outputs.only_job == 'build-linux-encrypted_env-no_compression' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: encrypted-env-no-compression
-    - run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
-    - run: "./sst_dump --help | grep -E -q 'Supported built-in compression types: kNoCompression$' # Verify no compiled in compression\n"
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  # ======================== Linux No Test Runs ======================= #
-  build-linux-release:
-    if: needs.config.outputs.only_job == 'build-linux-release' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: release
-    - run: make V=1 -j32 LIB_MODE=shared release
-    - run: ls librocksdb.so
-    - run: "./trace_analyzer --version" # A tool dependent on gflags that can run in release build
-    - run: make clean
-    - run: USE_RTTI=1 make V=1 -j32 release
-    - run: ls librocksdb.a
-    - run: "./trace_analyzer --version"
-    - run: make clean
-    - run: apt-get remove -y libgflags-dev
-    - run: make V=1 -j32 LIB_MODE=shared release
-    - run: ls librocksdb.so
-    - run: if ./trace_analyzer --version; then false; else true; fi
-    - run: make clean
-    - run: USE_RTTI=1 make V=1 -j32 release
-    - run: ls librocksdb.a
-    - run: if ./trace_analyzer --version; then false; else true; fi
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-clang-13-no_test_run:
-    if: needs.config.outputs.only_job == 'build-linux-clang-13-no_test_run' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 8-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: clang-13
-    # FIXME: get back to "all microbench" targets
-    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 EXTRA_CXXFLAGS=-stdlib=libc++ EXTRA_LDFLAGS=-stdlib=libc++ make -j32 shared_lib
-    - run: make clean
-    # FIXME: get back to "release" target
-    - run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 EXTRA_CXXFLAGS=-stdlib=libc++ EXTRA_LDFLAGS=-stdlib=libc++ DEBUG_LEVEL=0 make -j32 shared_lib
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-clang-21-no_test_run:
-    if: needs.config.outputs.only_job == 'build-linux-clang-21-no_test_run' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: clang-21
-    - run: CC=clang-21 CXX=clang++-21 USE_CLANG=1 make -j32 all microbench
-    - run: make clean
-    - run: CC=clang-21 CXX=clang++-21 USE_CLANG=1 DEBUG_LEVEL=0 make -j32 release
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-gcc-14-no_test_run:
-    if: needs.config.outputs.only_job == 'build-linux-gcc-14-no_test_run' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: gcc-14
-    - run: CC=gcc-14 CXX=g++-14 V=1 make -j32 all microbench
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-
-  # ======================== Linux Other Checks ======================= #
-
-  build-linux-unity-and-headers:
-    if: needs.config.outputs.only_job == 'build-linux-unity-and-headers' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: gcc:latest
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - run: apt-get update -y && apt-get install -y libgflags-dev
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: unity-headers
-    - name: Unity build
-      run: make V=1 -j8 unity_test
-    - run: make V=1 -j8 -k check-headers
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-mini-crashtest:
-    if: needs.config.outputs.only_job == 'build-linux-mini-crashtest' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - crash_test_target: blackbox_crash_test_with_atomic_flush
-          crash_duration: 480
-        - crash_test_target: blackbox_crash_test
-          crash_duration: 240
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: mini-crashtest
-    - run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=${{ matrix.crash_duration }} --max_key=2500000' ${{ matrix.crash_test_target }}
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-      with:
-        artifact-prefix: "${{ github.job }}-${{ matrix.crash_test_target }}"
-  # ======================= Linux with Sanitizers ===================== #
-  build-linux-clang21-asan-ubsan:
-    if: needs.config.outputs.only_job == 'build-linux-clang21-asan-ubsan' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 32-core-ubuntu
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [0, 1, 2]
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: clang21-asan-ubsan
-    - run: COMPILE_WITH_ASAN=1 COMPILE_WITH_UBSAN=1 CC=clang-21 CXX=clang++-21 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j40 check
-      env:
-        CI_SHARD_INDEX: ${{ matrix.shard }}
-        CI_TOTAL_SHARDS: 3
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-      with:
-        artifact-prefix: "${{ github.job }}-shard${{ matrix.shard }}"
-  build-linux-clang21-mini-tsan:
-    if: needs.config.outputs.only_job == 'build-linux-clang21-mini-tsan' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 32-core-ubuntu
-    strategy:
-      fail-fast: false
-      matrix:
-        shard: [0, 1, 2]
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:24.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: clang21-tsan
-    - run: COMPILE_WITH_TSAN=1 CC=clang-21 CXX=clang++-21 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
-      env:
-        CI_SHARD_INDEX: ${{ matrix.shard }}
-        CI_TOTAL_SHARDS: 3
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-      with:
-        artifact-prefix: "${{ github.job }}-shard${{ matrix.shard }}"
-  build-linux-static_lib-alt_namespace-status_checked:
-    if: needs.config.outputs.only_job == 'build-linux-static_lib-alt_namespace-status_checked' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 16-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: static-alt-namespace
-    - run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_USE_STD_SEMAPHORES -DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  # ========================= MacOS build only ======================== #
-  build-macos:
-    if: needs.config.outputs.only_job == 'build-macos' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: macos-15-xlarge
-    env:
-      ROCKSDB_DISABLE_JEMALLOC: 1
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: maxim-lobanov/setup-xcode@v1.6.0
-      with:
-        xcode-version: 16.4.0
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: macos
-    - uses: "./.github/actions/increase-max-open-files-on-macos"
-    - uses: "./.github/actions/install-gflags-on-macos"
-    - uses: "./.github/actions/pre-steps-macos"
-    - name: Build
-      run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j8 all
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  # ========================= MacOS with Tests ======================== #
-  build-macos-cmake:
-    if: needs.config.outputs.only_job == 'build-macos-cmake' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: macos-15-xlarge
-    strategy:
-      matrix:
-        run_sharded_tests: [0, 1, 2, 3]
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: maxim-lobanov/setup-xcode@v1.6.0
-      with:
-        xcode-version: 16.4.0
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: macos-cmake
-    - uses: "./.github/actions/increase-max-open-files-on-macos"
-    - uses: "./.github/actions/install-gflags-on-macos"
-    - uses: "./.github/actions/pre-steps-macos"
-    - name: cmake generate project file
-      run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
-    - name: Build tests
-      run: cd build && make VERBOSE=1 -j8
-    - name: Run shard 0 out of 4 test shards
-      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 0,,4
-      if: ${{ matrix.run_sharded_tests == 0 }}
-    - name: Run shard 1 out of 4 test shards
-      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 1,,4
-      if: ${{ matrix.run_sharded_tests == 1 }}
-    - name: Run shard 2 out of 4 test shards
-      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 2,,4
-      if: ${{ matrix.run_sharded_tests == 2 }}
-    - name: Run shard 3 out of 4 test shards
-      run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j8 -I 3,,4
-      if: ${{ matrix.run_sharded_tests == 3 }}
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  # ======================== Windows with Tests ======================= #
-  # NOTE: some windows jobs are in "nightly" to save resources
-  build-windows-vs2022:
-    if: needs.config.outputs.only_job == 'build-windows-vs2022' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: windows-8-core
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - test_shard: db_test
-          suite_run: db_test
-          run_java: "false"
-        - test_shard: other
-          suite_run: arena_test,db_basic_test,db_test2,db_merge_operand_test,bloom_test,c_test,coding_test,crc32c_test,dynamic_bloom_test,env_basic_test,env_test,hash_test,random_test
-          run_java: "false"
-        - test_shard: java
-          suite_run: ""
-          run_java: "true"
-    env:
-      CMAKE_GENERATOR: Visual Studio 17 2022
-      CMAKE_PORTABLE: 1
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/windows-build-steps"
-      with:
-        suite-run: ${{ matrix.suite_run }}
-        run-java: ${{ matrix.run_java }}
-  # ============================ Java Jobs ============================ #
-  build-linux-java:
-    if: needs.config.outputs.only_job == 'build-linux-java' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: java
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-    - name: Test RocksDBJava
-      run: make V=1 J=8 -j8 jtest
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-  build-linux-java-static:
-    if: needs.config.outputs.only_job == 'build-linux-java-static' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: java-static
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-    - name: Build RocksDBJava Static Library
-      run: make V=1 J=8 -j8 rocksdbjavastatic
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-  build-macos-java:
-    if: needs.config.outputs.only_job == 'build-macos-java' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: macos-15-xlarge
-    env:
-      JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
-      ROCKSDB_DISABLE_JEMALLOC: 1
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: maxim-lobanov/setup-xcode@v1.6.0
-      with:
-        xcode-version: 16.4.0
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: macos-java
-    - uses: "./.github/actions/increase-max-open-files-on-macos"
-    - uses: "./.github/actions/install-gflags-on-macos"
-    - uses: "./.github/actions/install-jdk8-on-macos"
-    - uses: "./.github/actions/pre-steps-macos"
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-    - name: Test RocksDBJava
-      run: make V=1 J=16 -j16 jtest
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-macos-java-static:
-    if: needs.config.outputs.only_job == 'build-macos-java-static' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: macos-15-xlarge
-    env:
-      JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: maxim-lobanov/setup-xcode@v1.6.0
-      with:
-        xcode-version: 16.4.0
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: macos-java-static
-    - uses: "./.github/actions/increase-max-open-files-on-macos"
-    - uses: "./.github/actions/install-gflags-on-macos"
-    - uses: "./.github/actions/install-jdk8-on-macos"
-    - uses: "./.github/actions/pre-steps-macos"
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-    - name: Build RocksDBJava x86 and ARM Static Libraries
-      run: make V=1 J=16 -j16 rocksdbjavastaticosx
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-macos-java-static-universal:
-    if: needs.config.outputs.only_job == 'build-macos-java-static-universal' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on: macos-15-xlarge
-    env:
-      JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: maxim-lobanov/setup-xcode@v1.6.0
-      with:
-        xcode-version: 16.4.0
-    - uses: "./.github/actions/setup-ccache"
-      with:
-        cache-key-prefix: macos-java-static-universal
-    - uses: "./.github/actions/increase-max-open-files-on-macos"
-    - uses: "./.github/actions/install-gflags-on-macos"
-    - uses: "./.github/actions/install-jdk8-on-macos"
-    - uses: "./.github/actions/pre-steps-macos"
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-    - name: Build RocksDBJava Universal Binary Static Library
-      run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub
-    - uses: "./.github/actions/teardown-ccache"
-      if: always()
-    - uses: "./.github/actions/post-steps"
-  build-linux-java-pmd:
-    if: needs.config.outputs.only_job == 'build-linux-java-pmd' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu
-    container:
-      image: evolvedbinary/rocksjava:alpine3_x64-be
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/install-maven"
-    - uses: "./.github/actions/pre-steps"
-    - name: Set Java Environment
-      run: |-
-        echo "JAVA_HOME=${JAVA_HOME}"
-        which java && java -version
-        which javac && javac -version
-    - name: PMD RocksDBJava
-      run: make V=1 J=8 -j8 jpmd
-    - uses: actions/upload-artifact@v4.0.0
-      with:
-        name: pmd-report
-        path: "${{ github.workspace }}/java/target/pmd.xml"
-    - uses: actions/upload-artifact@v4.0.0
-      with:
-        name: maven-site
-        path: "${{ github.workspace }}/java/target/site"
-  build-linux-arm:
-    if: needs.config.outputs.only_job == 'build-linux-arm' || (needs.config.outputs.only_job == '' && github.repository_owner == 'facebook')
-    needs: config
-    runs-on:
-      labels: 4-core-ubuntu-arm
-    steps:
-      - uses: actions/checkout@v4.1.0
-      - uses: "./.github/actions/pre-steps"
-      - run: sudo apt-get update && sudo apt-get install -y build-essential ccache
-      - uses: "./.github/actions/setup-ccache"
-        with:
-          cache-key-prefix: arm
-      - run: ROCKSDBTESTS_PLATFORM_DEPENDENT=only make V=1 J=4 -j4 all_but_some_tests check_some
-      - name: Print ccache stats
-        run: ccache -s
-        if: always()
-        shell: bash
-      - uses: "./.github/actions/post-steps"

.github/workflows/weekly.yml

@@ -1,20 +0,0 @@
-name: facebook/rocksdb/weekly
-on:
-  schedule:
-  - cron: 0 9 * * 0
-  workflow_dispatch:
-permissions: {}
-jobs:
-  build-linux-valgrind:
-    if: ${{ github.repository_owner == 'facebook' }}
-    runs-on:
-      labels: 16-core-ubuntu
-    timeout-minutes: 840
-    container:
-      image: ghcr.io/facebook/rocksdb_ubuntu:22.1
-      options: --shm-size=16gb
-    steps:
-    - uses: actions/checkout@v4.1.0
-    - uses: "./.github/actions/pre-steps"
-    - run: make V=1 -j20 valgrind_test
-    - uses: "./.github/actions/post-steps"

.gitignore

@@ -1,5 +1,4 @@
 make_config.mk
-rocksdb.pc
 
 *.a
 *.arc
@@ -8,7 +7,6 @@ rocksdb.pc
 *.gcda
 *.gcno
 *.o
-*.o.tmp
 *.so
 *.so.*
 *_test
@@ -26,7 +24,6 @@ rocksdb.pc
 *.vcxproj.filters
 *.sln
 *.cmake
-.watchmanconfig
 CMakeCache.txt
 CMakeFiles/
 build/
@@ -35,8 +32,6 @@ ldb
 manifest_dump
 sst_dump
 blob_dump
-block_cache_trace_analyzer
-tools/block_cache_analyzer/*.pyc
 column_aware_encoding_exp
 util/build_version.cc
 build_tools/VALGRIND_LOGS/
@@ -47,19 +42,9 @@ package/
 unity.a
 tags
 etags
-GPATH
-GRTAGS
-GTAGS
 rocksdb_dump
 rocksdb_undump
 db_test2
-trace_analyzer
-block_cache_trace_analyzer
-io_tracer_parser
-.DS_Store
-.vs
-.vscode
-.clangd
 
 java/out
 java/target
@@ -87,26 +72,3 @@ tp2/
 fbcode/
 fbcode
 buckifier/*.pyc
-buckifier/__pycache__
-.arcconfig
-
-compile_commands.json
-meta_gen_cpp_compile_commands.json
-clang-format-diff.py
-.py3/
-
-fuzz/proto/gen/
-fuzz/crash-*
-
-cmake-build-*
-third-party/folly/
-.cache
-*.sublime-*
-
-# Claude Code local settings
-.claude/settings.local.json
-
-tools/__pycache__/
-# Keep documentation trackable even if broader ignores match names like "*_test".
-!docs/
-!docs/**

.lgtm.yml

@@ -1,4 +0,0 @@
-extraction:
-  cpp:
-    index:
-      build_command: make static_lib

.travis.yml

@@ -0,0 +1,84 @@
+sudo: false
+dist: trusty
+language: cpp
+os:
+  - linux
+  - osx
+compiler:
+  - clang
+  - gcc
+osx_image: xcode8.3
+jdk:
+  - oraclejdk7
+cache:
+  - ccache
+  - apt
+
+addons:
+   apt:
+      packages: ['zlib1g-dev', 'libbz2-dev', 'libsnappy-dev', 'curl', 'libgflags-dev', 'mingw-w64']
+env:
+  - TEST_GROUP=platform_dependent # 16-18 minutes
+  - TEST_GROUP=1 # 33-35 minutes
+  - TEST_GROUP=2 # 18-20 minutes
+  - TEST_GROUP=3 # 20-22 minutes
+  - TEST_GROUP=4 # 12-14 minutes
+  # Run java tests
+  - JOB_NAME=java_test # 4-11 minutes
+  # Build ROCKSDB_LITE
+  - JOB_NAME=lite_build # 3-4 minutes
+  # Build examples
+  - JOB_NAME=examples # 5-7 minutes
+  - JOB_NAME=cmake # 3-5 minutes
+  - JOB_NAME=cmake-mingw # 3 minutes
+
+matrix:
+  exclude:
+  - os: osx
+    env: TEST_GROUP=1
+  - os: osx
+    env: TEST_GROUP=2
+  - os: osx
+    env: TEST_GROUP=3
+  - os: osx
+    env: TEST_GROUP=4
+  - os : osx
+    env: JOB_NAME=cmake-mingw
+  - os : linux
+    compiler: clang
+  - os : osx
+    compiler: gcc
+
+# https://docs.travis-ci.com/user/caching/#ccache-cache
+install:
+  - if [ "${TRAVIS_OS_NAME}" == osx ]; then
+      brew install ccache;
+      PATH=$PATH:/usr/local/opt/ccache/libexec;
+    fi
+  - if [[ "${JOB_NAME}" == cmake* ]] && [ "${TRAVIS_OS_NAME}" == linux ]; then
+      mkdir cmake-dist && curl -sfSL https://cmake.org/files/v3.8/cmake-3.8.1-Linux-x86_64.tar.gz | tar --strip-components=1 -C cmake-dist -xz && export PATH=$PWD/cmake-dist/bin:$PATH;
+    fi
+
+before_script:
+  # Increase the maximum number of open file descriptors, since some tests use
+  # more FDs than the default limit.
+  - ulimit -n 8192
+
+script:
+  - ${CXX} --version
+  - if [ `command -v ccache` ]; then ccache -C; fi
+  - if [ "${TEST_GROUP}" == 'platform_dependent' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_END=db_block_cache_test make -j4 all_but_some_tests check_some; fi
+  - if [ "${TEST_GROUP}" == '1' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=db_block_cache_test ROCKSDBTESTS_END=full_filter_block_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '2' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=full_filter_block_test ROCKSDBTESTS_END=write_batch_with_index_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '3' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_batch_with_index_test ROCKSDBTESTS_END=write_prepared_transaction_test make -j4 check_some; fi
+  - if [ "${TEST_GROUP}" == '4' ]; then OPT=-DTRAVIS V=1 ROCKSDBTESTS_START=write_prepared_transaction_test make -j4 check_some; fi
+  - if [ "${JOB_NAME}" == 'java_test' ]; then OPT=-DTRAVIS V=1 make clean jclean && make rocksdbjava jtest; fi
+  - if [ "${JOB_NAME}" == 'lite_build' ]; then OPT="-DTRAVIS -DROCKSDB_LITE" V=1 make -j4 static_lib tools; fi
+  - if [ "${JOB_NAME}" == 'examples' ]; then OPT=-DTRAVIS V=1 make -j4 static_lib; cd examples; make -j4; fi
+  - if [ "${JOB_NAME}" == 'cmake' ]; then mkdir build && cd build && cmake -DJNI=1 .. && make -j4 rocksdb rocksdbjni; fi
+  - if [ "${JOB_NAME}" == 'cmake-mingw' ]; then mkdir build && cd build && cmake -DJNI=1 .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni; fi
+notifications:
+    email:
+      - leveldb@fb.com
+    webhooks:
+      - https://buildtimetrend.herokuapp.com/travis

.watchmanconfig

@@ -1,6 +0,0 @@
-{
-  "content_hash_warming": true,
-  "content_hash_max_items": 333333,
-  "hint_num_files_per_dir": 8,
-  "fsevents_latency": 0.05
-}

AGENTS.md

@@ -1,9 +0,0 @@
-# Agent Instructions
-
-This repository's authoritative agent instructions live in `CLAUDE.md`.
-
-Read and follow [`CLAUDE.md`](./CLAUDE.md) in full before making changes or
-reviewing code in this checkout.
-
-If there is any ambiguity between this file and `CLAUDE.md`, `CLAUDE.md` takes
-precedence.

CLAUDE.md

@@ -1,366 +0,0 @@
-# RocksDB Code Generation and Review Guidance
-
-This document provides guidance for generating and reviewing code in the RocksDB project, derived from analysis of code review feedback across hundreds of complex merged Pull Requests. Use this as a reference when writing code with AI assistants or conducting code reviews.
-
----
-
-## General Best Practices
-
-### Code Quality and Maintainability
-
-**Clarity and Readability:** Write clear, self-documenting code. Use meaningful variable names, add comments for complex logic, and structure code to minimize cognitive load. Avoid clever tricks that sacrifice readability for marginal performance gains unless absolutely necessary.
-
-**Consistent Style:** Follow existing code style conventions. RocksDB uses `.clang-format` for formatting, specific naming conventions, and structural patterns. Deviations from these patterns are frequently flagged in reviews.
-
-**Error Handling:** Ensure robust error handling throughout the codebase. Use RocksDB's `Status` type consistently, propagate errors appropriately, and avoid silently ignoring failures. Reviewers pay close attention to edge cases and failure modes.
-
-### Testing Philosophy
-
-**Comprehensive Coverage:** Every change should include appropriate test coverage. This includes unit tests for isolated functionality, integration tests for component interactions, and stress tests for concurrency and performance validation. Reviewers will ask for additional tests if coverage is insufficient.
-
-**Edge Cases and Failure Modes:** Tests should explicitly cover edge cases, boundary conditions, and potential failure scenarios. This is especially important for changes affecting core database operations, compaction, or recovery logic.
-
-**Platform-Specific Testing:** RocksDB supports multiple platforms (Linux, Windows, macOS) and compilers (GCC, Clang, MSVC). Changes should be tested across relevant platforms, particularly when touching platform-specific code or using compiler-specific features.
-
-### Performance Considerations
-
-**⚠️ PERFORMANCE IS CRITICAL:** RocksDB is a high-performance storage engine where every CPU cycle and memory access matters. When writing code, always evaluate from a performance perspective. This is not optional—performance-aware coding is a fundamental requirement for all contributions.
-
-**Benchmarking and Profiling:** Performance claims should be backed by empirical evidence. Use RocksDB's benchmarking tools (e.g., `db_bench`) to validate improvements. Reviewers will request benchmark results for changes that could impact performance.
-
-**Memory Allocation:** Minimize dynamic memory allocations, especially in hot paths. Prefer stack allocation over heap allocation. Reuse buffers when possible. Consider using arena allocators or memory pools for frequent small allocations. Every `new`, `malloc`, or container resize has a cost.
-
-**Memory Copy:** Avoid unnecessary memory copies. Use move semantics, `std::string_view`, `Slice`, and pass-by-reference where appropriate. Be aware of implicit copies in STL containers and function returns. Prefer in-place operations over copy-and-modify patterns.
-
-**CPU Cache Efficiency:** Design data structures and access patterns to be cache-friendly. Keep frequently accessed data together (data locality). Prefer sequential memory access over random access. Be mindful of cache line sizes (typically 64 bytes) and avoid false sharing in concurrent code. Consider struct packing and field ordering to improve cache utilization.
-
-**Loop Optimization:** Look for opportunities to collapse nested loops, reduce loop overhead, and minimize branch mispredictions. Hoist invariant computations out of loops. Consider loop unrolling for tight inner loops. Batch operations when possible to amortize per-operation overhead.
-
-**SIMD and Vectorization:** Leverage SIMD instructions (SSE, AVX) for data-parallel operations when appropriate. Structure data to enable auto-vectorization by the compiler. Consider explicit SIMD intrinsics for critical hot paths like checksum computation, encoding/decoding, and bulk data processing.
-
-**Branch Prediction:** Minimize unpredictable branches in hot paths. Use `LIKELY`/`UNLIKELY` macros to hint branch prediction. Consider branchless alternatives for simple conditionals. Order switch cases and if-else chains by frequency.
-
-**Memory and Resource Management:** Be mindful of memory allocations, especially in hot paths. Use RAII patterns, smart pointers, and RocksDB's memory management utilities appropriately.
-
-**Hot Path Analysis:** When deciding how aggressively to optimize code, consider whether it's on a hot path:
-- **Hot path** (executed thousands+ times, e.g., data access, iteration, compaction loops): Performance is paramount. Apply all optimization techniques—loop collapsing, SIMD, cache optimization, pre-allocation, etc. The cost of each operation is multiplied by execution frequency.
-- **Cold path** (executed rarely, e.g., DB open, configuration parsing, error handling): Maintainability and clarity are more important. Prefer readable code over micro-optimizations. Complex optimizations here add maintenance burden with negligible performance benefit.
-- **Warm path** (moderate frequency): Balance both concerns. Use profiling data to guide optimization decisions.
-
-**Avoid Premature Optimization:** While performance is critical, focus on correctness first, then optimize based on profiling data. However, be performance-aware from the start—choosing the right algorithm and data structure upfront is not premature optimization. Use the hot path analysis above to decide how much optimization effort is warranted.
-
-### API Design and Compatibility
-
-**Backwards Compatibility:** RocksDB maintains strong backwards compatibility guarantees. Breaking changes are rare and require extensive justification. When deprecating features, follow the project's deprecation policy (typically spanning multiple releases).
-
-**API Consistency:** New APIs should be consistent with existing patterns. Use similar naming conventions, parameter ordering, and return types. Reviewers will suggest changes to improve consistency with the broader codebase.
-
-**Documentation:** Public APIs must be thoroughly documented. Include usage examples, parameter descriptions, and notes on thread safety, performance characteristics, and compatibility considerations.
-
----
-
-## Component-Specific Guidance
-
-### Database Core (`db`)
-
-The database core handles write-ahead logging (WAL), memtables, compaction, and recovery. This component receives the most scrutiny in code reviews.
-
-**Concurrency and Thread Safety:** Database operations are highly concurrent. Reviewers carefully examine locking strategies, atomic operations, and memory ordering. Document synchronization assumptions clearly. Use appropriate memory ordering semantics (`acquire`/`release` vs. `seq_cst`).
-
-**Compaction Logic:** Changes to compaction are complex and high-risk. Ensure that compaction logic respects configured parameters, handles edge cases (empty databases, single-file compactions), and maintains correctness under concurrent operations.
-
-**Error Propagation:** Database operations can fail in many ways (I/O errors, corruption, resource exhaustion). Ensure that errors are properly propagated, logged, and handled. Avoid assertions in production code paths.
-
-**Testing:** Database core changes require extensive testing, including unit tests, integration tests, and stress tests. Test with various configurations, compaction styles, and concurrent workloads.
-
-### Public Headers (`include`)
-
-Public headers define RocksDB's API surface. Changes here have the highest compatibility impact.
-
-**API Design:** New APIs should be intuitive, consistent with existing patterns, and well-documented. Consider how the API will be used in practice and avoid adding unnecessary complexity.
-
-**Backwards Compatibility:** Breaking changes to public APIs require extensive justification and a deprecation plan. Maintain ABI compatibility for bug fixes and patch releases.
-
-**Documentation:** Every public API must be thoroughly documented with usage examples, parameter descriptions, and notes on thread safety and performance characteristics.
-
-**Deprecation:** When deprecating APIs, follow the project's policy. Mark deprecated APIs clearly, provide migration guidance, and maintain support for at least one major release.
-
-### Internal Utilities (`util`)
-
-Internal utilities provide common functionality used throughout the codebase.
-
-**Code Reuse:** Utilities should be general-purpose and reusable. Avoid duplicating functionality that already exists elsewhere in the codebase.
-
-**Error Handling:** Utility functions should handle errors robustly and propagate them appropriately. Consider edge cases like overflow, underflow, and invalid inputs.
-
-**Testing:** Utility functions should have comprehensive test coverage, including edge cases and failure modes. Consider adding death tests for assertions.
-
-**Performance:** Utilities are often used in hot paths. Ensure that implementations are efficient and avoid unnecessary allocations or copies.
-
-### Table Management (`table`)
-
-Table management handles SST file format, block-based tables, and table readers/writers.
-
-**Block Format and Checksums:** Changes to block format require extreme care. Ensure that checksums are computed and verified correctly. Test with various compression algorithms and block sizes.
-
-**Iterator Correctness:** Table iterators are used throughout the codebase. Ensure that iterator semantics (Seek, Next, Prev) are correct, especially at boundaries and with deletions.
-
-**Caching and Prefetching:** Table readers interact with the block cache and prefetching logic. Ensure that cache keys are unique and that prefetching respects configured limits.
-
-**Performance:** Table operations are performance-critical. Benchmark changes that could impact read or write performance.
-
-### Utilities (`utilities`)
-
-Utilities include optional features like transactions, backup engine, and checkpoint.
-
-**Feature Isolation:** Utilities should be self-contained and not introduce unnecessary dependencies on core database internals.
-
-**Deprecation and Cleanup:** Legacy features are being phased out. When removing deprecated code, ensure that migration paths are documented and that users have sufficient warning.
-
-**Cross-Platform Compatibility:** Utilities often interact with OS-specific APIs. Ensure that code works on all supported platforms.
-
-### Options and Configuration (`options`)
-
-Options define RocksDB's configuration system.
-
-**Type Safety:** Use appropriate types for options (e.g., `uint32_t` for flags, scoped enums for enumerated values).
-
-**Deprecation Policy:** When deprecating options, follow the project's policy. Document the deprecation, provide migration guidance, and maintain support for at least one major release.
-
-**Dynamic Configuration:** Some options can be changed dynamically. Ensure that dynamic changes are thread-safe and take effect correctly.
-
-**Validation:** Validate option values and provide clear error messages for invalid configurations.
-
-### Cache (`cache`)
-
-Cache management is critical for RocksDB's performance.
-
-**Concurrency:** Cache operations are highly concurrent. Ensure that implementations are thread-safe and use appropriate synchronization primitives.
-
-**Performance:** Cache operations are in the hot path. Optimize for low latency and high throughput. Benchmark changes carefully.
-
-**Memory Management:** Cache implementations must manage memory carefully to avoid leaks and excessive allocations.
-
-**Eviction Policies:** Changes to eviction policies should be well-tested and benchmarked to ensure they improve overall performance.
-
----
-
-## Code Review Checklist
-
-When reviewing RocksDB code (or preparing code for review), use this checklist:
-
-### Correctness
-- [ ] Does the change preserve database semantics (e.g., snapshot isolation, key ordering)?
-- [ ] Are all error cases handled appropriately?
-- [ ] Is the change thread-safe? Are synchronization primitives used correctly?
-- [ ] Are there any potential data races or deadlocks?
-
-### Testing
-- [ ] Does the change include appropriate test coverage?
-- [ ] Are edge cases and failure modes tested?
-- [ ] Have the tests been run on all supported platforms?
-- [ ] Are stress tests passing?
-
-### Performance
-- [ ] Are there benchmark results for performance-sensitive changes?
-- [ ] Does the change avoid unnecessary allocations or copies?
-- [ ] Are hot paths optimized appropriately?
-
-### API and Compatibility
-- [ ] Is the change backwards compatible?
-- [ ] Are new APIs consistent with existing patterns?
-- [ ] Is the public API documented?
-- [ ] Are deprecated features handled according to policy?
-
-### Code Quality
-- [ ] Does the code follow RocksDB's style conventions?
-- [ ] Is the code clear and maintainable?
-- [ ] Are comments and documentation sufficient?
-- [ ] Are there any code smells or anti-patterns?
-
----
-
-## Common Review Feedback Patterns
-
-The following patterns emerged as frequent sources of review feedback:
-
-1. **Test Coverage:** Reviewers frequently request additional tests for edge cases, platform-specific behavior, and failure modes. Complex changes require comprehensive test coverage including unit tests, integration tests, and stress tests.
-
-2. **Error Handling:** Ensure proper error propagation using RocksDB's `Status` type. Avoid silent failures and provide clear error messages that include context about what failed and why.
-
-3. **API Design:** New APIs should be consistent with existing patterns. Use descriptive names that follow established conventions. Avoid breaking changes without strong justification and a clear deprecation plan.
-
-4. **Documentation:** Public APIs must be documented with usage examples and notes on thread safety, performance characteristics, and compatibility considerations. Complex internal logic should also be well-commented.
-
-5. **Performance:** Performance-sensitive changes require benchmark results to validate improvements. Use `db_bench` and other profiling tools to measure impact. Avoid premature optimization that adds complexity without measurable benefit.
-
-6. **Concurrency:** Thread safety is critical in RocksDB. Document synchronization assumptions clearly. Use appropriate memory ordering semantics. Consider potential race conditions and deadlocks.
-
-7. **Code Style:** Follow existing conventions for naming, formatting, and structure. Use `.clang-format` for consistent formatting. Prefer scoped enums (`enum class`) over unscoped enums.
-
-8. **Backwards Compatibility:** RocksDB maintains strong compatibility guarantees. Breaking changes require extensive justification. When deprecating features, provide migration guidance and maintain support across multiple releases.
-
-9. **Refactoring:** Reviewers appreciate refactoring that improves code readability and maintainability. Look for opportunities to deduplicate code and simplify complex logic.
-
-10. **Platform Compatibility:** Ensure changes work correctly on all supported platforms (Linux, Windows, macOS) and with all supported compilers (GCC, Clang, MSVC).
-
----
-
-## Important tips
-
-### Build system
-* There are 3 build system. Make, CMake, BUCK(meta internal).
-* When a new .cc file is added, update Makefile, CMakeLists.txt, src.mk, BUCK.
-* Don't manually edit BUCK file, after updating src.mk, run
-    /usr/local/bin/python3 buckifier/buckify_rocksdb.py to update it
-* Use make to build and run the test. CMake and BUCK are not used locally.
-* Use `make dbg` command to build all of the unit test in debug mode.
-* For -j in make command, use the number of CPU cores to decide it.
-
-### When to run `make clean` (avoid mixing build modes)
-
-The Makefile does **not** track build mode, so object files from a prior
-build are silently reused even when compiled with different flags, leading
-to confusing linker errors, sanitizer false negatives, ODR violations, or
-"phantom" bugs.
-
-Run `make clean` before switching any of these:
-* **`ASSERT_STATUS_CHECKED=1` ↔ unset** — changes the `Status` class layout (ABI break).
-* **Sanitizer builds** — toggling any of `COMPILE_WITH_ASAN=1`,
-    `COMPILE_WITH_UBSAN=1`, `COMPILE_WITH_TSAN=1` on/off.
-* `DEBUG_LEVEL=0` (release) ↔ `DEBUG_LEVEL=1` (debug, default for `make dbg`).
-* Different compilers, `OPT` levels, or other flags affecting codegen/ABI.
-
-**Notable exception:** `DEBUG_LEVEL=2` can be safely mixed with
-`DEBUG_LEVEL=1` — rebuild a subset of files with `DEBUG_LEVEL=2` to get
-extra/more accurate runtime checks for those files without a full clean.
-
-When in doubt, `make clean` is cheap insurance compared to chasing a
-phantom bug.
-
-### Source checks
-* Run `make check-sources` before committing. This catches non-ASCII
-    characters in source files and other source-level issues that CI will
-    reject. In particular, **do not use Unicode characters** (em dashes,
-    smart quotes, etc.) in comments or strings -- use ASCII equivalents
-    (`--` instead of em dash, `'` instead of smart quote, etc.).
-
-### RTTI and dynamic_cast
-* Production code and `db_stress` must build in **release mode
-    (`-fno-rtti`)**. Do not use `dynamic_cast` anywhere except unit tests.
-    Use `static_cast_with_check` from `util/cast_util.h` (validates with
-    `dynamic_cast` in debug builds, plain `static_cast` in release).
-* Unit tests (`*_test.cc`) are built in debug mode with RTTI enabled.
-
-### Unit Test
-* After all of the unit tests are added, review them and try to extract common
-    reusable utility functions to reduce code duplication due to copy past between
-    unit tests. This should be done every time unit test is updated.
-* Don't use sleep to wait for certain events to happen. This will cause test to
-    be flaky. Instead, use sync point to synchronize thread progress.
-* Cap unit test execution with 60 seconds timeout.
-* When there are multiple unit tests need to be executed, try to use
-    gtest_parallel.py if available. E.g.
-    python3 ${GTEST_PARALLEL}/gtest_parallel.py ./table_test
-* After writing a test, stress-test for flakiness:
-    ```bash
-    COERCE_CONTEXT_SWITCH=1 make {test_binary}
-    ./{test_binary} --gtest_filter="*YourTestName*" --gtest_repeat=5
-    ```
-
-### Unit test dedup guidelines
-* Extract helper functions for repeated patterns such as object
-    construction, round-trip (encode → decode → verify), and common
-    assertion sequences.
-* Use table-driven tests (struct array + loop) when multiple test cases
-    share the same logic but differ only in input/expected data.
-* Prefer randomized tests over exhaustive parameter permutations. Use
-    `Random` from `util/random.h` (not `std::mt19937`). Use a time-based
-    seed with `SCOPED_TRACE("seed=" + std::to_string(seed))` so failures
-    are reproducible.
-* Keep deterministic edge-case tests separate from randomized tests
-    (error paths, boundary conditions, format verification).
-* Methods only used in tests should be private with `friend class` +
-    `TEST_F` fixture wrappers. In wrappers, always fully qualify the
-    target method to avoid infinite recursion.
-
-### Adding new public API
-    Refer to claude_md/add_public_api.md
-
-### Adding new option
-    Refer to claude_md/add_option.md
-
-### Removing deprecated option
-    Refer to claude_md/remove_option.md
-
-### Metrics
-* When adding a new feature, evaluate whether there is opportunity to add
-    metrics. Try to avoid causing performance regression on hot path when adding
-    metrics.
-
-### Stress test
-* When adding a new feature, make sure stress test covers the new option.
-
-### Component docs
-* For component-level design notes and implementation walkthroughs, start with
-    `docs/components/index.md`.
-* Documentation under `docs/components/` is organized by subsystem in
-    `docs/components/<area>/`.
-* Each subsystem directory should have an `index.md` entry point plus focused
-    chapter files for deeper topics.
-
-### DB bench update
-* When adding a performance related feature, support it in db_bench
-
-### Adding release note
-* Release note should be kept short at high level for external user consumption.
-
-### Blog posts (docs/_posts)
-* Blog post authors must be defined in `docs/_data/authors.yml` to be displayed
-
-### Final verification of the change
-* Execute make clean to clean all of the changes.
-* Execute make check to build all of the changes and execute all of the tests.
-    Note that executing all of the tests could take multiple minutes.
-* Run `ASSERT_STATUS_CHECKED=1 make check` to verify all Status objects are
-    properly checked. This catches missing error handling that can lead to
-    silent data corruption.
-
-### Monitoring make check progress
-* Use `make check-progress` to get machine-parseable JSON progress while
-    `make check` is running. This is useful for Claude Code to monitor long
-    builds without timeout issues.
-* Run `make check` in background, then poll progress:
-    ```bash
-    make check &
-    # Poll periodically:
-    make check-progress
-    ```
-* The output shows current phase and progress:
-    ```json
-    {"status":"running","phase":"compiling","completed":300,"total":919,...}
-    {"status":"running","phase":"testing","completed":1500,"total":29962,"failed":0,"percent":5,...}
-    {"status":"completed","phase":"testing","completed":29962,"total":29962,"failed":0,"percent":100,...}
-    ```
-* Phases: `compiling` -> `linking` -> `generating` -> `testing` -> `completed`
-* Key fields: `status`, `phase`, `completed`, `total`, `failed`, `percent`
-* When tests fail, `failed_tests` array shows details (up to 10 failures):
-    ```json
-    {"status":"running",...,"failed":3,"failed_tests":[
-      {"test":"cache_test-CacheTest.Usage","exit_code":1,"signal":0,"output":"...test log..."},
-      {"test":"env_test-EnvTest.Open","exit_code":0,"signal":11,"output":"...Segmentation fault..."}
-    ]}
-    ```
-* `exit_code`: non-zero means test assertion failed
-* `signal`: non-zero means test was killed (e.g., 9=SIGKILL, 6=SIGABRT, 11=SIGSEGV)
-* `output`: last 50 lines of test log including error messages and stack traces
-
-### Executing benchmark using db_bench
-* Since the goal is to measure performance, we need to build a release binary
-    using `make clean && DEBUG_LEVEL=0 make db_bench`. If there is an engine
-    crash due to bug, we need to switch back to debug build. Make sure to run
-    `make clean` before running `make dbg`.
-
-### Formatting code
-* After making change, use `make format-auto` to auto-apply formatting without
-    interactive prompts (Claude Code friendly).

CODE_OF_CONDUCT.md

@@ -1,77 +1,3 @@
 # Code of Conduct
 
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to make participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
-  advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
-  address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies within all project spaces, and it also applies when
-an individual is representing the project or its community in public spaces.
-Examples of representing a project or community include using an official
-project e-mail address, posting via an official social media account, or acting
-as an appointed representative at an online or offline event. Representation of
-a project may be further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at <opensource-conduct@fb.com>. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq
-
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.facebook.com/codeofconduct) so that you can understand what actions will and will not be tolerated.

DEFAULT_OPTIONS_HISTORY.md

@@ -1,4 +1,4 @@
-# RocksDB default options change log (NO LONGER MAINTAINED)
+# RocksDB default options change log
 ## Unreleased
 * delayed_write_rate takes the rate given by rate_limiter if not specified.
 

Directory.Build.props

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<Project>
-    <PropertyGroup>
-        <CLToolExe>ccache_msvc_compiler.bat</CLToolExe>
-        <CLToolPath>$(MSBuildThisFileDirectory)</CLToolPath>
-        <UseMultiToolTask>true</UseMultiToolTask>
-        <EnforceProcessCountAcrossBuilds>true</EnforceProcessCountAcrossBuilds>
-    </PropertyGroup>
-</Project>

INSTALL.md

@@ -6,7 +6,7 @@ than release mode.
 
 RocksDB's library should be able to compile without any dependency installed,
 although we recommend installing some compression libraries (see below).
-We do depend on newer gcc/clang with C++20 support (GCC >= 11, Clang >= 10).
+We do depend on newer gcc/clang with C++11 support.
 
 There are few options when compiling RocksDB:
 
@@ -17,18 +17,15 @@ There are few options when compiling RocksDB:
 * `make check` will compile and run all the unit tests. `make check` will compile RocksDB in debug mode.
 
 * `make all` will compile our static library, and all our tools and unit tests. Our tools
-depend on gflags 2.2.0 or newer. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't
+depend on gflags. You will need to have gflags installed to run `make all`. This will compile RocksDB in debug mode. Don't
 use binaries compiled by `make all` in production.
 
-* By default the binary we produce is optimized for the CPU you're compiling on
-(`-march=native` or the equivalent). To build a binary compatible with the most
-general architecture supported by your CPU and compiler, set `PORTABLE=1` for
-the build, but performance will suffer as many operations benefit from newer
-and wider instructions. In addition to `PORTABLE=0` (default) and `PORTABLE=1`,
-it can be set to an architecture name recognized by your compiler. For example,
-on 64-bit x86, a reasonable compromise is `PORTABLE=haswell` which supports
-many or most of the available optimizations while still being compatible with
-most processors made since roughly 2013.
+* By default the binary we produce is optimized for the platform you're compiling on
+(`-march=native` or the equivalent). SSE4.2 will thus be enabled automatically if your
+CPU supports it. To print a warning if your CPU does not support SSE4.2, build with
+`USE_SSE=1 make static_lib` or, if using CMake, `cmake -DFORCE_SSE42=ON`. If you want
+to build a portable binary, add `PORTABLE=1` before your make commands, like this:
+`PORTABLE=1 make static_lib`.
 
 ## Dependencies
 
@@ -46,21 +43,10 @@ most processors made since roughly 2013.
       command line flags processing. You can compile rocksdb library even
       if you don't have gflags installed.
 
-* `make check` will also check code formatting, which requires [clang-format](https://clang.llvm.org/docs/ClangFormat.html)
-
-* If you wish to build the RocksJava static target, then cmake is required for building Snappy.
-
-* If you wish to run microbench (e.g, `make microbench`, `make ribbon_bench` or `cmake -DWITH_BENCHMARK=1`), Google benchmark >= 1.6.0 is needed.
-* You can do the following to install Google benchmark. These commands are copied from `./build_tools/ubuntu20_image/Dockerfile`:
-
-`$ git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark`
-
-`$ cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install`
-
 ## Supported platforms
 
 * **Linux - Ubuntu**
-    * Upgrade your gcc to version at least 11 to get C++20 support.
+    * Upgrade your gcc to version at least 4.8 to get C++11 support.
     * Install gflags. First, try: `sudo apt-get install libgflags-dev`
       If this doesn't work and you're using Ubuntu, here's a nice tutorial:
       (http://askubuntu.com/questions/312173/installing-gflags-12-04)
@@ -72,12 +58,13 @@ most processors made since roughly 2013.
     * Install zstandard: `sudo apt-get install libzstd-dev`.
 
 * **Linux - CentOS / RHEL**
-    * Upgrade your gcc to version at least 11 to get C++20 support
+    * Upgrade your gcc to version at least 4.8 to get C++11 support:
+      `yum install gcc48-c++`
     * Install gflags:
 
               git clone https://github.com/gflags/gflags.git
               cd gflags
-              git checkout v2.2.0
+              git checkout v2.0
               ./configure && make && sudo make install
 
       **Notice**: Once installed, please add the include path for gflags to your `CPATH` environment variable and the
@@ -105,27 +92,19 @@ most processors made since roughly 2013.
               sudo yum install libasan
 
     * Install zstandard:
-        * With [EPEL](https://fedoraproject.org/wiki/EPEL):
 
-              sudo yum install libzstd-devel
-
-        * With CentOS 8:
-
-              sudo dnf install libzstd-devel
-
-        * From source:
-
-              wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
-              mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
-              tar zxvf zstd-1.1.3.tar.gz
-              cd zstd-1.1.3
-              make && sudo make install
+             wget https://github.com/facebook/zstd/archive/v1.1.3.tar.gz
+             mv v1.1.3.tar.gz zstd-1.1.3.tar.gz
+             tar zxvf zstd-1.1.3.tar.gz
+             cd zstd-1.1.3
+             make && sudo make install
 
 * **OS X**:
-    * Install latest C++ compiler that supports C++20:
+    * Install latest C++ compiler that supports C++ 11:
         * Update XCode:  run `xcode-select --install` (or install it from XCode App's settting).
         * Install via [homebrew](http://brew.sh/).
             * If you're first time developer in MacOS, you still need to run: `xcode-select --install` in your command line.
+            * run `brew tap homebrew/versions; brew install gcc48 --use-llvm` to install gcc 4.8 (or higher).
     * run `brew install rocksdb`
 
 * **FreeBSD** (11.01):
@@ -168,39 +147,35 @@ most processors made since roughly 2013.
 
     * Install the dependencies for RocksDB:
 
-      `pkg_add gmake gflags snappy bzip2 lz4 zstd git bash findutils gnuwatch`
+        pkg_add gmake gflags snappy bzip2 lz4 zstd git jdk bash findutils gnuwatch 
 
     * Build RocksDB from source:
 
-        ```bash
         cd ~
         git clone https://github.com/facebook/rocksdb.git
         cd rocksdb
         gmake static_lib
-        ```
 
     * Build RocksJava from source (optional):
-        * In OpenBSD, JDK depends on XWindows system, so please check that you installed OpenBSD with `xbase` package.
-        * Install dependencies : `pkg_add -v jdk%1.8`
-        ```bash
+
         cd rocksdb
         export JAVA_HOME=/usr/local/jdk-1.8.0
         export PATH=$PATH:/usr/local/jdk-1.8.0/bin
-        gmake rocksdbjava SHA256_CMD='sha256 -q'
-        ```
+        gmake rocksdbjava
 
 * **iOS**:
-  * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define an important pre-processing macros: `IOS_CROSS_COMPILE`.
+  * Run: `TARGET_OS=IOS make static_lib`. When building the project which uses rocksdb iOS library, make sure to define two important pre-processing macros: `ROCKSDB_LITE` and `IOS_CROSS_COMPILE`.
 
-* **Windows** (Visual Studio 2017 to up):
+* **Windows**:
+  * For building with MS Visual Studio 13 you will need Update 4 installed.
   * Read and follow the instructions at CMakeLists.txt
-  * Or install via [vcpkg](https://github.com/microsoft/vcpkg)
+  * Or install via [vcpkg](https://github.com/microsoft/vcpkg) 
        * run `vcpkg install rocksdb:x64-windows`
 
 * **AIX 6.1**
     * Install AIX Toolbox rpms with gcc
     * Use these environment variables:
-
+  
              export PORTABLE=1
              export CC=gcc
              export AR="ar -X64"
@@ -211,9 +186,9 @@ most processors made since roughly 2013.
              export LIBPATH=/opt/freeware/lib
              export JAVA_HOME=/usr/java8_64
              export PATH=/opt/freeware/bin:$PATH
-
+  
 * **Solaris Sparc**
-    * Install GCC 11 and higher.
+    * Install GCC 4.8.2 and higher.
     * Use these environment variables:
 
              export CC=gcc
@@ -222,3 +197,4 @@ most processors made since roughly 2013.
              export EXTRA_LDFLAGS=-m64
              export PORTABLE=1
              export PLATFORM_LDFLAGS="-static-libstdc++ -static-libgcc"
+

LANGUAGE-BINDINGS.md

@@ -1,27 +1,18 @@
 This is the list of all known third-party language bindings for RocksDB. If something is missing, please open a pull request to add it.
 
-* Java - https://github.com/facebook/rocksdb/tree/main/java
-* Python
-    * https://github.com/rocksdict/RocksDict
-    * http://python-rocksdb.readthedocs.io/en/latest/ (unmaintained) 
-    * http://pyrocksdb.readthedocs.org/en/latest/ (unmaintained)
+* Java - https://github.com/facebook/rocksdb/tree/master/java
+* Python - http://pyrocksdb.readthedocs.org/en/latest/
 * Perl - https://metacpan.org/pod/RocksDB
 * Node.js - https://npmjs.org/package/rocksdb
-* Go 
-  * https://github.com/linxGnu/grocksdb
-  * https://github.com/tecbot/gorocksdb (unmaintained)
+* Go - https://github.com/tecbot/gorocksdb
 * Ruby - http://rubygems.org/gems/rocksdb-ruby
 * Haskell - https://hackage.haskell.org/package/rocksdb-haskell
 * PHP - https://github.com/Photonios/rocksdb-php
-* C#
-    * https://github.com/warrenfalk/rocksdb-sharp
-    * https://github.com/curiosity-ai/rocksdb-sharp
+* C# - https://github.com/warrenfalk/rocksdb-sharp
 * Rust
-    * https://github.com/pingcap/rust-rocksdb (used in production fork of https://github.com/spacejam/rust-rocksdb)
     * https://github.com/spacejam/rust-rocksdb
     * https://github.com/bh1xuw/rust-rocks
 * D programming language - https://github.com/b1naryth1ef/rocksdb
 * Erlang - https://gitlab.com/barrel-db/erlang-rocksdb
 * Elixir - https://github.com/urbint/rox
 * Nim - https://github.com/status-im/nim-rocksdb
-* Swift and Objective-C (iOS/OSX) - https://github.com/iabudiab/ObjectiveRocks 

PLUGINS.md

@@ -1,9 +0,0 @@
-This is the list of all known third-party plugins for RocksDB. If something is missing, please open a pull request to add it.
-
-* [Dedupfs](https://github.com/ajkr/dedupfs): an example for plugin developers to reference
-* [HDFS](https://github.com/riversand963/rocksdb-hdfs-env): an Env used for interacting with HDFS. Migrated from main RocksDB repo
-* [ZenFS](https://github.com/westerndigitalcorporation/zenfs): a file system for zoned block devices
-* [RADOS](https://github.com/riversand963/rocksdb-rados-env): an Env used for interacting with RADOS. Migrated from RocksDB main repo.
-* [PMEM](https://github.com/pmem/pmem-rocksdb-plugin): a collection of plugins to enable Persistent Memory on RocksDB.
-* [IPPCP](https://github.com/intel/ippcp-plugin-rocksdb): a plugin to enable encryption on RocksDB based on Intel optimized open source IPP-Crypto library.
-* [encfs](https://github.com/pegasus-kv/encfs): a plugin to enable encryption on RocksDB based on OpenSSL library.

README.md

@@ -1,20 +1,23 @@
 ## RocksDB: A Persistent Key-Value Store for Flash and RAM Storage
 
-[![CircleCI Status](https://circleci.com/gh/facebook/rocksdb.svg?style=svg)](https://circleci.com/gh/facebook/rocksdb)
+[![Linux/Mac Build Status](https://travis-ci.org/facebook/rocksdb.svg?branch=master)](https://travis-ci.org/facebook/rocksdb)
+[![Windows Build status](https://ci.appveyor.com/api/projects/status/fbgfu0so3afcno78/branch/master?svg=true)](https://ci.appveyor.com/project/Facebook/rocksdb/branch/master)
+[![PPC64le Build Status](http://140.211.168.68:8080/buildStatus/icon?job=Rocksdb)](http://140.211.168.68:8080/job/Rocksdb)
+
 
 RocksDB is developed and maintained by Facebook Database Engineering Team.
-It is built on earlier work on [LevelDB](https://github.com/google/leveldb) by Sanjay Ghemawat (sanjay@google.com)
+It is built on earlier work on LevelDB by Sanjay Ghemawat (sanjay@google.com)
 and Jeff Dean (jeff@google.com)
 
 This code is a library that forms the core building block for a fast
-key-value server, especially suited for storing data on flash drives.
+key value server, especially suited for storing data on flash drives.
 It has a Log-Structured-Merge-Database (LSM) design with flexible tradeoffs
 between Write-Amplification-Factor (WAF), Read-Amplification-Factor (RAF)
 and Space-Amplification-Factor (SAF). It has multi-threaded compactions,
-making it especially suitable for storing multiple terabytes of data in a
+making it specially suitable for storing multiple terabytes of data in a
 single database.
 
-Start with example usage here: https://github.com/facebook/rocksdb/tree/main/examples
+Start with example usage here: https://github.com/facebook/rocksdb/tree/master/examples
 
 See the [github wiki](https://github.com/facebook/rocksdb/wiki) for more explanation.
 
@@ -22,7 +25,7 @@ The public interface is in `include/`.  Callers should not include or
 rely on the details of any other header files in this package.  Those
 internal APIs may be changed without warning.
 
-Questions and discussions are welcome on the [RocksDB Developers Public](https://www.facebook.com/groups/rocksdb.dev/) Facebook group and [email list](https://groups.google.com/g/rocksdb) on Google Groups.
+Design discussions are conducted in https://www.facebook.com/groups/rocksdb.dev/
 
 ## License
 

ROCKSDB_LITE.md

@@ -0,0 +1,21 @@
+# RocksDBLite
+
+RocksDBLite is a project focused on mobile use cases, which don't need a lot of fancy things we've built for server workloads and they are very sensitive to binary size. For that reason, we added a compile flag ROCKSDB_LITE that comments out a lot of the nonessential code and keeps the binary lean.
+
+Some examples of the features disabled by ROCKSDB_LITE:
+* compiled-in support for LDB tool
+* No backupable DB
+* No support for replication (which we provide in form of TransactionalIterator)
+* No advanced monitoring tools
+* No special-purpose memtables that are highly optimized for specific use cases
+* No Transactions
+
+When adding a new big feature to RocksDB, please add ROCKSDB_LITE compile guard if:
+* Nobody from mobile really needs your feature,
+* Your feature is adding a lot of weight to the binary.
+
+Don't add ROCKSDB_LITE compile guard if:
+* It would introduce a lot of code complexity. Compile guards make code harder to read. It's a trade-off.
+* Your feature is not adding a lot of weight.
+
+If unsure, ask. :)

USERS.md

@@ -12,49 +12,21 @@ At Facebook, we use RocksDB as storage engines in multiple data management servi
 6. LogDevice -- a distributed data store for logs [2]
 
 [1] https://research.facebook.com/publications/realtime-data-processing-at-facebook/
-
 [2] https://code.facebook.com/posts/357056558062811/logdevice-a-distributed-data-store-for-logs/
 
-## Bilibili
-[Bilibili](bilibili.com) [uses](https://www.alluxio.io/blog/when-ai-meets-alluxio-at-bilibili-building-an-efficient-ai-platform-for-data-preprocessing-and-model-training/) Alluxio to speed up its ML training workloads, and Alluxio uses RocksDB to store its filesystem metadata, so Bilibili uses RocksDB.
-
-Bilibili's [real-time platform](https://www.alibabacloud.com/blog/architecture-and-practices-of-bilibilis-real-time-platform_596676) uses Flink, and uses RocksDB as Flink's state store.
-
-## TikTok
-TikTok, or its parent company ByteDance, uses RocksDB as the storage engine for some storage systems, such as its distributed graph database [ByteGraph](https://vldb.org/pvldb/vol15/p3306-li.pdf). 
-
-Also, TikTok uses [Alluxio](alluxio.io) to [speed up Presto queries](https://www.alluxio.io/resources/videos/improving-presto-performance-with-alluxio-at-tiktok/), and Alluxio stores the files' metadata in RocksDB.
-
-## FoundationDB
-[FoundationDB](https://www.foundationdb.org/) [uses](https://github.com/apple/foundationdb/blob/377f1f692da6ab2fe5bdac57035651db3e5fb66d/fdbserver/KeyValueStoreRocksDB.actor.cpp) RocksDB to implement a [key-value store interface](https://github.com/apple/foundationdb/blob/377f1f692da6ab2fe5bdac57035651db3e5fb66d/fdbserver/KeyValueStoreRocksDB.actor.cpp#L1127) in its server backend.
-
-## Apple
-Apple [uses](https://opensource.apple.com/projects/foundationdb/) FoundationDB, so it also uses RocksDB.
-
-## Snowflake
-Snowflake [uses](https://www.snowflake.com/blog/how-foundationdb-powers-snowflake-metadata-forward/) FoundationDB, so it also uses RocksDB.
-
-## Microsoft
-The Bing search engine from Microsoft uses RocksDB as the storage engine for its web data platform: https://blogs.bing.com/Engineering-Blog/october-2021/RocksDB-in-Microsoft-Bing
-
 ## LinkedIn
-1. [Venice](https://venicedb.org/) is a derived data platform using RocksDB as its storage engine. It is LinkedIn's ML feature store, powering thousands of recommender use cases, including the Feed, Video recommendations, and People You May Know.
-2. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter
-3. Apache Samza, open source framework for stream processing.
+Two different use cases at Linkedin are using RocksDB as a storage engine:
 
-Learn more about LinkedIn's follow feed and Apache Samza in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg
+1. LinkedIn's follow feed for storing user's activities. Check out the blog post: https://engineering.linkedin.com/blog/2016/03/followfeed--linkedin-s-feed-made-faster-and-smarter
+2. Apache Samza, open source framework for stream processing
+
+Learn more about those use cases in a Tech Talk by Ankit Gupta and Naveen Somasundaram: http://www.youtube.com/watch?v=plqVp_OnSzg
 
 ## Yahoo
 Yahoo is using RocksDB as a storage engine for their biggest distributed data store Sherpa. Learn more about it here: http://yahooeng.tumblr.com/post/120730204806/sherpa-scales-new-heights
 
-## Tencent
-[PaxosStore](https://github.com/Tencent/paxosstore) is a distributed database supporting WeChat. It uses RocksDB as its storage engine.
-
-## Baidu
-[Apache Doris](http://doris.apache.org/master/en/) is a MPP analytical database engine released by Baidu. It [uses RocksDB](http://doris.apache.org/master/en/administrator-guide/operation/tablet-meta-tool.html) to manage its tablet's metadata.
-
 ## CockroachDB
-CockroachDB is an open-source geo-replicated transactional database. They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
+CockroachDB is an open-source geo-replicated transactional database (still in development). They are using RocksDB as their storage engine. Check out their github: https://github.com/cockroachdb/cockroach
 
 ## DNANexus
 DNANexus is using RocksDB to speed up processing of genomics data.
@@ -71,16 +43,12 @@ Tango is using RocksDB as a graph storage to store all users' connection data an
 Turn is using RocksDB as a storage layer for their key/value store, serving at peak 2.4MM QPS out of different datacenters.
 Check out our RocksDB Protobuf merge operator at: https://github.com/vladb38/rocksdb_protobuf
 
-## Santander UK/Cloudera Profession Services
+## Santanader UK/Cloudera Profession Services
 Check out their blog post: http://blog.cloudera.com/blog/2015/08/inside-santanders-near-real-time-data-ingest-architecture/
 
 ## Airbnb
 Airbnb is using RocksDB as a storage engine for their personalized search service. You can learn more about it here: https://www.youtube.com/watch?v=ASQ6XMtogMs
 
-## Alluxio
-[Alluxio](https://www.alluxio.io) uses RocksDB to serve and scale file system metadata to beyond 1 Billion files. The detailed design and implementation is described in this engineering blog:
-https://www.alluxio.io/blog/scalable-metadata-service-in-alluxio-storing-billions-of-files/
-
 ## Pinterest
 Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtube.com/watch?v=MtFEVEs_2Vo
 
@@ -94,7 +62,7 @@ Pinterest's Object Retrieval System uses RocksDB for storage: https://www.youtub
 [VWO's](https://vwo.com/) Smart Code checker and URL helper uses RocksDB to store all the URLs where VWO's Smart Code is installed.
 
 ## quasardb
-[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark.
+[quasardb](https://www.quasardb.net) is a high-performance, distributed, transactional key-value database that integrates well with in-memory analytics engines such as Apache Spark. 
 quasardb uses a heavily tuned RocksDB as its persistence layer.
 
 ## Netflix
@@ -103,18 +71,6 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 ## TiKV
 [TiKV](https://github.com/pingcap/tikv) is a GEO-replicated, high-performance, distributed, transactional key-value database. TiKV is powered by Rust and Raft. TiKV uses RocksDB as its persistence layer.
 
-## TiDB
-[TiDB](https://github.com/pingcap/tidb) uses the TiKV distributed key-value database, so it uses RocksDB.
-
-## PingCAP
-[PingCAP](https://www.pingcap.com/) is the company behind TiDB, its cloud database service uses RocksDB.
-
-## Apache Spark
-[Spark Structured Streaming](https://docs.databricks.com/structured-streaming/rocksdb-state-store.html) uses RocksDB as the local state store.
-
-## Databricks
-[Databricks](https://www.databricks.com/) [replaces AWS RDS with TiDB](https://www.pingcap.com/case-study/how-databricks-tackles-the-scalability-limit-with-a-mysql-alternative/) for scalability, so it uses RocksDB.
-
 ## Apache Flink
 [Apache Flink](https://flink.apache.org/news/2016/03/08/release-1.0.0.html) uses RocksDB to store state locally on a machine.
 
@@ -125,47 +81,10 @@ quasardb uses a heavily tuned RocksDB as its persistence layer.
 [Uber](http://eng.uber.com/cherami/) uses RocksDB as a durable and scalable task queue.
 
 ## 360 Pika
-[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been used in many companies.
+[360](http://www.360.cn/) [Pika](https://github.com/Qihoo360/pika) is a nosql compatible with redis. With the huge amount of data stored, redis may suffer for a capacity bottleneck, and pika was born for solving it. It has widely been widely used in many company
 
 ## LzLabs
 LzLabs is using RocksDB as a storage engine in their multi-database distributed framework to store application configuration and user data.
 
 ## ProfaneDB
 [ProfaneDB](https://profanedb.gitlab.io/) is a database for Protocol Buffers, and uses RocksDB for storage. It is accessible via gRPC, and the schema is defined using directly `.proto` files.
-
-## IOTA Foundation
- [IOTA Foundation](https://www.iota.org/) is using RocksDB in the [IOTA Reference Implementation (IRI)](https://github.com/iotaledger/iri) to store the local state of the Tangle. The Tangle is the first open-source distributed ledger powering the future of the Internet of Things.
-
-## Avrio Project
- [Avrio Project](http://avrio-project.github.io/avrio.network/) is using RocksDB in [Avrio ](https://github.com/avrio-project/avrio) to store blocks, account balances and data and other blockchain-releated data. Avrio is a multiblockchain decentralized cryptocurrency empowering monetary transactions.
-
-## Crux
-[Crux](https://github.com/juxt/crux) is a document database that uses RocksDB for local [EAV](https://en.wikipedia.org/wiki/Entity%E2%80%93attribute%E2%80%93value_model) index storage to enable point-in-time bitemporal Datalog queries. The "unbundled" architecture uses Kafka to provide horizontal scalability.
-
-## Nebula Graph
-[Nebula Graph](https://github.com/vesoft-inc/nebula) is a distributed, scalable, lightning-fast, open source graph database capable of hosting super large scale graphs with dozens of billions of vertices (nodes) and trillions of edges, with milliseconds of latency.
-
-## YugabyteDB
-[YugabyteDB](https://www.yugabyte.com/) is an open source, high performance, distributed SQL database that uses RocksDB as its storage layer. For more information, please see https://github.com/yugabyte/yugabyte-db/.
-
-## ArangoDB
-[ArangoDB](https://www.arangodb.com/) is a native multi-model database with flexible data models for documents, graphs, and key-values, for building high performance applications using a convenient SQL-like query language or JavaScript extensions. It uses RocksDB as its storage engine.
-
-## Qdrant
-[Qdrant](https://qdrant.tech/) is an open source vector database, it [uses](https://qdrant.tech/documentation/concepts/storage/) RocksDB as its persistent storage.
-
-## Milvus
-[Milvus](https://milvus.io/) is an open source vector database for unstructured data. It uses RocksDB not only as one of the supported kv storage engines, but also as a message queue.
-
-## Kafka
-[Kafka](https://kafka.apache.org/) is an open-source distributed event streaming platform, it uses RocksDB to store state in Kafka Streams: https://www.confluent.io/blog/how-to-tune-rocksdb-kafka-streams-state-stores-performance/.
-
-## Solana Labs
-[Solana](https://github.com/solana-labs/solana) is a fast, secure, scalable, and decentralized blockchain.  It uses RocksDB as the underlying storage for its ledger store.
-
-## Apache Kvrocks
-
-[Apache Kvrocks](https://github.com/apache/kvrocks) is an open-source distributed key-value NoSQL database built on top of RocksDB. It serves as a cost-saving and capacity-increasing alternative drop-in replacement for Redis.
-
-## Others
-More databases using RocksDB can be found at [dbdb.io](https://dbdb.io/browse?embeds=rocksdb).

WINDOWS_PORT.md

@@ -24,7 +24,7 @@ We strive to achieve the following goals:
 * make all unit test pass both in debug and release builds. 
   * Note: latest introduction of SyncPoint seems to disable running db_test in Release.
 * make performance on par with published benchmarks accounting for HW differences
-* we would like to keep the port code inline with the main branch with no forking
+* we would like to keep the port code inline with the master branch with no forking
 
 ## Build system
 We have chosen CMake as a widely accepted build system to build the Windows port. It is very fast and convenient. 
@@ -43,9 +43,9 @@ We plan to use this port for our business purposes here at Bing and this provide
 
 * Certain headers that are not present and not necessary on Windows were simply `#ifndef OS_WIN` in a few places (`unistd.h`)
 * All posix specific headers were replaced to port/port.h which worked well
-* Replaced `dirent.h` for `port/port_dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
+* Replaced `dirent.h` for `port/dirent.h` (very few places) with the implementation of the relevant interfaces within `rocksdb::port` namespace
 * Replaced `sys/time.h` to `port/sys_time.h` (few places) implemented equivalents within `rocksdb::port`
-* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `zu` on posix systems and to `Iu` on windows.
+* `printf %z` specification is not supported on Windows. To imitate existing standards we came up with a string macro `ROCKSDB_PRIszt` which expands to `%z` on posix systems and to Iu on windows.
 * in class member initialization were moved to a __ctors in some cases
 * `constexpr` is not supported. We had to replace `std::numeric_limits<>::max/min()` to its C macros for constants. Sometimes we had to make class members `static const` and place a definition within a .cc file.
 * `constexpr` for functions was replaced to a template specialization (1 place)
@@ -66,7 +66,7 @@ We endeavored to make it functionally on par with posix_env. This means we repli
 Even though Windows provides its own efficient thread-pool implementation we chose to replicate posix logic using `std::thread` primitives. This allows anyone to quickly detect any changes within the posix source code and replicate them within windows env. This has proven to work very well. At the same time for anyone who wishes to replace the built-in thread-pool can do so using RocksDB stackable environments.
 
 For disk access we implemented all of the functionality present within the posix_env which includes memory mapped files, random access, rate-limiter support etc.
-The `use_os_buffer` flag on Posix platforms currently denotes disabling read-ahead log via `fadvise` mechanism. Windows does not have `fadvise` system call. What is more, it implements disk cache in a way that differs from Linux greatly. It's not an uncommon practice on Windows to perform un-buffered disk access to gain control of the memory consumption. We think that in our use case this may also be a good configuration option at the expense of disk throughput. To compensate one may increase the configured in-memory cache size instead. Thus we have chosen  `use_os_buffer=false` to disable OS disk buffering for `WinWritableFile` and `WinRandomAccessFile`. The OS imposes restrictions on the alignment of the disk offsets, buffers used and the amount of data that is read/written when accessing files in un-buffered mode. When the option is true, the classes behave in a standard way. This allows to perform writes and reads in cases when un-buffered access does not make sense such as WAL and MANIFEST.
+The `use_os_buffer` flag on Posix platforms currently denotes disabling read-ahead log via `fadvise` mechanism. Windows does not have `fadvise` system call. What is more, it implements disk cache in a way that differs from Linux greatly. It’s not an uncommon practice on Windows to perform un-buffered disk access to gain control of the memory consumption. We think that in our use case this may also be a good configuration option at the expense of disk throughput. To compensate one may increase the configured in-memory cache size instead. Thus we have chosen  `use_os_buffer=false` to disable OS disk buffering for `WinWritableFile` and `WinRandomAccessFile`. The OS imposes restrictions on the alignment of the disk offsets, buffers used and the amount of data that is read/written when accessing files in un-buffered mode. When the option is true, the classes behave in a standard way. This allows to perform writes and reads in cases when un-buffered access does not make sense such as WAL and MANIFEST.
 
 We have replaced `pread/pwrite` with `WriteFile/ReadFile` with `OVERLAPPED` structure so we can atomically seek to the position of the disk operation but still perform the operation synchronously. Thus we able to emulate that functionality of `pread/pwrite` reasonably well. The only difference is that the file pointer is not returned to its original position but that hardly matters given the random nature of access.
 

appveyor.yml

@@ -0,0 +1,15 @@
+version: 1.0.{build}
+image: Visual Studio 2017
+before_build:
+- md %APPVEYOR_BUILD_FOLDER%\build
+- cd %APPVEYOR_BUILD_FOLDER%\build
+- cmake -G "Visual Studio 15 Win64" -DOPTDBG=1 -DWITH_XPRESS=1 -DPORTABLE=1 ..
+- cd ..
+build:
+  project: build\rocksdb.sln
+  parallel: true
+  verbosity: normal
+test:
+test_script:
+- ps: build_tools\run_ci_db_test.ps1 -SuiteRun db_basic_test,db_test2,db_test,env_basic_test,env_test -Concurrency 8
+

buckifier/buckify_rocksdb.py

@@ -1,36 +1,16 @@
-#!/usr/bin/env python3
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-
-try:
-    from builtins import str
-except ImportError:
-    from __builtin__ import str
-import fnmatch
-import json
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from targets_builder import TARGETSBuilder
+from optparse import OptionParser
 import os
+import fnmatch
 import sys
-
-from targets_builder import TARGETSBuilder, LiteralValue
+import tempfile
 
 from util import ColorString
-
-# This script generates BUCK file for Buck.
-# Buck is a build tool specifying dependencies among different build targets.
-# User can pass extra dependencies as a JSON object via command line, and this
-# script can include these dependencies in the generate BUCK file.
-# Usage:
-# $python3 buckifier/buckify_rocksdb.py
-# (This generates a TARGET file without user-specified dependency for unit
-# tests.)
-# $python3 buckifier/buckify_rocksdb.py \
-#        '{"fake": {
-#                      "extra_deps": [":test_dep", "//fakes/module:mock1"],
-#                      "extra_compiler_flags": ["-DFOO_BAR", "-Os"]
-#                  }
-#         }'
-# (Generated BUCK file has test_dep and mock1 as dependencies for RocksDB
-# unit tests, and will use the extra_compiler_flags to compile the unit test
-# source.)
+import util
 
 # tests to export as libraries for inclusion in other projects
 _EXPORTED_TEST_LIBS = ["env_basic_test"]
@@ -42,13 +22,13 @@ def parse_src_mk(repo_path):
     src_files = {}
     for line in open(src_mk):
         line = line.strip()
-        if len(line) == 0 or line[0] == "#":
+        if len(line) == 0 or line[0] == '#':
             continue
-        if "=" in line:
-            current_src = line.split("=")[0].strip()
+        if '=' in line:
+            current_src = line.split('=')[0].strip()
             src_files[current_src] = []
-        elif ".c" in line:
-            src_path = line.split("\\")[0].strip()
+        elif '.cc' in line:
+            src_path = line.split('.cc')[0].strip() + '.cc'
             src_files[current_src].append(src_path)
     return src_files
 
@@ -56,315 +36,115 @@ def parse_src_mk(repo_path):
 # get all .cc / .c files
 def get_cc_files(repo_path):
     cc_files = []
-    for root, _dirnames, filenames in os.walk(
-        repo_path
-    ):  # noqa: B007 T25377293 Grandfathered in
-        root = root[(len(repo_path) + 1) :]
+    for root, dirnames, filenames in os.walk(repo_path):  # noqa: B007 T25377293 Grandfathered in
+        root = root[(len(repo_path) + 1):]
         if "java" in root:
             # Skip java
             continue
-        for filename in fnmatch.filter(filenames, "*.cc"):
+        for filename in fnmatch.filter(filenames, '*.cc'):
             cc_files.append(os.path.join(root, filename))
-        for filename in fnmatch.filter(filenames, "*.c"):
+        for filename in fnmatch.filter(filenames, '*.c'):
             cc_files.append(os.path.join(root, filename))
     return cc_files
 
 
-# Get non_parallel tests from Makefile
-def get_non_parallel_tests(repo_path):
+# Get tests from Makefile
+def get_tests(repo_path):
     Makefile = repo_path + "/Makefile"
 
-    s = set({})
+    # Dictionary TEST_NAME => IS_PARALLEL
+    tests = {}
 
-    found_non_parallel_tests = False
+    found_tests = False
     for line in open(Makefile):
         line = line.strip()
-        if line.startswith("NON_PARALLEL_TEST ="):
-            found_non_parallel_tests = True
-        elif found_non_parallel_tests:
+        if line.startswith("TESTS ="):
+            found_tests = True
+        elif found_tests:
             if line.endswith("\\"):
                 # remove the trailing \
                 line = line[:-1]
                 line = line.strip()
-                s.add(line)
+                tests[line] = False
             else:
-                # we consumed all the non_parallel tests
+                # we consumed all the tests
                 break
 
-    return s
+    found_parallel_tests = False
+    for line in open(Makefile):
+        line = line.strip()
+        if line.startswith("PARALLEL_TEST ="):
+            found_parallel_tests = True
+        elif found_parallel_tests:
+            if line.endswith("\\"):
+                # remove the trailing \
+                line = line[:-1]
+                line = line.strip()
+                tests[line] = True
+            else:
+                # we consumed all the parallel tests
+                break
+    
+    return tests
 
 
-# Parse extra dependencies passed by user from command line
-def get_dependencies():
-    deps_map = {"": {"extra_deps": [], "extra_compiler_flags": []}}
-    if len(sys.argv) < 2:
-        return deps_map
-
-    def encode_dict(data):
-        rv = {}
-        for k, v in data.items():
-            if isinstance(v, dict):
-                v = encode_dict(v)
-            rv[k] = v
-        return rv
-
-    extra_deps = json.loads(sys.argv[1], object_hook=encode_dict)
-    for target_alias, deps in extra_deps.items():
-        deps_map[target_alias] = deps
-    return deps_map
-
-
-# Prepare BUCK file for buck
-def generate_buck(repo_path, deps_map):
-    print(ColorString.info("Generating BUCK"))
+# Prepare TARGETS file for buck
+def generate_targets(repo_path):
+    print(ColorString.info("Generating TARGETS"))
     # parsed src.mk file
     src_mk = parse_src_mk(repo_path)
     # get all .cc files
     cc_files = get_cc_files(repo_path)
-    # get non_parallel tests from Makefile
-    non_parallel_tests = get_non_parallel_tests(repo_path)
+    # get tests from Makefile
+    tests = get_tests(repo_path)
 
-    if src_mk is None or cc_files is None or non_parallel_tests is None:
+    if src_mk is None or cc_files is None or tests is None:
         return False
 
-    extra_argv = ""
-    if len(sys.argv) >= 2:
-        # Heuristically quote and canonicalize whitespace for inclusion
-        # in how the file was generated.
-        extra_argv = " '{}'".format(" ".join(sys.argv[1].split()))
-
-    BUCK = TARGETSBuilder("%s/BUCK" % repo_path, extra_argv)
-
-    # Add oncall("rocksdb_point_of_contact") at the top
-    BUCK.add_oncall("rocksdb_point_of_contact")
-
+    TARGETS = TARGETSBuilder("%s/TARGETS" % repo_path)
     # rocksdb_lib
-    BUCK.add_library(
+    TARGETS.add_library(
         "rocksdb_lib",
         src_mk["LIB_SOURCES"] +
-        # always add range_tree, it's only excluded on ppc64, which we don't use internally
-        src_mk["RANGE_TREE_SOURCES"] + src_mk["TOOL_LIB_SOURCES"],
-        deps=[
-            "//folly/container:f14_hash",
-            "//folly/coro:blocking_wait",
-            "//folly/coro:collect",
-            "//folly/coro:coroutine",
-            "//folly/coro:task",
-            "//folly/synchronization:distributed_mutex",
-        ],
-        headers=LiteralValue("glob([\"**/*.h\"])")
-    )
-    # rocksdb_whole_archive_lib
-    BUCK.add_library(
-        "rocksdb_whole_archive_lib",
-        [],
-        deps=[
-            ":rocksdb_lib",
-        ],
-        extra_external_deps="",
-        link_whole=True,
-    )
-    # rocksdb_with_faiss_lib
-    BUCK.add_library(
-        "rocksdb_with_faiss_lib",
-        src_mk.get("WITH_FAISS_LIB_SOURCES", []),
-        deps=[
-            "//faiss:faiss",
-            ":rocksdb_lib",
-        ],
-    )
+        src_mk["TOOL_LIB_SOURCES"])
     # rocksdb_test_lib
-    BUCK.add_library(
+    TARGETS.add_library(
         "rocksdb_test_lib",
-        src_mk.get("MOCK_LIB_SOURCES", [])
-        + src_mk.get("TEST_LIB_SOURCES", [])
-        + src_mk.get("EXP_LIB_SOURCES", [])
-        + src_mk.get("ANALYZER_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-        extra_test_libs=True,
-    )
-    # rocksdb_with_faiss_test_lib
-    BUCK.add_library(
-        "rocksdb_with_faiss_test_lib",
-        src_mk.get("MOCK_LIB_SOURCES", [])
-        + src_mk.get("TEST_LIB_SOURCES", [])
-        + src_mk.get("EXP_LIB_SOURCES", [])
-        + src_mk.get("ANALYZER_LIB_SOURCES", []),
-        deps=[
-            ":rocksdb_with_faiss_lib",
-        ],
-        extra_test_libs=True,
-    )
+        src_mk.get("MOCK_LIB_SOURCES", []) +
+        src_mk.get("TEST_LIB_SOURCES", []) +
+        src_mk.get("EXP_LIB_SOURCES", []),
+        [":rocksdb_lib"])
     # rocksdb_tools_lib
-    BUCK.add_library(
+    TARGETS.add_library(
         "rocksdb_tools_lib",
-        src_mk.get("BENCH_LIB_SOURCES", [])
-        + src_mk.get("ANALYZER_LIB_SOURCES", [])
-        + ["test_util/testutil.cc"],
-        [":rocksdb_lib"],
-    )
-    # rocksdb_cache_bench_tools_lib
-    BUCK.add_library(
-        "rocksdb_cache_bench_tools_lib",
-        src_mk.get("CACHE_BENCH_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-    )
-    # rocksdb_point_lock_bench_tools_lib
-    BUCK.add_library(
-        "rocksdb_point_lock_bench_tools_lib",
-        src_mk.get("POINT_LOCK_BENCH_LIB_SOURCES", []),
-        [":rocksdb_lib"],
-    )
-    # rocksdb_stress_lib
-    BUCK.add_rocksdb_library(
-        "rocksdb_stress_lib",
-        src_mk.get("ANALYZER_LIB_SOURCES", [])
-        + src_mk.get("STRESS_LIB_SOURCES", [])
-        + ["test_util/testutil.cc"],
-    )
-    # ldb binary
-    BUCK.add_binary(
-        "ldb", ["tools/ldb.cc"], [":rocksdb_tools_lib"]
-    )
-    # db_stress binary
-    BUCK.add_binary(
-        "db_stress", ["db_stress_tool/db_stress.cc"], [":rocksdb_stress_lib"]
-    )
-    # db_bench binary
-    BUCK.add_binary(
-        "db_bench", ["tools/db_bench.cc"], [":rocksdb_tools_lib"]
-    )
-    # cache_bench binary
-    BUCK.add_binary(
-        "cache_bench", ["cache/cache_bench.cc"], [":rocksdb_cache_bench_tools_lib"]
-    )
-    # point_lock_bench binary
-    BUCK.add_binary(
-        "point_lock_bench",
-        ["utilities/transactions/lock/point/point_lock_bench.cc"],
-        [":rocksdb_point_lock_bench_tools_lib"]
-    )
-    # bench binaries
-    for src in src_mk.get("MICROBENCH_SOURCES", []):
-        name = src.rsplit("/", 1)[1].split(".")[0] if "/" in src else src.split(".")[0]
-        BUCK.add_binary(name, [src], [], extra_bench_libs=True)
-    print(f"Extra dependencies:\n{json.dumps(deps_map)}")
+        src_mk.get("BENCH_LIB_SOURCES", []) +
+        ["util/testutil.cc"],
+        [":rocksdb_lib"])
 
-    # Dictionary test executable name -> relative source file path
-    test_source_map = {}
+    # test for every test we found in the Makefile
+    for test in sorted(tests):
+        match_src = [src for src in cc_files if ("/%s.c" % test) in src]
+        if len(match_src) == 0:
+            print(ColorString.warning("Cannot find .cc file for %s" % test))
+            continue
+        elif len(match_src) > 1:
+            print(ColorString.warning("Found more than one .cc for %s" % test))
+            print(match_src)
+            continue
 
-    # c_test.c is added through BUCK.add_c_test(). If there
-    # are more than one .c test file, we need to extend
-    # BUCK.add_c_test() to include other C tests too.
-    for test_src in src_mk.get("TEST_MAIN_SOURCES_C", []):
-        if test_src != "db/c_test.c":
-            print("Don't know how to deal with " + test_src)
-            return False
-    BUCK.add_c_test()
+        assert(len(match_src) == 1)
+        is_parallel = tests[test]
+        TARGETS.register_test(test, match_src[0], is_parallel)
 
-    try:
-        with open(f"{repo_path}/buckifier/bench.json") as json_file:
-            fast_fancy_bench_config_list = json.load(json_file)
-            for config_dict in fast_fancy_bench_config_list:
-                clean_benchmarks = {}
-                benchmarks = config_dict["benchmarks"]
-                for binary, benchmark_dict in benchmarks.items():
-                    clean_benchmarks[binary] = {}
-                    for benchmark, overloaded_metric_list in benchmark_dict.items():
-                        clean_benchmarks[binary][benchmark] = []
-                        for metric in overloaded_metric_list:
-                            if not isinstance(metric, dict):
-                                clean_benchmarks[binary][benchmark].append(metric)
-                BUCK.add_fancy_bench_config(
-                    config_dict["name"],
-                    clean_benchmarks,
-                    False,
-                    config_dict["expected_runtime_one_iter"],
-                    config_dict["sl_iterations"],
-                    config_dict["regression_threshold"],
-                )
+        if test in _EXPORTED_TEST_LIBS:
+            test_library = "%s_lib" % test
+            TARGETS.add_library(test_library, match_src, [":rocksdb_test_lib"])
+    TARGETS.flush_tests()
 
-        with open(f"{repo_path}/buckifier/bench-slow.json") as json_file:
-            slow_fancy_bench_config_list = json.load(json_file)
-            for config_dict in slow_fancy_bench_config_list:
-                clean_benchmarks = {}
-                benchmarks = config_dict["benchmarks"]
-                for binary, benchmark_dict in benchmarks.items():
-                    clean_benchmarks[binary] = {}
-                    for benchmark, overloaded_metric_list in benchmark_dict.items():
-                        clean_benchmarks[binary][benchmark] = []
-                        for metric in overloaded_metric_list:
-                            if not isinstance(metric, dict):
-                                clean_benchmarks[binary][benchmark].append(metric)
-            for config_dict in slow_fancy_bench_config_list:
-                BUCK.add_fancy_bench_config(
-                    config_dict["name"] + "_slow",
-                    clean_benchmarks,
-                    True,
-                    config_dict["expected_runtime_one_iter"],
-                    config_dict["sl_iterations"],
-                    config_dict["regression_threshold"],
-                )
-    # it is better servicelab experiments break
-    # than rocksdb github ci
-    except Exception:
-        pass
-
-    BUCK.add_test_header()
-
-    for test_src in src_mk.get("TEST_MAIN_SOURCES", []):
-        test = test_src.split(".c")[0].strip().split("/")[-1].strip()
-        test_source_map[test] = (test_src, False)
-        print("" + test + " " + test_src)
-
-    for test_src in src_mk.get("WITH_FAISS_TEST_MAIN_SOURCES", []):
-        test = test_src.split(".c")[0].strip().split("/")[-1].strip()
-        test_source_map[test] = (test_src, True)
-        print("" + test + " " + test_src + " [FAISS]")
-
-    for target_alias, deps in deps_map.items():
-        for test, (test_src, with_faiss) in sorted(test_source_map.items()):
-            if len(test) == 0:
-                print(ColorString.warning("Failed to get test name for %s" % test_src))
-                continue
-
-            test_target_name = test if not target_alias else test + "_" + target_alias
-
-            if test in _EXPORTED_TEST_LIBS:
-                test_library = "%s_lib" % test_target_name
-                BUCK.add_library(
-                    test_library,
-                    [test_src],
-                    deps=[":rocksdb_test_lib"],
-                    extra_test_libs=True,
-                )
-                BUCK.register_test(
-                    test_target_name,
-                    test_src,
-                    deps=json.dumps(deps["extra_deps"] + [":" + test_library]),
-                    extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
-                )
-            else:
-                if with_faiss:
-                    BUCK.register_test(
-                        test_target_name,
-                        test_src,
-                        deps=json.dumps(deps["extra_deps"] + [":rocksdb_with_faiss_test_lib"]),
-                        extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
-                    )
-                else:
-                    BUCK.register_test(
-                        test_target_name,
-                        test_src,
-                        deps=json.dumps(deps["extra_deps"] + [":rocksdb_test_lib"]),
-                        extra_compiler_flags=json.dumps(deps["extra_compiler_flags"]),
-                    )
-    BUCK.export_file("tools/db_crashtest.py")
-
-    print(ColorString.info("Generated BUCK Summary:"))
-    print(ColorString.info("- %d libs" % BUCK.total_lib))
-    print(ColorString.info("- %d binarys" % BUCK.total_bin))
-    print(ColorString.info("- %d tests" % BUCK.total_test))
+    print(ColorString.info("Generated TARGETS Summary:"))
+    print(ColorString.info("- %d libs" % TARGETS.total_lib))
+    print(ColorString.info("- %d binarys" % TARGETS.total_bin))
+    print(ColorString.info("- %d tests" % TARGETS.total_test))
     return True
 
 
@@ -372,23 +152,21 @@ def get_rocksdb_path():
     # rocksdb = {script_dir}/..
     script_dir = os.path.dirname(sys.argv[0])
     script_dir = os.path.abspath(script_dir)
-    rocksdb_path = os.path.abspath(os.path.join(script_dir, "../"))
+    rocksdb_path = os.path.abspath(
+        os.path.join(script_dir, "../"))
 
     return rocksdb_path
 
-
 def exit_with_error(msg):
     print(ColorString.error(msg))
     sys.exit(1)
 
 
 def main():
-    deps_map = get_dependencies()
-    # Generate BUCK file for buck
-    ok = generate_buck(get_rocksdb_path(), deps_map)
+    # Generate TARGETS file for buck
+    ok = generate_targets(get_rocksdb_path())
     if not ok:
-        exit_with_error("Failed to generate BUCK files")
-
+        exit_with_error("Failed to generate TARGETS files")
 
 if __name__ == "__main__":
     main()

buckifier/check_buck_targets.sh

@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-
-if [[ ! -f "BUCK" ]]
-then
-  echo "BUCK file is missing!"
-  echo "Please do not remove / rename BUCK file in your commit(s)."
-  exit 1
-fi
-
-TGT_DIFF=`git diff BUCK | head -n 1`
-
-if [ ! -z "$TGT_DIFF" ]
-then
-  echo "BUCK file has uncommitted changes. Skip this check."
-  exit 0
-fi
-
-echo Backup original BUCK file.
-
-cp BUCK BUCK.bkp
-
-${PYTHON:-python3} buckifier/buckify_rocksdb.py
-
-if [[ ! -f "BUCK" ]]
-then
-  echo "BUCK file went missing after running buckifier/buckify_rocksdb.py!"
-  echo "Please do not remove the BUCK file."
-  exit 1
-fi
-
-TGT_DIFF=`git diff BUCK | head -n 1`
-
-if [ -z "$TGT_DIFF" ]
-then
-  mv BUCK.bkp BUCK
-  exit 0
-else
-  echo "Please run '${PYTHON:-python3} buckifier/buckify_rocksdb.py' to update BUCK file."
-  echo "Do not manually update BUCK file."
-  ${PYTHON:-python3} --version
-  mv BUCK.bkp BUCK
-  exit 1
-fi

buckifier/rocks_test_runner.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # Create a tmp directory for the test to use
 TEST_DIR=$(mktemp -d /dev/shm/fbcode_rocksdb_XXXXXXX)
 # shellcheck disable=SC2068

buckifier/targets_builder.py

@@ -1,175 +1,63 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-
-try:
-    from builtins import object, str
-except ImportError:
-    from __builtin__ import object, str
-import pprint
-
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 import targets_cfg
 
-class LiteralValue:
-    def __init__(self, value):
-        self.value = value
-
-    def __str__(self):
-        return str(self.value)
-
-def smart_quote_value(val):
-    if isinstance(val, LiteralValue):
-        return str(val)
-    return '"%s"' % val
-
 def pretty_list(lst, indent=8):
     if lst is None or len(lst) == 0:
         return ""
 
     if len(lst) == 1:
-        return smart_quote_value(lst[0])
-
-    separator = ',\n%s' % (" " * indent)
-    res = separator.join(sorted(map(smart_quote_value, lst)))
-    res = "\n" + (" " * indent) + res + ',\n' + (" " * (indent - 4))
+        return "\"%s\"" % lst[0]
+    
+    separator = "\",\n%s\"" % (" " * indent)
+    res = separator.join(sorted(lst))
+    res = "\n" + (" " * indent) + "\"" + res + "\",\n" + (" " * (indent - 4))
     return res
 
 
 class TARGETSBuilder:
-    def __init__(self, path, extra_argv):
+    def __init__(self, path):
         self.path = path
-        header = targets_cfg.rocksdb_target_header_template.format(
-            extra_argv=extra_argv
-        )
-        with open(path, "wb") as targets_file:
-            targets_file.write(header.encode("utf-8"))
+        self.targets_file = open(path, 'w')
+        self.targets_file.write(targets_cfg.rocksdb_target_header)
         self.total_lib = 0
         self.total_bin = 0
         self.total_test = 0
         self.tests_cfg = ""
-    
-    def add_oncall(self, oncall):
-       with open(self.path, "ab") as targets_file:
-            targets_file.write(targets_cfg.oncall_template.format(name=oncall).encode("utf-8"))
-                
 
-    def add_library(
-        self,
-        name,
-        srcs,
-        deps=None,
-        headers=None,
-        extra_external_deps="",
-        link_whole=False,
-        external_dependencies=None,
-        extra_test_libs=False,
-    ):
-        if headers is not None:
-            if isinstance(headers, LiteralValue):
-                headers = str(headers)
-            else:
-                headers = "[" + pretty_list(headers) + "]"
-        else:
-            headers = "[]"
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                targets_cfg.library_template.format(
-                    name=name,
-                    srcs=pretty_list(srcs),
-                    headers=headers,
-                    deps=pretty_list(deps),
-                    extra_external_deps=extra_external_deps,
-                    link_whole=link_whole,
-                    external_dependencies=pretty_list(external_dependencies),
-                    extra_test_libs=extra_test_libs,
-                ).encode("utf-8")
-            )
+    def __del__(self):
+        self.targets_file.close()
+
+    def add_library(self, name, srcs, deps=None, headers=None):
+        if headers is None:
+            headers = "AutoHeaders.RECURSIVE_GLOB"
+        self.targets_file.write(targets_cfg.library_template % (
+            name,
+            pretty_list(srcs),
+            headers,
+            pretty_list(deps)))
         self.total_lib = self.total_lib + 1
 
-    def add_rocksdb_library(self, name, srcs, headers=None, external_dependencies=None):
-        headers = "[" + pretty_list(headers) + "]"
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                targets_cfg.rocksdb_library_template.format(
-                    name=name,
-                    srcs=pretty_list(srcs),
-                    headers=headers,
-                    external_dependencies=pretty_list(external_dependencies),
-                ).encode("utf-8")
-            )
-        self.total_lib = self.total_lib + 1
-
-    def add_binary(
-        self,
-        name,
-        srcs,
-        deps=None,
-        extra_preprocessor_flags=None,
-        extra_bench_libs=False,
-    ):
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                targets_cfg.binary_template.format(
-                    name=name,
-                    srcs=pretty_list(srcs),
-                    deps=pretty_list(deps),
-                    extra_preprocessor_flags=pretty_list(extra_preprocessor_flags),
-                    extra_bench_libs=extra_bench_libs,
-                ).encode("utf-8")
-            )
+    def add_binary(self, name, srcs, deps=None):
+        self.targets_file.write(targets_cfg.binary_template % (
+            name,
+            pretty_list(srcs),
+            pretty_list(deps)))
         self.total_bin = self.total_bin + 1
 
-    def add_c_test(self):
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                b"""
-add_c_test_wrapper()
-"""
-            )
+    def register_test(self, test_name, src, is_parallel):
+        exec_mode = "serial"
+        if is_parallel:
+            exec_mode = "parallel"
+        self.tests_cfg += targets_cfg.test_cfg_template % (
+            test_name,
+            str(src),
+            str(exec_mode))
 
-    def add_test_header(self):
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                b"""
-        # Generate a test rule for each entry in ROCKS_TESTS
-        # Do not build the tests in opt mode, since SyncPoint and other test code
-        # will not be included.
-"""
-            )
-
-    def add_fancy_bench_config(
-        self,
-        name,
-        bench_config,
-        slow,
-        expected_runtime,
-        sl_iterations,
-        regression_threshold,
-    ):
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                targets_cfg.fancy_bench_template.format(
-                    name=name,
-                    bench_config=pprint.pformat(bench_config),
-                    slow=slow,
-                    expected_runtime=expected_runtime,
-                    sl_iterations=sl_iterations,
-                    regression_threshold=regression_threshold,
-                ).encode("utf-8")
-            )
-
-    def register_test(self, test_name, src, deps, extra_compiler_flags):
-        with open(self.path, "ab") as targets_file:
-            targets_file.write(
-                targets_cfg.unittests_template.format(
-                    test_name=test_name,
-                    test_cc=str(src),
-                    deps=deps,
-                    extra_compiler_flags=extra_compiler_flags,
-                ).encode("utf-8")
-            )
         self.total_test = self.total_test + 1
 
-    def export_file(self, name):
-        with open(self.path, "a") as targets_file:
-            targets_file.write(
-                targets_cfg.export_file_template.format(name=name)
-            )
+    def flush_tests(self):
+        self.targets_file.write(targets_cfg.unittests_template % self.tests_cfg)
+        self.tests_cfg = ""

buckifier/targets_cfg.py

@@ -1,50 +1,149 @@
-#  Copyright (c) Meta Platforms, Inc. and affiliates.
-#  This source code is licensed under both the GPLv2 (found in the COPYING file in the root directory)
-#  and the Apache 2.0 License (found in the LICENSE.Apache file in the root directory).
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+rocksdb_target_header = """REPO_PATH = package_name() + "/"
 
-rocksdb_target_header_template = """# This file \100generated by:
-#$ python3 buckifier/buckify_rocksdb.py{extra_argv}
-# --> DO NOT EDIT MANUALLY <--
-# This file is a Meta-specific integration for buck builds, so can
-# only be validated by Meta employees.
-load("//rocks/buckifier:defs.bzl", "cpp_library_wrapper","rocks_cpp_library_wrapper","cpp_binary_wrapper","cpp_unittest_wrapper","fancy_bench_wrapper","add_c_test_wrapper")
-load("@fbcode_macros//build_defs:export_files.bzl", "export_file")
+BUCK_BINS = "buck-out/gen/" + REPO_PATH
 
+TEST_RUNNER = REPO_PATH + "buckifier/rocks_test_runner.sh"
+
+rocksdb_compiler_flags = [
+    "-fno-builtin-memcmp",
+    "-DROCKSDB_PLATFORM_POSIX",
+    "-DROCKSDB_LIB_IO_POSIX",
+    "-DROCKSDB_FALLOCATE_PRESENT",
+    "-DROCKSDB_MALLOC_USABLE_SIZE",
+    "-DROCKSDB_RANGESYNC_PRESENT",
+    "-DROCKSDB_SCHED_GETCPU_PRESENT",
+    "-DROCKSDB_SUPPORT_THREAD_LOCAL",
+    "-DOS_LINUX",
+    # Flags to enable libs we include
+    "-DSNAPPY",
+    "-DZLIB",
+    "-DBZIP2",
+    "-DLZ4",
+    "-DZSTD",
+    "-DGFLAGS=gflags",
+    "-DNUMA",
+    "-DTBB",
+    # Needed to compile in fbcode
+    "-Wno-expansion-to-defined",
+    # Added missing flags from output of build_detect_platform
+    "-DROCKSDB_PTHREAD_ADAPTIVE_MUTEX",
+    "-DROCKSDB_BACKTRACE",
+    "-Wshorten-64-to-32",
+]
+
+rocksdb_external_deps = [
+    ("bzip2", None, "bz2"),
+    ("snappy", None, "snappy"),
+    ("zlib", None, "z"),
+    ("gflags", None, "gflags"),
+    ("lz4", None, "lz4"),
+    ("zstd", None),
+    ("tbb", None),
+    ("numa", None, "numa"),
+    ("googletest", None, "gtest"),
+]
+
+rocksdb_preprocessor_flags = [
+    # Directories with files for #include
+    "-I" + REPO_PATH + "include/",
+    "-I" + REPO_PATH,
+]
+
+rocksdb_arch_preprocessor_flags = {
+    "x86_64": [
+        "-DHAVE_SSE42",
+        "-DHAVE_PCLMUL",
+    ],
+}
+
+build_mode = read_config("fbcode", "build_mode")
+
+is_opt_mode = build_mode.startswith("opt")
+
+# -DNDEBUG is added by default in opt mode in fbcode. But adding it twice
+# doesn't harm and avoid forgetting to add it.
+if is_opt_mode:
+    rocksdb_compiler_flags.append("-DNDEBUG")
 """
 
 
 library_template = """
-cpp_library_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], headers={headers}, link_whole={link_whole}, extra_test_libs={extra_test_libs})
+cpp_library(
+    name = "%s",
+    srcs = [%s],
+    headers = %s,
+    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+    compiler_flags = rocksdb_compiler_flags,
+    preprocessor_flags = rocksdb_preprocessor_flags,
+    deps = [%s],
+    external_deps = rocksdb_external_deps,
+)
 """
 
-rocksdb_library_template = """
-rocks_cpp_library_wrapper(name="{name}", srcs=[{srcs}], headers={headers})
-
-"""
-
-
 binary_template = """
-cpp_binary_wrapper(name="{name}", srcs=[{srcs}], deps=[{deps}], extra_preprocessor_flags=[{extra_preprocessor_flags}], extra_bench_libs={extra_bench_libs})
+cpp_binary(
+    name = "%s",
+    srcs = [%s],
+    arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+    compiler_flags = rocksdb_compiler_flags,
+    preprocessor_flags = rocksdb_preprocessor_flags,
+    deps = [%s],
+    external_deps = rocksdb_external_deps,
+)
+"""
+
+test_cfg_template = """    [
+        "%s",
+        "%s",
+        "%s",
+    ],
 """
 
 unittests_template = """
-cpp_unittest_wrapper(name="{test_name}",
-            srcs=["{test_cc}"],
-            deps={deps},
-            extra_compiler_flags={extra_compiler_flags})
+# [test_name, test_src, test_type]
+ROCKS_TESTS = [
+%s]
 
-"""
-
-fancy_bench_template = """
-fancy_bench_wrapper(suite_name="{name}", binary_to_bench_to_metric_list_map={bench_config}, slow={slow}, expected_runtime={expected_runtime}, sl_iterations={sl_iterations}, regression_threshold={regression_threshold})
-
-"""
-
-export_file_template = """
-export_file(name = "{name}")
-"""
-
-
-oncall_template = """
-oncall("{name}")
+# Generate a test rule for each entry in ROCKS_TESTS
+# Do not build the tests in opt mode, since SyncPoint and other test code
+# will not be included.
+if not is_opt_mode:
+    for test_cfg in ROCKS_TESTS:
+        test_name = test_cfg[0]
+        test_cc = test_cfg[1]
+        ttype = "gtest" if test_cfg[2] == "parallel" else "simple"
+        test_bin = test_name + "_bin"
+
+        cpp_binary (
+          name = test_bin,
+          srcs = [test_cc],
+          deps = [":rocksdb_test_lib"],
+          preprocessor_flags = rocksdb_preprocessor_flags,
+          arch_preprocessor_flags = rocksdb_arch_preprocessor_flags,
+          compiler_flags = rocksdb_compiler_flags,
+          external_deps = rocksdb_external_deps,
+        )
+
+        custom_unittest(
+          name = test_name,
+          type = ttype,
+          deps = [":" + test_bin],
+          command = [TEST_RUNNER, BUCK_BINS + test_bin]
+        )
+
+custom_unittest(
+    name = "make_rocksdbjavastatic",
+    command = ["internal_repo_rocksdb/make_rocksdbjavastatic.sh"],
+    type = "simple",
+)
+
+custom_unittest(
+    name = "make_rocksdb_lite_release",
+    command = ["internal_repo_rocksdb/make_rocksdb_lite_release.sh"],
+    type = "simple",
+)
 """

buckifier/util.py

@@ -1,35 +1,26 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 """
 This module keeps commonly used components.
 """
-
-try:
-    from builtins import object
-except ImportError:
-    from __builtin__ import object
-import os
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 import subprocess
-import sys
+import os
 import time
 
-
 class ColorString:
-    """Generate colorful strings on terminal"""
-
-    HEADER = "\033[95m"
-    BLUE = "\033[94m"
-    GREEN = "\033[92m"
-    WARNING = "\033[93m"
-    FAIL = "\033[91m"
-    ENDC = "\033[0m"
+    """ Generate colorful strings on terminal """
+    HEADER = '\033[95m'
+    BLUE = '\033[94m'
+    GREEN = '\033[92m'
+    WARNING = '\033[93m'
+    FAIL = '\033[91m'
+    ENDC = '\033[0m'
 
     @staticmethod
     def _make_color_str(text, color):
-        # In Python2, default encoding for unicode string is ASCII
-        if sys.version_info.major <= 2:
-            return "".join([color, text.encode("utf-8"), ColorString.ENDC])
-        # From Python3, default encoding for unicode string is UTF-8
-        return "".join([color, text, ColorString.ENDC])
+        return "".join([color, text.encode('utf-8'), ColorString.ENDC])
 
     @staticmethod
     def ok(text):
@@ -65,38 +56,37 @@ class ColorString:
 
 
 def run_shell_command(shell_cmd, cmd_dir=None):
-    """Run a single shell command.
-    @returns a tuple of shell command return code, stdout, stderr"""
+    """ Run a single shell command.
+        @returns a tuple of shell command return code, stdout, stderr """
 
     if cmd_dir is not None and not os.path.exists(cmd_dir):
         run_shell_command("mkdir -p %s" % cmd_dir)
 
     start = time.time()
     print("\t>>> Running: " + shell_cmd)
-    p = subprocess.Popen(  # noqa
-        shell_cmd,
-        shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        cwd=cmd_dir,
-    )
+    p = subprocess.Popen(shell_cmd,
+                         shell=True,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE,
+                         cwd=cmd_dir)
     stdout, stderr = p.communicate()
     end = time.time()
 
     # Report time if we spent more than 5 minutes executing a command
     execution_time = end - start
     if execution_time > (60 * 5):
-        mins = execution_time / 60
-        secs = execution_time % 60
+        mins = (execution_time / 60)
+        secs = (execution_time % 60)
         print("\t>time spent: %d minutes %d seconds" % (mins, secs))
 
+
     return p.returncode, stdout, stderr
 
 
 def run_shell_commands(shell_cmds, cmd_dir=None, verbose=False):
-    """Execute a sequence of shell commands, which is equivalent to
-    running `cmd1 && cmd2 && cmd3`
-    @returns boolean indication if all commands succeeds.
+    """ Execute a sequence of shell commands, which is equivalent to
+        running `cmd1 && cmd2 && cmd3`
+        @returns boolean indication if all commands succeeds.
     """
 
     if cmd_dir:

build_tools/RocksDBCommonHelper.php

@@ -0,0 +1,377 @@
+<?php
+// Copyright 2004-present Facebook. All Rights Reserved.
+//  This source code is licensed under both the GPLv2 (found in the
+//  COPYING file in the root directory) and Apache 2.0 License
+//  (found in the LICENSE.Apache file in the root directory).
+
+// Name of the environment variables which need to be set by the entity which
+// triggers continuous runs so that code at the end of the file gets executed
+// and Sandcastle run starts.
+define("ENV_POST_RECEIVE_HOOK", "POST_RECEIVE_HOOK");
+define("ENV_HTTPS_APP_VALUE", "HTTPS_APP_VALUE");
+define("ENV_HTTPS_TOKEN_VALUE", "HTTPS_TOKEN_VALUE");
+
+define("PRIMARY_TOKEN_FILE", '/home/krad/.sandcastle');
+define("CONT_RUN_ALIAS", "leveldb");
+
+//////////////////////////////////////////////////////////////////////
+/*  Run tests in sandcastle */
+function postURL($diffID, $url) {
+  assert(strlen($diffID) > 0);
+  assert(is_numeric($diffID));
+  assert(strlen($url) > 0);
+
+  $cmd_args = array(
+    'diff_id' => (int)$diffID,
+    'name' => sprintf(
+      'click here for sandcastle tests for D%d',
+      (int)$diffID
+    ),
+    'link' => $url
+  );
+  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
+         . ' | arc call-conduit differential.updateunitresults';
+
+  shell_exec($cmd);
+}
+
+function buildUpdateTestStatusCmd($diffID, $test, $status) {
+  assert(strlen($diffID) > 0);
+  assert(is_numeric($diffID));
+  assert(strlen($test) > 0);
+  assert(strlen($status) > 0);
+
+  $cmd_args = array(
+    'diff_id' => (int)$diffID,
+    'name' => $test,
+    'result' => $status
+  );
+
+  $cmd = 'echo ' . escapeshellarg(json_encode($cmd_args))
+         . ' | arc call-conduit differential.updateunitresults';
+
+  return $cmd;
+}
+
+function updateTestStatus($diffID, $test) {
+  assert(strlen($diffID) > 0);
+  assert(is_numeric($diffID));
+  assert(strlen($test) > 0);
+
+  shell_exec(buildUpdateTestStatusCmd($diffID, $test, "waiting"));
+}
+
+function getSteps($applyDiff, $diffID, $username, $test) {
+  assert(strlen($username) > 0);
+  assert(strlen($test) > 0);
+
+  if ($applyDiff) {
+    assert(strlen($diffID) > 0);
+    assert(is_numeric($diffID));
+
+    $arcrc_content = (PHP_OS == "Darwin" ?
+        exec("cat ~/.arcrc | gzip -f | base64") :
+            exec("cat ~/.arcrc | gzip -f | base64 -w0"));
+    assert(strlen($arcrc_content) > 0);
+
+    // Sandcastle machines don't have arc setup. We copy the user certificate
+    // and authenticate using that in Sandcastle.
+    $setup = array(
+      "name" => "Setup arcrc",
+      "shell" => "echo " . escapeshellarg($arcrc_content) . " | base64 --decode"
+                 . " | gzip -d > ~/.arcrc",
+      "user" => "root"
+    );
+
+    // arc demands certain permission on its config.
+    // also fix the sticky bit issue in sandcastle
+    $fix_permission = array(
+      "name" => "Fix environment",
+      "shell" => "chmod 600 ~/.arcrc && chmod +t /dev/shm",
+      "user" => "root"
+    );
+
+    // Construct the steps in the order of execution.
+    $steps[] = $setup;
+    $steps[] = $fix_permission;
+  }
+
+  // fbcode is a sub-repo. We cannot patch until we add it to ignore otherwise
+  // Git thinks it is an uncommited change.
+  $fix_git_ignore = array(
+    "name" => "Fix git ignore",
+    "shell" => "echo fbcode >> .git/info/exclude",
+    "user" => "root"
+  );
+
+  // This fixes "FATAL: ThreadSanitizer can not mmap the shadow memory"
+  // Source:
+  // https://github.com/google/sanitizers/wiki/ThreadSanitizerCppManual#FAQ
+  $fix_kernel_issue = array(
+    "name" => "Fix kernel issue with tsan",
+    "shell" => "echo 2 >/proc/sys/kernel/randomize_va_space",
+    "user" => "root"
+  );
+
+  $steps[] = $fix_git_ignore;
+  $steps[] = $fix_kernel_issue;
+
+  // This will be the command used to execute particular type of tests.
+  $cmd = "";
+
+  if ($applyDiff) {
+    // Patch the code (keep your fingures crossed).
+    $patch = array(
+      "name" => "Patch " . $diffID,
+      "shell" => "arc --arcrc-file ~/.arcrc "
+                  . "patch --nocommit --diff " . escapeshellarg($diffID),
+      "user" => "root"
+    );
+
+    $steps[] = $patch;
+
+    updateTestStatus($diffID, $test);
+    $cmd = buildUpdateTestStatusCmd($diffID, $test, "running") . "; ";
+  }
+
+  // Run the actual command.
+  $cmd = $cmd . "J=$(nproc) ./build_tools/precommit_checker.py " .
+           escapeshellarg($test) . "; exit_code=$?; ";
+
+  if ($applyDiff) {
+    $cmd = $cmd . "([[ \$exit_code -eq 0 ]] &&"
+                . buildUpdateTestStatusCmd($diffID, $test, "pass") . ")"
+                . "||" . buildUpdateTestStatusCmd($diffID, $test, "fail")
+                . "; ";
+  }
+
+  // shell command to sort the tests based on exit code and print
+  // the output of the log files.
+  $cat_sorted_logs = "
+    while read code log_file;
+      do echo \"################ cat \$log_file [exit_code : \$code] ################\";
+      cat \$log_file;
+    done < <(tail -n +2 LOG | sort -k7,7n -k4,4gr | awk '{print \$7,\$NF}')";
+
+  // Shell command to cat all log files
+  $cat_all_logs = "for f in `ls t/!(run-*)`; do echo \$f;cat \$f; done";
+
+  // If LOG file exist use it to cat log files sorted by exit code, otherwise
+  // cat everything
+  $logs_cmd = "if [ -f LOG ]; then {$cat_sorted_logs}; else {$cat_all_logs}; fi";
+
+  $cmd = $cmd . " cat /tmp/precommit-check.log"
+              . "; shopt -s extglob; {$logs_cmd}"
+              . "; shopt -u extglob; [[ \$exit_code -eq 0 ]]";
+  assert(strlen($cmd) > 0);
+
+  $run_test = array(
+    "name" => "Run " . $test,
+    "shell" => $cmd,
+    "user" => "root",
+    "parser" => "python build_tools/error_filter.py " . escapeshellarg($test),
+  );
+
+  $steps[] = $run_test;
+
+  if ($applyDiff) {
+    // Clean up the user arc config we are using.
+    $cleanup = array(
+      "name" => "Arc cleanup",
+      "shell" => "rm -f ~/.arcrc",
+      "user" => "root"
+    );
+
+    $steps[] = $cleanup;
+  }
+
+  assert(count($steps) > 0);
+  return $steps;
+}
+
+function getSandcastleConfig() {
+  $sandcastle_config = array();
+
+  $cwd = getcwd();
+  $cwd_token_file = "{$cwd}/.sandcastle";
+  // This is a case when we're executed from a continuous run. Fetch the values
+  // from the environment.
+  if (getenv(ENV_POST_RECEIVE_HOOK)) {
+    $sandcastle_config[0] = getenv(ENV_HTTPS_APP_VALUE);
+    $sandcastle_config[1] = getenv(ENV_HTTPS_TOKEN_VALUE);
+  } else {
+    // This is a typical `[p]arc diff` case. Fetch the values from the specific
+    // configuration files.
+    for ($i = 0; $i < 50; $i++) {
+      if (file_exists(PRIMARY_TOKEN_FILE) ||
+          file_exists($cwd_token_file)) {
+        break;
+      }
+      // If we failed to fetch the tokens, sleep for 0.2 second and try again
+      usleep(200000);
+    }
+    assert(file_exists(PRIMARY_TOKEN_FILE) ||
+           file_exists($cwd_token_file));
+
+    // Try the primary location first, followed by a secondary.
+    if (file_exists(PRIMARY_TOKEN_FILE)) {
+      $cmd = 'cat ' . PRIMARY_TOKEN_FILE;
+    } else {
+      $cmd = 'cat ' . escapeshellarg($cwd_token_file);
+    }
+
+    assert(strlen($cmd) > 0);
+    $sandcastle_config = explode(':', rtrim(shell_exec($cmd)));
+  }
+
+  // In this case be very explicit about the implications.
+  if (count($sandcastle_config) != 2) {
+    echo "Sandcastle configuration files don't contain valid information " .
+         "or the necessary environment variables aren't defined. Unable " .
+         "to validate the code changes.";
+    exit(1);
+  }
+
+  assert(strlen($sandcastle_config[0]) > 0);
+  assert(strlen($sandcastle_config[1]) > 0);
+  assert(count($sandcastle_config) > 0);
+
+  return $sandcastle_config;
+}
+
+// This function can be called either from `[p]arc diff` command or during
+// the Git post-receive hook.
+ function startTestsInSandcastle($applyDiff, $workflow, $diffID) {
+  // Default options don't terminate on failure, but that's what we want. In
+  // the current case we use assertions intentionally as "terminate on failure
+  // invariants".
+  assert_options(ASSERT_BAIL, true);
+
+  // In case of a diff we'll send notificatios to the author. Else it'll go to
+  // the entire team because failures indicate that build quality has regressed.
+  $username = $applyDiff ? exec("whoami") : CONT_RUN_ALIAS;
+  assert(strlen($username) > 0);
+
+  if ($applyDiff) {
+    assert($workflow);
+    assert(strlen($diffID) > 0);
+    assert(is_numeric($diffID));
+  }
+
+  // List of tests we want to run in Sandcastle.
+  $tests = array("unit", "unit_non_shm", "unit_481", "clang_unit", "tsan",
+                 "asan", "lite_test", "valgrind", "release", "release_481",
+                 "clang_release", "clang_analyze", "code_cov",
+                 "java_build", "no_compression", "unity", "ubsan");
+
+  $send_email_template = array(
+    'type' => 'email',
+    'triggers' => array('fail'),
+    'emails' => array($username . '@fb.com'),
+  );
+
+  // Construct a job definition for each test and add it to the master plan.
+  foreach ($tests as $test) {
+    $stepName = "RocksDB diff " . $diffID . " test " . $test;
+
+    if (!$applyDiff) {
+      $stepName = "RocksDB continuous integration test " . $test;
+    }
+
+    $arg[] = array(
+      "name" => $stepName,
+      "report" => array($send_email_template),
+      "steps" => getSteps($applyDiff, $diffID, $username, $test)
+    );
+  }
+
+  // We cannot submit the parallel execution master plan to Sandcastle and
+  // need supply the job plan as a determinator. So we construct a small job
+  // that will spit out the master job plan which Sandcastle will parse and
+  // execute. Why compress the job definitions? Otherwise we run over the max
+  // string size.
+  $cmd = "echo " . base64_encode(json_encode($arg))
+         . (PHP_OS == "Darwin" ?
+             " | gzip -f | base64" :
+                 " | gzip -f | base64 -w0");
+  assert(strlen($cmd) > 0);
+
+  $arg_encoded = shell_exec($cmd);
+  assert(strlen($arg_encoded) > 0);
+
+  $runName = "Run diff " . $diffID . "for user " . $username;
+
+  if (!$applyDiff) {
+    $runName = "RocksDB continuous integration build and test run";
+  }
+
+  $command = array(
+    "name" => $runName,
+    "steps" => array()
+  );
+
+  $command["steps"][] = array(
+    "name" => "Generate determinator",
+    "shell" => "echo " . $arg_encoded . " | base64 --decode | gzip -d"
+               . " | base64 --decode",
+    "determinator" => true,
+    "user" => "root"
+  );
+
+  // Submit to Sandcastle.
+  $url = 'https://interngraph.intern.facebook.com/sandcastle/create';
+
+  $job = array(
+    'command' => 'SandcastleUniversalCommand',
+    'args' => $command,
+    'capabilities' => array(
+      'vcs' => 'rocksdb-int-git',
+      'type' => 'lego',
+    ),
+    'hash' => 'origin/master',
+    'user' => $username,
+    'alias' => 'rocksdb-precommit',
+    'tags' => array('rocksdb'),
+    'description' => 'Rocksdb precommit job',
+  );
+
+  // Fetch the configuration necessary to submit a successful HTTPS request.
+  $sandcastle_config = getSandcastleConfig();
+
+  $app = $sandcastle_config[0];
+  $token = $sandcastle_config[1];
+
+  $cmd = 'curl -s -k '
+          . ' -F app=' . escapeshellarg($app)
+          . ' -F token=' . escapeshellarg($token)
+          . ' -F job=' . escapeshellarg(json_encode($job))
+          .' ' . escapeshellarg($url);
+
+  $output = shell_exec($cmd);
+  assert(strlen($output) > 0);
+
+  // Extract Sandcastle URL from the response.
+  preg_match('/url": "(.+)"/', $output, $sandcastle_url);
+
+  assert(count($sandcastle_url) > 0, "Unable to submit Sandcastle request.");
+  assert(strlen($sandcastle_url[1]) > 0, "Unable to extract Sandcastle URL.");
+
+  if ($applyDiff) {
+    echo "\nSandcastle URL: " . $sandcastle_url[1] . "\n";
+    // Ask Phabricator to display it on the diff UI.
+    postURL($diffID, $sandcastle_url[1]);
+  } else {
+    echo "Continuous integration started Sandcastle tests. You can look at ";
+    echo "the progress at:\n" . $sandcastle_url[1] . "\n";
+  }
+}
+
+// Continuous run cript will set the environment variable and based on that
+// we'll trigger the execution of tests in Sandcastle. In that case we don't
+// need to apply any diffs and there's no associated workflow either.
+if (getenv(ENV_POST_RECEIVE_HOOK)) {
+  startTestsInSandcastle(
+    false /* $applyDiff */,
+    NULL /* $workflow */,
+    NULL /* $diffID */);
+}

build_tools/amalgamate.py

@@ -1,5 +1,4 @@
 #!/usr/bin/python
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 # amalgamate.py creates an amalgamation from a unity build.
 # It can be run with either Python 2 or 3.
@@ -25,17 +24,17 @@
 #
 # The solution is to move the include out of the #ifdef.
 
+from __future__ import print_function
 
 import argparse
+from os import path
 import re
 import sys
-from os import path
 
 include_re = re.compile('^[ \t]*#include[ \t]+"(.*)"[ \t]*$')
 included = set()
 excluded = set()
 
-
 def find_header(name, abs_path, include_paths):
     samedir = path.join(path.dirname(abs_path), name)
     if path.exists(samedir):
@@ -46,31 +45,17 @@ def find_header(name, abs_path, include_paths):
             return include_path
     return None
 
-
-def expand_include(
-    include_path,
-    f,
-    abs_path,
-    source_out,
-    header_out,
-    include_paths,
-    public_include_paths,
-):
+def expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths):
     if include_path in included:
         return False
 
     included.add(include_path)
     with open(include_path) as f:
-        print(f'#line 1 "{include_path}"', file=source_out)
-        process_file(
-            f, include_path, source_out, header_out, include_paths, public_include_paths
-        )
+        print('#line 1 "{}"'.format(include_path), file=source_out)
+        process_file(f, include_path, source_out, header_out, include_paths, public_include_paths)
     return True
 
-
-def process_file(
-    f, abs_path, source_out, header_out, include_paths, public_include_paths
-):
+def process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths):
     for (line, text) in enumerate(f):
         m = include_re.match(text)
         if m:
@@ -82,15 +67,7 @@ def process_file(
                     source_out.write(text)
                     expanded = False
                 else:
-                    expanded = expand_include(
-                        include_path,
-                        f,
-                        abs_path,
-                        source_out,
-                        header_out,
-                        include_paths,
-                        public_include_paths,
-                    )
+                    expanded = expand_include(include_path, f, abs_path, source_out, header_out, include_paths, public_include_paths)
             else:
                 # now try public headers
                 include_path = find_header(filename, abs_path, public_include_paths)
@@ -100,52 +77,23 @@ def process_file(
                     if include_path in excluded:
                         source_out.write(text)
                     else:
-                        expand_include(
-                            include_path,
-                            f,
-                            abs_path,
-                            header_out,
-                            None,
-                            public_include_paths,
-                            [],
-                        )
+                        expand_include(include_path, f, abs_path, header_out, None, public_include_paths, [])
                 else:
-                    sys.exit(
-                        "unable to find {}, included in {} on line {}".format(
-                            filename, abs_path, line
-                        )
-                    )
+                    sys.exit("unable to find {}, included in {} on line {}".format(filename, abs_path, line))
 
             if expanded:
-                print(f'#line {line + 1} "{abs_path}"', file=source_out)
+                print('#line {} "{}"'.format(line+1, abs_path), file=source_out)
         elif text != "#pragma once\n":
             source_out.write(text)
 
-
 def main():
-    parser = argparse.ArgumentParser(
-        description="Transform a unity build into an amalgamation"
-    )
+    parser = argparse.ArgumentParser(description="Transform a unity build into an amalgamation")
     parser.add_argument("source", help="source file")
-    parser.add_argument(
-        "-I",
-        action="append",
-        dest="include_paths",
-        help="include paths for private headers",
-    )
-    parser.add_argument(
-        "-i",
-        action="append",
-        dest="public_include_paths",
-        help="include paths for public headers",
-    )
-    parser.add_argument(
-        "-x", action="append", dest="excluded", help="excluded header files"
-    )
+    parser.add_argument("-I", action="append", dest="include_paths", help="include paths for private headers")
+    parser.add_argument("-i", action="append", dest="public_include_paths", help="include paths for public headers")
+    parser.add_argument("-x", action="append", dest="excluded", help="excluded header files")
     parser.add_argument("-o", dest="source_out", help="output C++ file", required=True)
-    parser.add_argument(
-        "-H", dest="header_out", help="output C++ header file", required=True
-    )
+    parser.add_argument("-H", dest="header_out", help="output C++ header file", required=True)
     args = parser.parse_args()
 
     include_paths = list(map(path.abspath, args.include_paths or []))
@@ -153,15 +101,10 @@ def main():
     excluded.update(map(path.abspath, args.excluded or []))
     filename = args.source
     abs_path = path.abspath(filename)
-    with open(filename) as f, open(args.source_out, "w") as source_out, open(
-        args.header_out, "w"
-    ) as header_out:
-        print(f'#line 1 "{filename}"', file=source_out)
-        print(f'#include "{header_out.name}"', file=source_out)
-        process_file(
-            f, abs_path, source_out, header_out, include_paths, public_include_paths
-        )
-
+    with open(filename) as f, open(args.source_out, 'w') as source_out, open(args.header_out, 'w') as header_out:
+        print('#line 1 "{}"'.format(filename), file=source_out)
+        print('#include "{}"'.format(header_out.name), file=source_out)
+        process_file(f, abs_path, source_out, header_out, include_paths, public_include_paths)
 
 if __name__ == "__main__":
     main()

build_tools/benchmark_log_tool.py

@@ -1,238 +0,0 @@
-#!/usr/bin/env python3
-#  Copyright (c) 2011-present, Facebook, Inc.  All rights reserved.
-#  This source code is licensed under both the GPLv2 (found in the
-#  COPYING file in the root directory) and Apache 2.0 License
-#  (found in the LICENSE.Apache file in the root directory).
-
-"""Access the results of benchmark runs
-Send these results on to OpenSearch graphing service
-"""
-
-import argparse
-import itertools
-import logging
-import os
-import re
-import sys
-
-import requests
-from dateutil import parser
-
-logging.basicConfig(level=logging.DEBUG)
-
-
-class Configuration:
-    opensearch_user = os.environ["ES_USER"]
-    opensearch_pass = os.environ["ES_PASS"]
-
-
-class BenchmarkResultException(Exception):
-    def __init__(self, message, content):
-        super().__init__(self, message)
-        self.content = content
-
-
-class BenchmarkUtils:
-
-    expected_keys = [
-        "ops_sec",
-        "mb_sec",
-        "lsm_sz",
-        "blob_sz",
-        "c_wgb",
-        "w_amp",
-        "c_mbps",
-        "c_wsecs",
-        "c_csecs",
-        "b_rgb",
-        "b_wgb",
-        "usec_op",
-        "p50",
-        "p99",
-        "p99.9",
-        "p99.99",
-        "pmax",
-        "uptime",
-        "stall%",
-        "Nstall",
-        "u_cpu",
-        "s_cpu",
-        "rss",
-        "test",
-        "date",
-        "version",
-        "job_id",
-    ]
-
-    def sanity_check(row):
-        if "test" not in row:
-            logging.debug(f"not 'test' in row: {row}")
-            return False
-        if row["test"] == "":
-            logging.debug(f"row['test'] == '': {row}")
-            return False
-        if "date" not in row:
-            logging.debug(f"not 'date' in row: {row}")
-            return False
-        if "ops_sec" not in row:
-            logging.debug(f"not 'ops_sec' in row: {row}")
-            return False
-        try:
-            _ = int(row["ops_sec"])
-        except (ValueError, TypeError):
-            logging.debug(f"int(row['ops_sec']): {row}")
-            return False
-        try:
-            (_, _) = parser.parse(row["date"], fuzzy_with_tokens=True)
-        except (parser.ParserError):
-            logging.error(
-                f"parser.parse((row['date']): not a valid format for date in row: {row}"
-            )
-            return False
-        return True
-
-    def conform_opensearch(row):
-        (dt, _) = parser.parse(row["date"], fuzzy_with_tokens=True)
-        # create a test_date field, which was previously what was expected
-        # repair the date field, which has what can be a WRONG ISO FORMAT, (no leading 0 on single-digit day-of-month)
-        # e.g. 2022-07-1T00:14:55 should be 2022-07-01T00:14:55
-        row["test_date"] = dt.isoformat()
-        row["date"] = dt.isoformat()
-        return {key.replace(".", "_"): value for key, value in row.items()}
-
-
-class ResultParser:
-    def __init__(self, field=r"(\w|[+-:.%])+", intrafield=r"(\s)+", separator="\t"):
-        self.field = re.compile(field)
-        self.intra = re.compile(intrafield)
-        self.sep = re.compile(separator)
-
-    def ignore(self, l_in: str):
-        if len(l_in) == 0:
-            return True
-        if l_in[0:1] == "#":
-            return True
-        return False
-
-    def line(self, line_in: str):
-        """Parse a line into items
-        Being clever about separators
-        """
-        line = line_in
-        row = []
-        while line != "":
-            match_item = self.field.match(line)
-            if match_item:
-                item = match_item.group(0)
-                row.append(item)
-                line = line[len(item) :]
-            else:
-                match_intra = self.intra.match(line)
-                if match_intra:
-                    intra = match_intra.group(0)
-                    # Count the separators
-                    # If there are >1 then generate extra blank fields
-                    # White space with no true separators fakes up a single separator
-                    tabbed = self.sep.split(intra)
-                    sep_count = len(tabbed) - 1
-                    if sep_count == 0:
-                        sep_count = 1
-                    for _ in range(sep_count - 1):
-                        row.append("")
-                    line = line[len(intra) :]
-                else:
-                    raise BenchmarkResultException(
-                        "Invalid TSV line", f"{line_in} at {line}"
-                    )
-        return row
-
-    def parse(self, lines):
-        """Parse something that iterates lines"""
-        rows = [self.line(line) for line in lines if not self.ignore(line)]
-        header = rows[0]
-        width = len(header)
-        records = [
-            {k: v for (k, v) in itertools.zip_longest(header, row[:width])}
-            for row in rows[1:]
-        ]
-        return records
-
-
-def load_report_from_tsv(filename: str):
-    file = open(filename)
-    contents = file.readlines()
-    file.close()
-    parser = ResultParser()
-    report = parser.parse(contents)
-    logging.debug(f"Loaded TSV Report: {report}")
-    return report
-
-
-def push_report_to_opensearch(report, esdocument):
-    sanitized = [
-        BenchmarkUtils.conform_opensearch(row)
-        for row in report
-        if BenchmarkUtils.sanity_check(row)
-    ]
-    logging.debug(
-        f"upload {len(sanitized)} sane of {len(report)} benchmarks to opensearch"
-    )
-    for single_benchmark in sanitized:
-        logging.debug(f"upload benchmark: {single_benchmark}")
-        response = requests.post(
-            esdocument,
-            json=single_benchmark,
-            auth=(os.environ["ES_USER"], os.environ["ES_PASS"]),
-        )
-        logging.debug(
-            f"Sent to OpenSearch, status: {response.status_code}, result: {response.text}"
-        )
-        response.raise_for_status()
-
-
-def push_report_to_null(report):
-
-    for row in report:
-        if BenchmarkUtils.sanity_check(row):
-            logging.debug(f"row {row}")
-            conformed = BenchmarkUtils.conform_opensearch(row)
-            logging.debug(f"conformed row {conformed}")
-
-
-def main():
-    """Tool for fetching, parsing and uploading benchmark results to OpenSearch / ElasticSearch
-    This tool will
-
-    (1) Open a local tsv benchmark report file
-    (2) Upload to OpenSearch document, via https/JSON
-    """
-
-    parser = argparse.ArgumentParser(description="CircleCI benchmark scraper.")
-
-    # --tsvfile is the name of the file to read results from
-    # --esdocument is the ElasticSearch document to push these results into
-    #
-    parser.add_argument(
-        "--tsvfile",
-        default="build_tools/circle_api_scraper_input.txt",
-        help="File from which to read tsv report",
-    )
-    parser.add_argument(
-        "--esdocument",
-        help="ElasticSearch/OpenSearch document URL to upload report into",
-    )
-    parser.add_argument(
-        "--upload", choices=["opensearch", "none"], default="opensearch"
-    )
-
-    args = parser.parse_args()
-    logging.debug(f"Arguments: {args}")
-    reports = load_report_from_tsv(args.tsvfile)
-    if args.upload == "opensearch":
-        push_report_to_opensearch(reports, args.esdocument)
-    else:
-        push_report_to_null(reports)
-
-
-if __name__ == "__main__":
-    sys.exit(main())

build_tools/build_detect_platform

@@ -9,7 +9,6 @@
 #   PLATFORM_LDFLAGS            Linker flags
 #   JAVA_LDFLAGS                Linker flags for RocksDBJava
 #   JAVA_STATIC_LDFLAGS         Linker flags for RocksDBJava static build
-#   JAVAC_ARGS                  Arguments for javac
 #   PLATFORM_SHARED_EXT         Extension for shared libraries
 #   PLATFORM_SHARED_LDFLAGS     Flags for building shared library
 #   PLATFORM_SHARED_CFLAGS      Flags for compiling objects for shared library
@@ -28,7 +27,6 @@
 #       -DZSTD                      if the ZSTD library is present
 #       -DNUMA                      if the NUMA library is present
 #       -DTBB                       if the TBB library is present
-#       -DMEMKIND                   if the memkind library is present
 #
 # Using gflags in rocksdb:
 # Our project depends on gflags, which requires users to take some extra steps
@@ -45,33 +43,22 @@ if test -z "$OUTPUT"; then
   exit 1
 fi
 
-# we depend on C++20, but should be compatible with newer standards
-if [ "$ROCKSDB_CXX_STANDARD" ]; then
-  PLATFORM_CXXFLAGS="-std=$ROCKSDB_CXX_STANDARD"
-else
-  PLATFORM_CXXFLAGS="-std=c++20"
-fi
-
+# we depend on C++11
+PLATFORM_CXXFLAGS="-std=c++11"
 # we currently depend on POSIX platform
 COMMON_FLAGS="-DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX"
 
-# Default to fbcode gcc on Meta internal machines
-IS_META_HOST="$(hostname | grep -E '(facebook|meta).com|fbinfra.net')"
-if [ -z "$ROCKSDB_NO_FBCODE" -a "$IS_META_HOST" ]; then
-  if [ -d /mnt/gvfs/third-party ]; then
-    echo "NOTE: Using fbcode build" >&2
+# Default to fbcode gcc on internal fb machines
+if [ -z "$ROCKSDB_NO_FBCODE" -a -d /mnt/gvfs/third-party ]; then
     FBCODE_BUILD="true"
-    # If we're compiling with TSAN or shared lib, we need pic build
+    # If we're compiling with TSAN we need pic build
     PIC_BUILD=$COMPILE_WITH_TSAN
-    if [ "$LIB_MODE" == "shared" ]; then
-      PIC_BUILD=1
+    if [ -z "$ROCKSDB_FBCODE_BUILD_WITH_481" ]; then
+      source "$PWD/build_tools/fbcode_config.sh"
+    else
+      # we need this to build with MySQL. Don't use for other purposes.
+      source "$PWD/build_tools/fbcode_config4.8.1.sh"
     fi
-    source "$PWD/build_tools/fbcode_config_platform010.sh"
-  else
-    echo "************************************************************************" >&2
-    echo "WARNING: -d /mnt/gvfs/third-party failed; no fbcode build" >&2
-    echo "************************************************************************" >&2
-  fi
 fi
 
 # Delete existing output, if it exists
@@ -79,9 +66,7 @@ rm -f "$OUTPUT"
 touch "$OUTPUT"
 
 if test -z "$CC"; then
-    if [ "$USE_CLANG" -a -x "$(command -v clang)" ]; then
-        CC=clang
-    elif [ -x "$(command -v cc)" ]; then
+    if [ -x "$(command -v cc)" ]; then
         CC=cc
     elif [ -x "$(command -v clang)" ]; then
         CC=clang
@@ -91,9 +76,7 @@ if test -z "$CC"; then
 fi
 
 if test -z "$CXX"; then
-    if [ "$USE_CLANG" -a -x "$(command -v clang++)" ]; then
-        CXX=clang++
-    elif [ -x "$(command -v g++)" ]; then
+    if [ -x "$(command -v g++)" ]; then
         CXX=g++
     elif [ -x "$(command -v clang++)" ]; then
         CXX=clang++
@@ -102,18 +85,6 @@ if test -z "$CXX"; then
     fi
 fi
 
-if test -z "$AR"; then
-    if [ "$USE_CLANG" -a -x "$(command -v llvm-ar)" ]; then
-        AR=llvm-ar
-    elif [ -x "$(command -v gcc-ar)" ]; then
-        AR=gcc-ar
-    elif [ -x "$(command -v llvm-ar)" ]; then
-        AR=llvm-ar
-    else
-        AR=ar
-    fi
-fi
-
 # Detect OS
 if test -z "$TARGET_OS"; then
     TARGET_OS=`uname -s`
@@ -148,24 +119,6 @@ PLATFORM_SHARED_LDFLAGS="-Wl,--no-as-needed -shared -Wl,-soname -Wl,"
 PLATFORM_SHARED_CFLAGS="-fPIC"
 PLATFORM_SHARED_VERSIONED=true
 
-# Prefer lld linker when available on Linux. lld is typically 5-10x faster
-# than the default ld.bfd for large C++ projects. macOS uses ld64 (or
-# ld-prime) which is already fast, so we skip lld detection there.
-# Set ROCKSDB_NO_FAST_LINKER=1 to disable this auto-detection.
-if [ -z "$ROCKSDB_NO_FAST_LINKER" ] && [ "$TARGET_OS" = "Linux" ]; then
-    if $CXX -fuse-ld=lld -L/usr/local/lib -x c++ - -o /dev/null 2>/dev/null <<EOF
-int main() { return 0; }
-EOF
-    then
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -fuse-ld=lld"
-        # Ensure lld can find libraries in /usr/local/lib (lld does not
-        # search there by default, unlike ld.bfd)
-        if [ -d /usr/local/lib ]; then
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -L/usr/local/lib"
-        fi
-    fi
-fi
-
 # generic port files (working on all platform by #ifdef) go directly in /port
 GENERIC_PORT_FILES=`cd "$ROCKSDB_ROOT"; find port -name '*.cc' | tr "\n" " "`
 
@@ -180,7 +133,7 @@ case "$TARGET_OS" in
         ;;
     IOS)
         PLATFORM=IOS
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE "
+        COMMON_FLAGS="$COMMON_FLAGS -DOS_MACOSX -DIOS_CROSS_COMPILE -DROCKSDB_LITE"
         PLATFORM_SHARED_EXT=dylib
         PLATFORM_SHARED_LDFLAGS="-dynamiclib -install_name "
         CROSS_COMPILE=true
@@ -194,7 +147,7 @@ case "$TARGET_OS" in
         else
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
         fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt -ldl"
+        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
         # PORT_FILES=port/linux/linux_specific.cc
         ;;
     SunOS)
@@ -217,17 +170,6 @@ case "$TARGET_OS" in
         PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread"
         # PORT_FILES=port/freebsd/freebsd_specific.cc
         ;;
-    GNU/kFreeBSD)
-        PLATFORM=OS_GNU_KFREEBSD
-        COMMON_FLAGS="$COMMON_FLAGS -DOS_GNU_KFREEBSD"
-        if [ -z "$USE_CLANG" ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp"
-        else
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -latomic"
-        fi
-        PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lpthread -lrt"
-        # PORT_FILES=port/gnu_kfreebsd/gnu_kfreebsd_specific.cc
-        ;;
     NetBSD)
         PLATFORM=OS_NETBSD
         COMMON_FLAGS="$COMMON_FLAGS -fno-builtin-memcmp -D_REENTRANT -DOS_NETBSD"
@@ -277,26 +219,20 @@ esac
 PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS ${CXXFLAGS}"
 JAVA_LDFLAGS="$PLATFORM_LDFLAGS"
 JAVA_STATIC_LDFLAGS="$PLATFORM_LDFLAGS"
-JAVAC_ARGS="-source 8"
 
 if [ "$CROSS_COMPILE" = "true" -o "$FBCODE_BUILD" = "true" ]; then
     # Cross-compiling; do not try any compilation tests.
     # Also don't need any compilation tests if compiling on fbcode
-    if [ "$FBCODE_BUILD" = "true" ]; then
-      # Enable backtrace on fbcode since the necessary libraries are present
-      COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
-      FOLLY_DIR="third-party/folly"
-    fi
     true
 else
     if ! test $ROCKSDB_DISABLE_FALLOCATE; then
         # Test whether fallocate is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <fcntl.h>
           #include <linux/falloc.h>
           int main() {
       int fd = open("/dev/null", 0);
-      fallocate(fd, FALLOC_FL_KEEP_SIZE, 0, 1024);
+      fallocate(fd, FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE, 0, 1024);
           }
 EOF
         if [ "$?" = 0 ]; then
@@ -307,7 +243,7 @@ EOF
     if ! test $ROCKSDB_DISABLE_SNAPPY; then
         # Test whether Snappy library is installed
         # http://code.google.com/p/snappy/
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <snappy.h>
           int main() {}
 EOF
@@ -322,40 +258,30 @@ EOF
         # Test whether gflags library is installed
         # http://gflags.github.io/gflags/
         # check if the namespace is gflags
-        if $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null << EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
           #include <gflags/gflags.h>
-          using namespace GFLAGS_NAMESPACE;
           int main() {}
 EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
-          # Hack: don't link extra gflags assuming it comes with folly
-          [ "$USE_FOLLY" ] || PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is gflags
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null << EOF
-            #include <gflags/gflags.h>
-            using namespace gflags;
-            int main() {}
-EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=gflags"
-          # Hack: don't link extra gflags assuming it comes with folly
-          [ "$USE_FOLLY" ] || PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
-        # check if namespace is google
-        elif $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null << EOF
+        if [ "$?" = 0 ]; then
+            COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1"
+            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+        else
+          # check if namespace is google
+          $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null << EOF
             #include <gflags/gflags.h>
             using namespace google;
             int main() {}
 EOF
-        then
-          COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=1 -DGFLAGS_NAMESPACE=google"
-          PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+          if [ "$?" = 0 ]; then
+              COMMON_FLAGS="$COMMON_FLAGS -DGFLAGS=google"
+              PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lgflags"
+          fi
         fi
     fi
 
     if ! test $ROCKSDB_DISABLE_ZLIB; then
         # Test whether zlib library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <zlib.h>
           int main() {}
 EOF
@@ -368,7 +294,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_BZIP; then
         # Test whether bzip library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <bzlib.h>
           int main() {}
 EOF
@@ -381,7 +307,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_LZ4; then
         # Test whether lz4 library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <lz4.h>
           #include <lz4hc.h>
           int main() {}
@@ -394,13 +320,9 @@ EOF
     fi
 
     if ! test $ROCKSDB_DISABLE_ZSTD; then
-        # Test whether zstd library is installed with minimum version
-        # (Keep in sync with compression.h)
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+        # Test whether zstd library is installed
+        $CXX $CFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <zstd.h>
-          #if ZSTD_VERSION_NUMBER < 10400
-          #error "ZSTD support requires version >= 1.4.0 (libzstd-devel)"
-          #endif  // ZSTD_VERSION_NUMBER
           int main() {}
 EOF
         if [ "$?" = 0 ]; then
@@ -412,7 +334,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_NUMA; then
         # Test whether numa is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o -lnuma 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null -lnuma 2>/dev/null  <<EOF
           #include <numa.h>
           #include <numaif.h>
           int main() {}
@@ -426,7 +348,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_TBB; then
         # Test whether tbb is available
-        $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -ltbb 2>/dev/null  <<EOF
+        $CXX $CFLAGS $LDFLAGS -x c++ - -o /dev/null -ltbb 2>/dev/null  <<EOF
           #include <tbb/tbb.h>
           int main() {}
 EOF
@@ -439,35 +361,18 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_JEMALLOC; then
         # Test whether jemalloc is available
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -ljemalloc \
+        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null -ljemalloc \
           2>/dev/null; then
             # This will enable some preprocessor identifiers in the Makefile
             JEMALLOC=1
             # JEMALLOC can be enabled either using the flag (like here) or by
             # providing direct link to the jemalloc library
             WITH_JEMALLOC_FLAG=1
-            # check for JEMALLOC installed with HomeBrew
-            if [ "$PLATFORM" == "OS_MACOSX" ]; then
-                if [ "$TARGET_ARCHITECTURE" = "arm64" ]; then
-                    # on M1 Macs, homebrew installs here instead of /usr/local
-                    JEMALLOC_PREFIX="/opt/homebrew"
-                else
-                    JEMALLOC_PREFIX="/usr/local"
-                fi
-                if hash brew 2>/dev/null && brew ls --versions jemalloc > /dev/null; then
-                    JEMALLOC_VER=$(brew ls --versions jemalloc | tail -n 1 | cut -f 2 -d ' ')
-                    JEMALLOC_INCLUDE="-I${JEMALLOC_PREFIX}/Cellar/jemalloc/${JEMALLOC_VER}/include"
-                    JEMALLOC_LIB="${JEMALLOC_PREFIX}/Cellar/jemalloc/${JEMALLOC_VER}/lib/libjemalloc_pic.a"
-                    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB"
-                    JAVA_LDFLAGS="$JAVA_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB"
-                    JAVA_STATIC_LDFLAGS="$JAVA_STATIC_LDFLAGS -L${JEMALLOC_PREFIX}/lib $JEMALLOC_LIB"
-                fi
-            fi
         fi
     fi
     if ! test $JEMALLOC && ! test $ROCKSDB_DISABLE_TCMALLOC; then
         # jemalloc is not available. Let's try tcmalloc
-        if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o \
+        if echo 'int main() {}' | $CXX $CFLAGS -x c++ - -o /dev/null \
           -ltcmalloc 2>/dev/null; then
             PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -ltcmalloc"
             JAVA_LDFLAGS="$JAVA_LDFLAGS -ltcmalloc"
@@ -476,11 +381,10 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_MALLOC_USABLE_SIZE; then
         # Test whether malloc_usable_size is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <malloc.h>
           int main() {
             size_t res = malloc_usable_size(0);
-            (void)res;
             return 0;
           }
 EOF
@@ -489,29 +393,12 @@ EOF
         fi
     fi
 
-    if ! test $ROCKSDB_DISABLE_MEMKIND; then
-        # Test whether memkind library is installed
-        $CXX $PLATFORM_CXXFLAGS $LDFLAGS -x c++ - -o test.o -lmemkind 2>/dev/null  <<EOF
-          #include <memkind.h>
-          int main() {
-            memkind_malloc(MEMKIND_DAX_KMEM, 1024);
-            return 0;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DMEMKIND"
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lmemkind"
-            JAVA_LDFLAGS="$JAVA_LDFLAGS -lmemkind"
-        fi
-    fi
-
     if ! test $ROCKSDB_DISABLE_PTHREAD_MUTEX_ADAPTIVE_NP; then
         # Test whether PTHREAD_MUTEX_ADAPTIVE_NP mutex type is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <pthread.h>
           int main() {
             int x = PTHREAD_MUTEX_ADAPTIVE_NP;
-            (void)x;
             return 0;
           }
 EOF
@@ -522,8 +409,8 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_BACKTRACE; then
         # Test whether backtrace is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
-          #include <execinfo.h>
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
+          #include <execinfo.h>>
           int main() {
             void* frames[1];
             backtrace_symbols(frames, backtrace(frames, 1));
@@ -534,7 +421,7 @@ EOF
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_BACKTRACE"
         else
             # Test whether execinfo library is installed
-            $CXX $PLATFORM_CXXFLAGS -lexecinfo -x c++ - -o test.o 2>/dev/null  <<EOF
+            $CXX $CFLAGS -lexecinfo -x c++ - -o /dev/null 2>/dev/null  <<EOF
               #include <execinfo.h>
               int main() {
                 void* frames[1];
@@ -551,7 +438,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_PG; then
         # Test if -pg is supported
-        $CXX $PLATFORM_CXXFLAGS -pg -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -pg -x c++ - -o /dev/null 2>/dev/null  <<EOF
           int main() {
             return 0;
           }
@@ -563,7 +450,7 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_SYNC_FILE_RANGE; then
         # Test whether sync_file_range is supported for compatibility with an old glibc
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <fcntl.h>
           int main() {
             int fd = open("/dev/null", 0);
@@ -577,88 +464,23 @@ EOF
 
     if ! test $ROCKSDB_DISABLE_SCHED_GETCPU; then
         # Test whether sched_getcpu is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
+        $CXX $CFLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
           #include <sched.h>
           int main() {
             int cpuid = sched_getcpu();
-            (void)cpuid;
           }
 EOF
         if [ "$?" = 0 ]; then
             COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SCHED_GETCPU_PRESENT"
         fi
     fi
-
-    if ! test $ROCKSDB_DISABLE_AUXV_GETAUXVAL; then
-        # Test whether getauxval is supported
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o 2>/dev/null  <<EOF
-          #include <sys/auxv.h>
-          int main() {
-            uint64_t auxv = getauxval(AT_HWCAP);
-            (void)auxv;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_AUXV_GETAUXVAL_PRESENT"
-        fi
-    fi
-
-    if ! test $ROCKSDB_DISABLE_ALIGNED_NEW; then
-        # Test whether c++17 aligned-new is supported
-        $CXX $PLATFORM_CXXFLAGS -faligned-new -x c++ - -o test.o 2>/dev/null <<EOF
-            struct alignas(1024) t {int a;};
-            int main() {}
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_CXXFLAGS="$PLATFORM_CXXFLAGS -faligned-new -DHAVE_ALIGNED_NEW"
-        fi
-    fi
-    if ! test $ROCKSDB_DISABLE_BENCHMARK; then
-        # Test whether google benchmark is available
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -o /dev/null -lbenchmark -lpthread 2>/dev/null  <<EOF
-          #include <benchmark/benchmark.h>
-          int main() {}
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -lbenchmark"
-        fi
-    fi
-    if test $USE_FOLLY || test $USE_FOLLY_LITE; then
-        # Test whether libfolly library is installed
-        $CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null  <<EOF
-          #include <folly/synchronization/DistributedMutex.h>
-          int main() {}
-EOF
-        if [ "$?" != 0 ]; then
-          FOLLY_DIR="./third-party/folly"
-        fi
-    fi
-
-    if test -z "$ROCKSDB_USE_IO_URING"; then
-        ROCKSDB_USE_IO_URING=1
-    fi
-    if [ "$ROCKSDB_USE_IO_URING" -ne 0 -a "$PLATFORM" = OS_LINUX ]; then
-        # check for liburing
-        $CXX $PLATFORM_CXXFLAGS -x c++ - -luring -o test.o 2>/dev/null  <<EOF
-          #include <liburing.h>
-          int main() {
-            struct io_uring ring;
-            io_uring_queue_init(1, &ring, 0);
-            return 0;
-          }
-EOF
-        if [ "$?" = 0 ]; then
-            PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -luring"
-            COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_IOURING_PRESENT"
-        fi
-    fi
 fi
 
 # TODO(tec): Fix -Wshorten-64-to-32 errors on FreeBSD and enable the warning.
-# -Wshorten-64-to-32 breaks compilation on FreeBSD aarch64 and i386
-if ! { [ "$TARGET_OS" = FreeBSD -o "$TARGET_OS" = OpenBSD ] && [ "$TARGET_ARCHITECTURE" = arm64 -o "$TARGET_ARCHITECTURE" = i386 ]; }; then
+# -Wshorten-64-to-32 breaks compilation on FreeBSD i386
+if ! [ "$TARGET_OS" = FreeBSD -a "$TARGET_ARCHITECTURE" = i386 ]; then
   # Test whether -Wshorten-64-to-32 is available
-  $CXX $PLATFORM_CXXFLAGS -x c++ - -o test.o -Wshorten-64-to-32 2>/dev/null  <<EOF
+  $CXX $CFLAGS -x c++ - -o /dev/null -Wshorten-64-to-32 2>/dev/null  <<EOF
     int main() {}
 EOF
   if [ "$?" = 0 ]; then
@@ -666,133 +488,88 @@ EOF
   fi
 fi
 
-if [ "$PORTABLE" == "" ] || [ "$PORTABLE" == 0 ]; then
+# shall we use HDFS?
+
+if test "$USE_HDFS"; then
+  if test -z "$JAVA_HOME"; then
+    echo "JAVA_HOME has to be set for HDFS usage."
+    exit 1
+  fi
+  HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS"
+  HDFS_LDFLAGS="$HDFS_LDFLAGS -lhdfs -L$JAVA_HOME/jre/lib/amd64"
+  HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib"
+  HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm"
+  COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS"
+  PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS $HDFS_LDFLAGS"
+  JAVA_LDFLAGS="$JAVA_LDFLAGS $HDFS_LDFLAGS"
+fi
+
+if test -z "$PORTABLE"; then
   if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
     # Tune for this POWER processor, treating '+' models as base models
     POWER=`LD_SHOW_AUXV=1 /bin/true | grep AT_PLATFORM | grep -E -o power[0-9]+`
     COMMON_FLAGS="$COMMON_FLAGS -mcpu=$POWER -mtune=$POWER "
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep -e^arm -e^aarch64`"; then
+  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
+    COMMON_FLAGS="$COMMON_FLAGS -march=z10 "
+  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^arm`"; then
     # TODO: Handle this with approprite options.
     COMMON_FLAGS="$COMMON_FLAGS"
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^aarch64`"; then
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
-    if echo 'int main() {}' | $CXX $PLATFORM_CXXFLAGS -x c++ \
-      -march=native - -o /dev/null 2>/dev/null; then
-      COMMON_FLAGS="$COMMON_FLAGS -march=native "
-    else
-      COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
-    fi
-    COMMON_FLAGS="$COMMON_FLAGS"
-  elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then
-    RISC_ISA=$(cat /proc/cpuinfo | grep -E '^isa\s*:' | head -1 | cut --delimiter=: -f 2 | cut -b 2-)
-    if [ -n "${RISCV_ISA}" ]; then
-      COMMON_FLAGS="$COMMON_FLAGS -march=${RISC_ISA}"
-    fi
   elif [ "$TARGET_OS" == "IOS" ]; then
     COMMON_FLAGS="$COMMON_FLAGS"
-  else
+  elif [ "$TARGET_OS" != "AIX" ] && [ "$TARGET_OS" != "SunOS" ]; then
     COMMON_FLAGS="$COMMON_FLAGS -march=native "
+  elif test "$USE_SSE"; then
+    COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
   fi
-else
-  # PORTABLE specified
-  if [ "$PORTABLE" == 1 ]; then
-    if test -n "`echo $TARGET_ARCHITECTURE | grep ^s390x`"; then
-      COMMON_FLAGS="$COMMON_FLAGS -march=z196 "
-    elif test -n "`echo $TARGET_ARCHITECTURE | grep ^riscv64`"; then
-      COMMON_FLAGS="$COMMON_FLAGS -march=rv64gc"
-    elif test "$USE_SSE"; then
-      # USE_SSE is DEPRECATED
-      # This is a rough approximation of the old USE_SSE behavior
-      COMMON_FLAGS="$COMMON_FLAGS -march=haswell"
-    fi
-    # Other than those cases, not setting -march= here.
-  else
-    # Assume PORTABLE is a minimum assumed cpu type, e.g. PORTABLE=haswell
-    COMMON_FLAGS="$COMMON_FLAGS -march=${PORTABLE}"
-  fi
-
-  if [[ "${PLATFORM}" == "OS_MACOSX" ]]; then
-    # For portability compile for macOS 10.14 (2018) or newer
-    COMMON_FLAGS="$COMMON_FLAGS -mmacosx-version-min=10.14"
-    PLATFORM_LDFLAGS="$PLATFORM_LDFLAGS -mmacosx-version-min=10.14"
-    # -mmacosx-version-min must come first here.
-    PLATFORM_SHARED_LDFLAGS="-mmacosx-version-min=10.14 $PLATFORM_SHARED_LDFLAGS"
-    PLATFORM_CMAKE_FLAGS="-DCMAKE_OSX_DEPLOYMENT_TARGET=10.14"
-    JAVA_STATIC_DEPS_COMMON_FLAGS="-mmacosx-version-min=10.14"
-    JAVA_STATIC_DEPS_LDFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CCFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-    JAVA_STATIC_DEPS_CXXFLAGS="$JAVA_STATIC_DEPS_COMMON_FLAGS"
-  fi
+elif test "$USE_SSE"; then
+  COMMON_FLAGS="$COMMON_FLAGS -msse4.2 -mpclmul"
 fi
 
-if test -n "`echo $TARGET_ARCHITECTURE | grep ^ppc64`"; then
-  # check for GNU libc on ppc64
-  $CXX -x c++ - -o /dev/null 2>/dev/null <<EOF
-    #include <stdio.h>
-    #include <stdlib.h>
-    #include <gnu/libc-version.h>
-
-    int main(int argc, char *argv[]) {
-      printf("GNU libc version: %s\n", gnu_get_libc_version());
-      return 0;
-    }
-EOF
-  if [ "$?" != 0 ]; then
-      PPC_LIBC_IS_GNU=0
-  fi
-fi
-
-$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o test.o 2>/dev/null <<EOF
+$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
   #include <cstdint>
+  #include <nmmintrin.h>
   int main() {
-    uint64_t a = 0xffffFFFFffffFFFF;
-    __uint128_t b = __uint128_t(a) * a;
-    a = static_cast<uint64_t>(b >> 64);
-    (void)a;
+    volatile uint32_t x = _mm_crc32_u32(0, 0);
   }
 EOF
 if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_UINT128_EXTENSION"
+  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_SSE42"
+elif test "$USE_SSE"; then
+  echo "warning: USE_SSE specified but compiler could not use SSE intrinsics, disabling"
+  exit 1
 fi
 
-if [ "$FBCODE_BUILD" != "true" -a "$PLATFORM" = OS_LINUX ]; then
-  $CXX $COMMON_FLAGS $PLATFORM_SHARED_CFLAGS -x c++ -c - -o test_dl.o 2>/dev/null <<EOF
-  void dummy_func() {}
+$CXX $PLATFORM_CXXFLAGS $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+  #include <cstdint>
+  #include <wmmintrin.h>
+  int main() {
+    const auto a = _mm_set_epi64x(0, 0);
+    const auto b = _mm_set_epi64x(0, 0);
+    const auto c = _mm_clmulepi64_si128(a, b, 0x00);
+    auto d = _mm_cvtsi128_si64(c);
+  }
+EOF
+if [ "$?" = 0 ]; then
+  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_PCLMUL"
+elif test "$USE_SSE"; then
+  echo "warning: USE_SSE specified but compiler could not use PCLMUL intrinsics, disabling"
+  exit 1
+fi
+
+# iOS doesn't support thread-local storage, but this check would erroneously
+# succeed because the cross-compiler flags are added by the Makefile, not this
+# script.
+if [ "$PLATFORM" != IOS ]; then
+  $CXX $COMMON_FLAGS -x c++ - -o /dev/null 2>/dev/null <<EOF
+  #if defined(_MSC_VER) && !defined(__thread)
+  #define __thread __declspec(thread)
+  #endif
+  int main() {
+    static __thread int tls;
+  }
 EOF
   if [ "$?" = 0 ]; then
-    $CXX $COMMON_FLAGS $PLATFORM_SHARED_LDFLAGS test_dl.o -o test.o 2>/dev/null
-    if [ "$?" = 0 ]; then
-      EXEC_LDFLAGS+="-ldl"
-      rm -f test_dl.o
-    fi
-  fi
-fi
-
-# check for F_FULLFSYNC
-$CXX $PLATFORM_CXXFALGS -x c++ - -o test.o 2>/dev/null  <<EOF
-  #include <fcntl.h>
-  int main() {
-    fcntl(0, F_FULLFSYNC);
-    return 0;
-  }
-EOF
-if [ "$?" = 0 ]; then
-  COMMON_FLAGS="$COMMON_FLAGS -DHAVE_FULLFSYNC"
-fi
-
-rm -f test.o test_dl.o
-
-# Get the path for the folly installation dir
-if [ "$USE_FOLLY" ]; then
-  if [ "$FOLLY_DIR" ]; then
-    FOLLY_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-inst-dir folly`
-  fi
-fi
-if [ "$USE_FOLLY_LITE" ]; then
-  if [ "$FOLLY_DIR" ]; then
-    BOOST_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir boost`
-    FMT_SOURCE_PATH=`cd $FOLLY_DIR && $PYTHON build/fbcode_builder/getdeps.py show-source-dir fmt`
+    COMMON_FLAGS="$COMMON_FLAGS -DROCKSDB_SUPPORT_THREAD_LOCAL"
   fi
 fi
 
@@ -807,16 +584,10 @@ ROCKSDB_PATCH=`build_tools/version.sh patch`
 
 echo "CC=$CC" >> "$OUTPUT"
 echo "CXX=$CXX" >> "$OUTPUT"
-echo "AR=$AR" >> "$OUTPUT"
 echo "PLATFORM=$PLATFORM" >> "$OUTPUT"
 echo "PLATFORM_LDFLAGS=$PLATFORM_LDFLAGS" >> "$OUTPUT"
-echo "PLATFORM_CMAKE_FLAGS=$PLATFORM_CMAKE_FLAGS" >> "$OUTPUT"
 echo "JAVA_LDFLAGS=$JAVA_LDFLAGS" >> "$OUTPUT"
 echo "JAVA_STATIC_LDFLAGS=$JAVA_STATIC_LDFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CCFLAGS=$JAVA_STATIC_DEPS_CCFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_CXXFLAGS=$JAVA_STATIC_DEPS_CXXFLAGS" >> "$OUTPUT"
-echo "JAVA_STATIC_DEPS_LDFLAGS=$JAVA_STATIC_DEPS_LDFLAGS" >> "$OUTPUT"
-echo "JAVAC_ARGS=$JAVAC_ARGS" >> "$OUTPUT"
 echo "VALGRIND_VER=$VALGRIND_VER" >> "$OUTPUT"
 echo "PLATFORM_CCFLAGS=$PLATFORM_CCFLAGS" >> "$OUTPUT"
 echo "PLATFORM_CXXFLAGS=$PLATFORM_CXXFLAGS" >> "$OUTPUT"
@@ -835,10 +606,6 @@ echo "CLANG_ANALYZER=$CLANG_ANALYZER" >> "$OUTPUT"
 echo "PROFILING_FLAGS=$PROFILING_FLAGS" >> "$OUTPUT"
 echo "FIND=$FIND" >> "$OUTPUT"
 echo "WATCH=$WATCH" >> "$OUTPUT"
-echo "FOLLY_PATH=$FOLLY_PATH" >> "$OUTPUT"
-echo "BOOST_SOURCE_PATH=$BOOST_SOURCE_PATH" >> "$OUTPUT"
-echo "FMT_SOURCE_PATH=$FMT_SOURCE_PATH" >> "$OUTPUT"
-
 # This will enable some related identifiers for the preprocessor
 if test -n "$JEMALLOC"; then
   echo "JEMALLOC=1" >> "$OUTPUT"
@@ -849,9 +616,4 @@ fi
 if test -n "$WITH_JEMALLOC_FLAG"; then
   echo "WITH_JEMALLOC_FLAG=$WITH_JEMALLOC_FLAG" >> "$OUTPUT"
 fi
-if test -n "$USE_FOLLY"; then
-  echo "USE_FOLLY=$USE_FOLLY" >> "$OUTPUT"
-fi
-if test -n "$PPC_LIBC_IS_GNU"; then
-  echo "PPC_LIBC_IS_GNU=$PPC_LIBC_IS_GNU" >> "$OUTPUT"
-fi
+echo "LUA_PATH=$LUA_PATH" >> "$OUTPUT"

build_tools/check-public-header.sh

@@ -1,31 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Meta Platforms, Inc. and affiliates. All Rights Reserved.
-#
-# Check for some simple mistakes in public headers (on the command line)
-# that should prevent commit or push
-
-BAD=""
-
-# Look for potential for ODR violations caused by public headers depending on
-# build parameters that could vary between RocksDB build and application build.
-# * Cases like ROCKSDB_NAMESPACE, and ROCKSDB_ASSERT_STATUS_CHECKED are
-#   intentional, hard to avoid. (We expect definitions to change and the user
-#   should also.)
-# * Cases like _WIN32, OS_WIN, and __cplusplus are essentially ODR-safe.
-# * Cases like
-#   #ifdef BLAH  // ODR-SAFE
-#   #undef BLAH
-#   #endif
-#   that should not cause ODR violations can be exempted with the ODR-SAFE
-#   marker recognized here.
-
-grep -nHE '^#if' -- "$@" | grep -vE 'ROCKSDB_NAMESPACE|ROCKSDB_ASSERT_STATUS_CHECKED|_WIN32|OS_WIN|ODR-SAFE|__cplusplus|ROCKSDB_DLL|ROCKSDB_LIBRARY_EXPORTS'
-if [ "$?" != "1" ]; then
-  echo "^^^^^ #if in public API could cause an ODR violation."
-  echo "      Add // ODR-SAFE if verified safe."
-  BAD=1
-fi
-
-if [ "$BAD" ]; then
-  exit 1
-fi

build_tools/check-sources.sh

@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Check for some simple mistakes that should prevent commit or push
-
-BAD=""
-
-git grep -n 'namespace rocksdb' -- '*.[ch]*'
-if [ "$?" != "1" ]; then
-  echo "^^^^^ Do not hardcode namespace rocksdb. Use ROCKSDB_NAMESPACE"
-  BAD=1
-fi
-
-git grep -n -i 'nocommit' -- ':!build_tools/check-sources.sh'
-if [ "$?" != "1" ]; then
-  echo "^^^^^ Code was not intended to be committed"
-  BAD=1
-fi
-
-git grep -n 'include <rocksdb/' -- ':!build_tools/check-sources.sh'
-if [ "$?" != "1" ]; then
-  echo '^^^^^ Use double-quotes as in #include "rocksdb/something.h"'
-  BAD=1
-fi
-
-git grep -n 'include "include/rocksdb/' -- ':!build_tools/check-sources.sh'
-if [ "$?" != "1" ]; then
-  echo '^^^^^ Use #include "rocksdb/something.h" instead of #include "include/rocksdb/something.h"'
-  BAD=1
-fi
-
-git grep -n 'using namespace' -- ':!build_tools' ':!docs' \
-    ':!third-party/folly/folly/lang/Align.h' \
-    ':!third-party/gtest-1.8.1/fused-src/gtest/gtest.h'
-if [ "$?" != "1" ]; then
-  echo '^^^^ Do not use "using namespace"'
-  BAD=1
-fi
-
-LC_ALL=C git grep -n $'[\x80-\xff]' -- ':!docs' ':!*.md' ':!.github'
-if [ "$?" != "1" ]; then
-  echo '^^^^ Use only ASCII characters in source files'
-  BAD=1
-fi
-
-if [ "$BAD" ]; then
-  exit 1
-fi

build_tools/check-workflow-yaml.sh

@@ -1,42 +0,0 @@
-#!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Validate GitHub Actions workflow YAML before it reaches CI runtime.
-
-set -euo pipefail
-
-if ! command -v ruby >/dev/null 2>&1; then
-  echo "ruby is required to validate GitHub Actions workflow YAML"
-  echo "On CentOS Stream: sudo dnf install ruby rubygems rubygem-psych"
-  exit 1
-fi
-
-if ! ruby -e 'require "psych"' 2>/dev/null; then
-  echo "ruby is installed but cannot load required library 'psych'"
-  echo "On CentOS Stream: sudo dnf install rubygems rubygem-psych"
-  exit 1
-fi
-
-ruby <<'RUBY'
-require "psych"
-
-bad = false
-workflow_files = Dir[".github/workflows/*.{yml,yaml}"].sort
-
-if workflow_files.empty?
-  warn "No workflow YAML files found under .github/workflows"
-  exit 1
-end
-
-workflow_files.each do |path|
-  begin
-    Psych.parse_file(path)
-    puts "OK #{path}"
-  rescue Psych::Exception => e
-    warn "Invalid YAML in #{path}: #{e.message}"
-    bad = true
-  end
-end
-
-exit(bad ? 1 : 0)
-RUBY

build_tools/check_progress.sh

@@ -1,231 +0,0 @@
-#!/bin/bash
-# Output test progress in JSON format for machine parsing
-# Usage: build_tools/check_progress.sh
-
-LOG_FILE="LOG"
-T_DIR="t"
-SRC_MK="src.mk"
-
-# Maximum lines of test output to include per failed test
-MAX_OUTPUT_LINES=50
-
-# Helper to escape string for JSON (handles newlines, quotes, backslashes, tabs)
-json_escape() {
-    local str="$1"
-    # Use python for reliable JSON escaping if available, otherwise use sed
-    if command -v python3 &>/dev/null; then
-        printf '%s' "$str" | python3 -c 'import json,sys; print(json.dumps(sys.stdin.read())[1:-1], end="")'
-    else
-        printf '%s' "$str" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g; s/\r/\\r/g' | awk '{printf "%s\\n", $0}' | sed 's/\\n$//'
-    fi
-}
-
-# Helper to output JSON and exit
-output_json() {
-    local status="$1"
-    local completed="${2:-0}"
-    local total="${3:-0}"
-    local failed="${4:-0}"
-    local percent="${5:-0}"
-    local eta="${6:-0}"
-    local avg_time="${7:-0}"
-    local last_item="${8:-}"
-    local phase="${9:-}"
-    local failed_tests="${10:-}"
-
-    # Build JSON output
-    local json="{\"status\":\"$status\""
-
-    if [[ -n "$phase" ]]; then
-        json="$json,\"phase\":\"$phase\""
-    fi
-
-    json="$json,\"completed\":$completed,\"total\":$total,\"failed\":$failed,\"percent\":$percent"
-    json="$json,\"eta_seconds\":$eta,\"avg_time\":\"$avg_time\",\"last_item\":\"$(json_escape "$last_item")\""
-
-    if [[ -n "$failed_tests" ]]; then
-        json="$json,\"failed_tests\":[$failed_tests]"
-    fi
-
-    json="$json}"
-    echo "$json"
-}
-
-# Get failed test info with log output
-get_failed_tests_json() {
-    local log_file="$1"
-    local t_dir="$2"
-    local max_failures=10
-    local count=0
-    local first=true
-
-    # Get failed tests from LOG file
-    while IFS=$'\t' read -r seq host starttime runtime send recv exitval signal cmd; do
-        # Skip header line
-        [[ "$seq" == "Seq" ]] && continue
-
-        # Check if failed (exitval != 0 or signal != 0)
-        if [[ "$exitval" != "0" || "$signal" != "0" ]]; then
-            # Extract test name from command
-            test_name=$(echo "$cmd" | sed 's,.*/run-,,;s, .*,,')
-
-            # Get log file path
-            log_path="$t_dir/log-run-$test_name"
-
-            # Read test output (last N lines)
-            if [[ -f "$log_path" ]]; then
-                output=$(tail -n "$MAX_OUTPUT_LINES" "$log_path" 2>/dev/null)
-            else
-                output="(log file not found: $log_path)"
-            fi
-
-            # Escape output for JSON
-            escaped_output=$(json_escape "$output")
-
-            # Build JSON object for this failure
-            if [[ "$first" == "true" ]]; then
-                first=false
-            else
-                printf ","
-            fi
-            printf '{"test":"%s","exit_code":%d,"signal":%d,"output":"%s"}' \
-                "$test_name" "$exitval" "$signal" "$escaped_output"
-
-            ((count++))
-            if [[ $count -ge $max_failures ]]; then
-                break
-            fi
-        fi
-    done < "$log_file"
-}
-
-# Check if tests are running (LOG file exists)
-if [[ -f "$LOG_FILE" ]]; then
-    # Count total tests from t/run-* files
-    if [[ -d "$T_DIR" ]]; then
-        total=$(find "$T_DIR" -name 'run-*' -type f 2>/dev/null | wc -l)
-    else
-        total=0
-    fi
-
-    # If no parallel tests generated yet
-    if [[ "$total" -eq 0 ]]; then
-        output_json "running" 0 0 0 0 0 "0" "" "generating"
-        exit 0
-    fi
-
-    # Parse LOG file (skip header line)
-    # LOG format: Seq Host Starttime JobRuntime Send Receive Exitval Signal Command
-    completed=$(tail -n +2 "$LOG_FILE" 2>/dev/null | wc -l)
-
-    # Count failures
-    failed=$(awk -F'\t' 'NR>1 && ($7 != 0 || $8 != 0) {count++} END {print count+0}' "$LOG_FILE" 2>/dev/null)
-
-    # Get failed tests JSON with output (only if there are failures)
-    if [[ "$failed" -gt 0 ]]; then
-        failed_tests=$(get_failed_tests_json "$LOG_FILE" "$T_DIR")
-    else
-        failed_tests=""
-    fi
-
-    # Calculate percentage
-    if [[ "$total" -gt 0 ]]; then
-        percent=$((completed * 100 / total))
-    else
-        percent=0
-    fi
-
-    # Get last completed test name (extract from command column)
-    last_test=$(tail -1 "$LOG_FILE" 2>/dev/null | awk -F'\t' '{print $9}' | sed 's,.*/run-,,;s, .*,,;s,^./,,')
-
-    # Calculate ETA based on average time
-    if [[ "$completed" -gt 0 ]]; then
-        avg_time=$(awk -F'\t' 'NR>1 {sum+=$4; count++} END {if(count>0) printf "%.1f", sum/count; else print "0"}' "$LOG_FILE")
-        remaining=$((total - completed))
-        eta=$(awk "BEGIN {printf \"%.0f\", $avg_time * $remaining}")
-    else
-        avg_time="0"
-        eta="0"
-    fi
-
-    # Determine status
-    if [[ "$completed" -ge "$total" ]]; then
-        status="completed"
-    elif [[ "$completed" -gt 0 ]]; then
-        status="running"
-    else
-        status="starting"
-    fi
-
-    output_json "$status" "$completed" "$total" "$failed" "$percent" "$eta" "$avg_time" "$last_test" "testing" "$failed_tests"
-    exit 0
-fi
-
-# No LOG file - check if we're in compilation/linking phase
-# Count expected source files from src.mk
-if [[ -f "$SRC_MK" ]]; then
-    # Count LIB_SOURCES (library object files to compile)
-    expected_lib_objects=$(grep -E '\.cc\s*\\?$' "$SRC_MK" | grep -v '^#' | wc -l)
-
-    # Count TEST_MAIN_SOURCES (test binaries to link)
-    expected_test_binaries=$(sed -n '/^TEST_MAIN_SOURCES =/,/^[^ ]/p' "$SRC_MK" | grep -cE '\.cc\s*\\?$' 2>/dev/null || echo 0)
-else
-    expected_lib_objects=0
-    expected_test_binaries=0
-fi
-
-# Check for test generation phase (t/ directory being created)
-if [[ -d "$T_DIR" ]]; then
-    total=$(find "$T_DIR" -name 'run-*' -type f 2>/dev/null | wc -l)
-    if [[ "$total" -gt 0 ]]; then
-        output_json "running" 0 "$total" 0 0 0 "0" "" "generating"
-        exit 0
-    fi
-fi
-
-# Count compiled object files (in subdirectories matching source structure)
-# Object files are created as dir/file.o (e.g., cache/cache.o, db/db_impl.o)
-compiled_objects=0
-if [[ "$expected_lib_objects" -gt 0 ]]; then
-    # Count .o files in source directories
-    compiled_objects=$(find cache db env file logging memory memtable monitoring options port table test_util trace_replay util utilities -name '*.o' -type f 2>/dev/null | wc -l)
-fi
-
-# Count linked test binaries (test binaries are in current directory with _test suffix)
-linked_tests=0
-if [[ "$expected_test_binaries" -gt 0 ]]; then
-    linked_tests=$(find . -maxdepth 1 -name '*_test' -type f -executable 2>/dev/null | wc -l)
-fi
-
-# Determine phase based on what exists
-if [[ "$compiled_objects" -eq 0 && "$linked_tests" -eq 0 ]]; then
-    # Nothing compiled yet - not started or just beginning
-    output_json "not_started" 0 0 0 0 0 "0" ""
-    exit 0
-fi
-
-# Calculate total work units: compiling + linking
-total_work=$((expected_lib_objects + expected_test_binaries))
-completed_work=$((compiled_objects + linked_tests))
-
-if [[ "$total_work" -gt 0 ]]; then
-    percent=$((completed_work * 100 / total_work))
-else
-    percent=0
-fi
-
-# Determine phase
-if [[ "$compiled_objects" -lt "$expected_lib_objects" ]]; then
-    phase="compiling"
-    # Get most recently modified .o file as last_item
-    last_item=$(find cache db env file logging memory memtable monitoring options port table test_util trace_replay util utilities -name '*.o' -type f -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2- | sed 's,^\./,,;s,\.o$,,')
-elif [[ "$linked_tests" -lt "$expected_test_binaries" ]]; then
-    phase="linking"
-    # Get most recently modified test binary as last_item
-    last_item=$(find . -maxdepth 1 -name '*_test' -type f -executable -printf '%T@ %p\n' 2>/dev/null | sort -rn | head -1 | cut -d' ' -f2- | sed 's,^\./,,')
-else
-    phase="generating"
-    last_item=""
-fi
-
-output_json "running" "$completed_work" "$total_work" 0 "$percent" 0 "0" "$last_item" "$phase"

build_tools/cont_integration.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2016, Facebook. All rights reserved.
+#
+# Overall wrapper script for RocksDB continuous builds. The implementation is a
+# trivial pulling scheme. We loop infinitely, check if any new changes have been
+# committed, if yes then trigger a Sandcastle run, and finally go to sleep again
+# for a certain interval.
+#
+
+SRC_GIT_REPO=/data/git/rocksdb-public
+error=0
+
+function log {
+  DATE=`date +%Y-%m-%d:%H:%M:%S`
+  # shellcheck disable=SC2068
+  echo $DATE $@
+}
+
+function log_err {
+  # shellcheck disable=SC2145
+  log "ERROR: $@ Error code: $error."
+}
+
+function update_repo_status {
+  # Update the parent first.
+  pushd $SRC_GIT_REPO
+
+  # This is a fatal error. Something in the environment isn't right and we will
+  # terminate the execution.
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "Where is $SRC_GIT_REPO?"
+    exit $error
+  fi
+
+  HTTPS_PROXY=fwdproxy:8080 git fetch -f
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git fetch -f failed."
+    popd
+    return $error
+  fi
+
+  git update-ref refs/heads/master refs/remotes/origin/master
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git update-ref failed."
+    popd
+    return $error
+  fi
+
+  popd
+
+  # We're back in an instance-specific directory. Get the latest changes.
+  git pull --rebase
+
+  error=$?
+  if [ ! $error -eq 0 ]; then
+    log_err "git pull --rebase failed."
+    return $error
+  fi
+}
+
+#
+# Execution starts here.
+#
+
+# Path to the determinator from the root of the RocksDB repo.
+CONTRUN_DETERMINATOR=./build_tools/RocksDBCommonHelper.php
+
+# Value of the previous commit.
+PREV_COMMIT=
+
+log "Starting to monitor for new RocksDB changes ..."
+log "Running under `pwd` as `whoami`."
+
+# Paranoia. Make sure that we're using the right branch.
+git checkout master
+
+error=$?
+if [ ! $error -eq 0 ]; then
+  log_err "This is not good. Can't checkout master. Bye-bye!"
+  exit 1
+fi
+
+# We'll run forever and let the execution environment terminate us if we'll
+# exceed whatever timeout is set for the job.
+while true;
+do
+  # Get the latest changes committed.
+  update_repo_status
+
+  error=$?
+  if [  $error -eq 0 ]; then
+    LAST_COMMIT=`git log -1 | head -1 | grep commit | awk '{ print $2; }'`
+
+    log "Last commit is '$LAST_COMMIT', previous commit is '$PREV_COMMIT'."
+
+    if [ "$PREV_COMMIT" == "$LAST_COMMIT" ]; then
+      log "There were no changes since the last time I checked. Going to sleep."
+    else
+      if [ ! -z "$LAST_COMMIT" ]; then
+        log "New code has been committed or previous commit not known. " \
+            "Will trigger the tests."
+
+        PREV_COMMIT=$LAST_COMMIT
+        log "Updated previous commit to '$PREV_COMMIT'."
+
+        #
+        # This is where we'll trigger the Sandcastle run. The values for
+        # HTTPS_APP_VALUE and HTTPS_APP_VALUE will be set in the container we're
+        # running in.
+        #
+        POST_RECEIVE_HOOK=1 php $CONTRUN_DETERMINATOR
+
+        error=$?
+        if [ $error -eq 0 ]; then
+          log "Sandcastle run successfully triggered."
+        else
+          log_err "Failed to trigger Sandcastle run."
+        fi
+      else
+        log_err "Previous commit not updated. Don't know what the last one is."
+      fi
+    fi
+  else
+    log_err "Getting latest changes failed. Will skip running tests for now."
+  fi
+
+  # Always sleep, even if errors happens while trying to determine the latest
+  # commit. This will prevent us terminating in case of transient errors.
+  log "Will go to sleep for 5 minutes."
+  sleep 5m
+done

build_tools/dependencies.sh

@@ -0,0 +1,19 @@
+# shellcheck disable=SC2148
+GCC_BASE=/mnt/gvfs/third-party2/gcc/8219ec1bcedf8ad9da05e121e193364de2cc4f61/5.x/centos6-native/c447969
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/64d8d58e3d84f8bde7a029763d4f5baf39d0d5b9/stable/centos6-native/6aaf4de
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/ba9be983c81de7299b59fe71950c664a84dcb5f8/5.x/gcc-5-glibc-2.23/339d858
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/f20197cf3d4bd50339c9777aaa0b2ccadad9e2cb/2.23/gcc-5-glibc-2.23/ca1d1c0
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/6427ce8c7496e4ab06c2da81543b94c0de8be3d0/1.1.3/gcc-5-glibc-2.23/9bc6787
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/8f1e8b867d26efef93eac2fabbdb2e1d512665d7/1.2.8/gcc-5-glibc-2.23/9bc6787
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/70471c0571559fe0af7db6d7e8860b93a7eadfe1/1.0.6/gcc-5-glibc-2.23/9bc6787
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/453c89d6f0e68cdf1c151c769197fabedad9cac8/r131/gcc-5-glibc-2.23/9bc6787
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/00a40fa5f8bd2cd0622f2e868552793aef37ccf4/1.3.0/gcc-5-glibc-2.23/03859b5
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/47eef08f9acb77de982fbda6047c26d330739538/2.2.0/gcc-5-glibc-2.23/9bc6787
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/4414ddc78df8008b35cc4adac23590ad29148584/master/gcc-5-glibc-2.23/d506c82
+NUMA_BASE=/mnt/gvfs/third-party2/numa/9d7ae2693d05d62f9a579cb21e6b717cf257a75d/2.0.11/gcc-5-glibc-2.23/9bc6787
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/2b2dd58e3a52ccf2c1d827def59e5f740de0ad15/1.2/gcc-5-glibc-2.23/b443de1
+TBB_BASE=/mnt/gvfs/third-party2/tbb/379addf7ab2468a2b4293b47456cfcd1c9cb318d/4.3/gcc-5-glibc-2.23/9bc6787
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/3f68f5fe65a85b7c2d3e66852268fbd1efdb3151/4.0.9-36_fbk5_2933_gd092e3f/gcc-5-glibc-2.23/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b9fab0aec99d9c36408e810b2677e91c12807afd/2.28/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/423431d61786b20bcc3bde8972901130cb29e6b3/3.11.0/gcc-5-glibc-2.23/9bc6787
+LUA_BASE=/mnt/gvfs/third-party2/lua/3b0bb3bd9a0f690a069c479fcc0f7424fc7456d2/5.2.3/gcc-5-glibc-2.23/65372bd

build_tools/dependencies_4.8.1.sh

@@ -0,0 +1,19 @@
+# shellcheck disable=SC2148
+GCC_BASE=/mnt/gvfs/third-party2/gcc/cf7d14c625ce30bae1a4661c2319c5a283e4dd22/4.8.1/centos6-native/cc6c9dc
+CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/8598c375b0e94e1448182eb3df034704144a838d/stable/centos6-native/3f16ddd
+LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d6e0a7da6faba45f5e5b1638f9edd7afc2f34e7d/4.8.1/gcc-4.8.1-glibc-2.17/8aac7fc
+GLIBC_BASE=/mnt/gvfs/third-party2/glibc/d282e6e8f3d20f4e40a516834847bdc038e07973/2.17/gcc-4.8.1-glibc-2.17/99df8fc
+SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/8c38a4c1e52b4c2cc8a9cdc31b9c947ed7dbfcb4/1.1.3/gcc-4.8.1-glibc-2.17/c3f970a
+ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0882df3713c7a84f15abe368dc004581f20b39d7/1.2.8/gcc-4.8.1-glibc-2.17/c3f970a
+BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/740325875f6729f42d28deaa2147b0854f3a347e/1.0.6/gcc-4.8.1-glibc-2.17/c3f970a
+LZ4_BASE=/mnt/gvfs/third-party2/lz4/0e790b441e2d9acd68d51e1d2e028f88c6a79ddf/r131/gcc-4.8.1-glibc-2.17/c3f970a
+ZSTD_BASE=/mnt/gvfs/third-party2/zstd/9455f75ff7f4831dc9fda02a6a0f8c68922fad8f/1.0.0/gcc-4.8.1-glibc-2.17/c3f970a
+GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/f001a51b2854957676d07306ef3abf67186b5c8b/2.1.1/gcc-4.8.1-glibc-2.17/c3f970a
+JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/fc8a13ca1fffa4d0765c716c5a0b49f0c107518f/master/gcc-4.8.1-glibc-2.17/8d31e51
+NUMA_BASE=/mnt/gvfs/third-party2/numa/17c514c4d102a25ca15f4558be564eeed76f4b6a/2.0.8/gcc-4.8.1-glibc-2.17/c3f970a
+LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/ad576de2a1ea560c4d3434304f0fc4e079bede42/trunk/gcc-4.8.1-glibc-2.17/675d945
+TBB_BASE=/mnt/gvfs/third-party2/tbb/9d9a554877d0c5bef330fe818ab7178806dd316a/4.0_update2/gcc-4.8.1-glibc-2.17/c3f970a
+KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/7c111ff27e0c466235163f00f280a9d617c3d2ec/4.0.9-36_fbk5_2933_gd092e3f/gcc-4.8.1-glibc-2.17/da39a3e
+BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/b7fd454c4b10c6a81015d4524ed06cdeab558490/2.26/centos6-native/da39a3e
+VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/d7f4d4d86674a57668e3a96f76f0e17dd0eb8765/3.8.1/gcc-4.8.1-glibc-2.17/c3f970a
+LUA_BASE=/mnt/gvfs/third-party2/lua/61e4abf5813bbc39bc4f548757ccfcadde175a48/5.2.3/centos6-native/730f94e

build_tools/dependencies_platform010.sh

@@ -1,21 +0,0 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-# The file is generated using update_dependencies.sh.
-GCC_BASE=/mnt/gvfs/third-party2/gcc/62de5a92e5f23c661c3d4b9f322e04eb14e7a5bd/11.x/centos8-native/886b5eb
-CLANG_BASE=/mnt/gvfs/third-party2/llvm-fb/1f6edd1ff15c99c861afc8f3cd69054cd974dd64/15/platform010/72a2ff8
-LIBGCC_BASE=/mnt/gvfs/third-party2/libgcc/d1129753c8361ac8e9453c0f4291337a4507ebe6/11.x/platform010/5684a5a
-GLIBC_BASE=/mnt/gvfs/third-party2/glibc/fed6e93d87571fb162734c86636119d45a398963/2.34/platform010/f259413
-SNAPPY_BASE=/mnt/gvfs/third-party2/snappy/31a346126a1f3b64812c362511cb04cc1bd40855/1.1.8/platform010/76ebdda
-ZLIB_BASE=/mnt/gvfs/third-party2/zlib/0c65c05468b5a38cef1a106a1f526463e120c8dd/1.2.8/platform010/76ebdda
-BZIP2_BASE=/mnt/gvfs/third-party2/bzip2/09703139cfc376bd8a82642385a0e97726b28287/1.0.6/platform010/76ebdda
-LZ4_BASE=/mnt/gvfs/third-party2/lz4/ff23d17b932725cc1734a14896a8b67c518ba169/1.9.4/platform010/76ebdda
-ZSTD_BASE=/mnt/gvfs/third-party2/zstd/576397d8b1d9cea7306ad1e454d5e55caaa2ff1c/1.4.x/platform010/64091f4
-GFLAGS_BASE=/mnt/gvfs/third-party2/gflags/fecac07861cb829f5e60dbeff0503d3272db73c0/2.2.0/platform010/76ebdda
-JEMALLOC_BASE=/mnt/gvfs/third-party2/jemalloc/0bb3f5756788ce26e2e16a1cb2f2af2c59b51abe/master/platform010/f57cc4a
-NUMA_BASE=/mnt/gvfs/third-party2/numa/5b602edd46fda54cdd7ea45f77dbe4061206e174/2.0.11/platform010/76ebdda
-LIBUNWIND_BASE=/mnt/gvfs/third-party2/libunwind/97cac22a149c2e202917e05d44e87e516b68216f/1.4/platform010/5074a48
-TBB_BASE=/mnt/gvfs/third-party2/tbb/53953ebc4e3eda85ad6fc3e429ba146035e97b90/2018_U5/platform010/76ebdda
-LIBURING_BASE=/mnt/gvfs/third-party2/liburing/a98e2d137007e3ebf7f33bd6f99c2c56bdaf8488/20210212/platform010/76ebdda
-BENCHMARK_BASE=/mnt/gvfs/third-party2/benchmark/780c7a0f9cf0967961e69ad08e61cddd85d61821/trunk/platform010/76ebdda
-KERNEL_HEADERS_BASE=/mnt/gvfs/third-party2/kernel-headers/624a2f8f6c93c3c1df8aa4a6255d8202631a6c80/fb/platform010/da39a3e
-BINUTILS_BASE=/mnt/gvfs/third-party2/binutils/39579e8603b48b3540f8b0633f43adf29acccb8b/2.37/centos8-native/da39a3e
-VALGRIND_BASE=/mnt/gvfs/third-party2/valgrind/cd9cc656d49ecb53797ce4d055e49fde29fd57ff/3.19.0/platform010/76ebdda

build_tools/dockerbuild.sh

@@ -1,3 +1,2 @@
 #!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 docker run -v $PWD:/rocks -w /rocks buildpack-deps make

build_tools/error_filter.py

@@ -3,37 +3,40 @@
 #  COPYING file in the root directory) and Apache 2.0 License
 #  (found in the LICENSE.Apache file in the root directory).
 
-"""Filter for error messages in test output:
+'''Filter for error messages in test output:
     - Receives merged stdout/stderr from test on stdin
     - Finds patterns of known error messages for test name (first argument)
     - Prints those error messages to stdout
-"""
+'''
 
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
 
 import re
 import sys
 
 
-class ErrorParserBase:
+class ErrorParserBase(object):
     def parse_error(self, line):
-        """Parses a line of test output. If it contains an error, returns a
+        '''Parses a line of test output. If it contains an error, returns a
         formatted message describing the error; otherwise, returns None.
         Subclasses must override this method.
-        """
+        '''
         raise NotImplementedError
 
 
 class GTestErrorParser(ErrorParserBase):
-    """A parser that remembers the last test that began running so it can print
+    '''A parser that remembers the last test that began running so it can print
     that test's name upon detecting failure.
-    """
-
-    _GTEST_NAME_PATTERN = re.compile(r"\[ RUN      \] (\S+)$")
+    '''
+    _GTEST_NAME_PATTERN = re.compile(r'\[ RUN      \] (\S+)$')
     # format: '<filename or "unknown file">:<line #>: Failure'
-    _GTEST_FAIL_PATTERN = re.compile(r"(unknown file|\S+:\d+): Failure$")
+    _GTEST_FAIL_PATTERN = re.compile(r'(unknown file|\S+:\d+): Failure$')
 
     def __init__(self):
-        self._last_gtest_name = "Unknown test"
+        self._last_gtest_name = 'Unknown test'
 
     def parse_error(self, line):
         gtest_name_match = self._GTEST_NAME_PATTERN.match(line)
@@ -42,13 +45,14 @@ class GTestErrorParser(ErrorParserBase):
             return None
         gtest_fail_match = self._GTEST_FAIL_PATTERN.match(line)
         if gtest_fail_match:
-            return "{} failed: {}".format(self._last_gtest_name, gtest_fail_match.group(1))
+            return '%s failed: %s' % (
+                    self._last_gtest_name, gtest_fail_match.group(1))
         return None
 
 
 class MatchErrorParser(ErrorParserBase):
-    """A simple parser that returns the whole line if it matches the pattern."""
-
+    '''A simple parser that returns the whole line if it matches the pattern.
+    '''
     def __init__(self, pattern):
         self._pattern = re.compile(pattern)
 
@@ -60,109 +64,92 @@ class MatchErrorParser(ErrorParserBase):
 
 class CompilerErrorParser(MatchErrorParser):
     def __init__(self):
-        # format (compile error):
-        #   '<filename>:<line #>:<column #>: error: <error msg>'
-        # format (link error):
-        #   '<filename>:<line #>: error: <error msg>'
-        # The below regex catches both
-        super().__init__(r"\S+:\d+: error:")
+        # format: '<filename>:<line #>:<column #>: error: <error msg>'
+        super(CompilerErrorParser, self).__init__(r'\S+:\d+:\d+: error:')
 
 
 class ScanBuildErrorParser(MatchErrorParser):
     def __init__(self):
-        super().__init__(r"scan-build: \d+ bugs found.$")
+        super(ScanBuildErrorParser, self).__init__(
+                r'scan-build: \d+ bugs found.$')
 
 
 class DbCrashErrorParser(MatchErrorParser):
     def __init__(self):
-        super().__init__(r"\*\*\*.*\^$|TEST FAILED.")
+        super(DbCrashErrorParser, self).__init__(r'\*\*\*.*\^$|TEST FAILED.')
 
 
 class WriteStressErrorParser(MatchErrorParser):
     def __init__(self):
-        super().__init__(
-            r"ERROR: write_stress died with exitcode=\d+"
-        )
+        super(WriteStressErrorParser, self).__init__(
+                r'ERROR: write_stress died with exitcode=\d+')
 
 
 class AsanErrorParser(MatchErrorParser):
     def __init__(self):
-        super().__init__(r"==\d+==ERROR: AddressSanitizer:")
+        super(AsanErrorParser, self).__init__(
+                r'==\d+==ERROR: AddressSanitizer:')
 
 
 class UbsanErrorParser(MatchErrorParser):
     def __init__(self):
         # format: '<filename>:<line #>:<column #>: runtime error: <error msg>'
-        super().__init__(r"\S+:\d+:\d+: runtime error:")
+        super(UbsanErrorParser, self).__init__(r'\S+:\d+:\d+: runtime error:')
 
 
 class ValgrindErrorParser(MatchErrorParser):
     def __init__(self):
         # just grab the summary, valgrind doesn't clearly distinguish errors
         # from other log messages.
-        super().__init__(r"==\d+== ERROR SUMMARY:")
+        super(ValgrindErrorParser, self).__init__(r'==\d+== ERROR SUMMARY:')
 
 
 class CompatErrorParser(MatchErrorParser):
     def __init__(self):
-        super().__init__(r"==== .*[Ee]rror.* ====$")
+        super(CompatErrorParser, self).__init__(r'==== .*[Ee]rror.* ====$')
 
 
 class TsanErrorParser(MatchErrorParser):
     def __init__(self):
-        super().__init__(r"WARNING: ThreadSanitizer:")
+        super(TsanErrorParser, self).__init__(r'WARNING: ThreadSanitizer:')
 
 
 _TEST_NAME_TO_PARSERS = {
-    "punit": [CompilerErrorParser, GTestErrorParser],
-    "unit": [CompilerErrorParser, GTestErrorParser],
-    "release": [CompilerErrorParser, GTestErrorParser],
-    "unit_481": [CompilerErrorParser, GTestErrorParser],
-    "release_481": [CompilerErrorParser, GTestErrorParser],
-    "clang_unit": [CompilerErrorParser, GTestErrorParser],
-    "clang_release": [CompilerErrorParser, GTestErrorParser],
-    "clang_analyze": [CompilerErrorParser, ScanBuildErrorParser],
-    "code_cov": [CompilerErrorParser, GTestErrorParser],
-    "unity": [CompilerErrorParser, GTestErrorParser],
-    "lite": [CompilerErrorParser],
-    "lite_test": [CompilerErrorParser, GTestErrorParser],
-    "stress_crash": [CompilerErrorParser, DbCrashErrorParser],
-    "stress_crash_with_atomic_flush": [CompilerErrorParser, DbCrashErrorParser],
-    "stress_crash_with_txn": [CompilerErrorParser, DbCrashErrorParser],
-    "write_stress": [CompilerErrorParser, WriteStressErrorParser],
-    "asan": [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
-    "asan_crash": [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    "asan_crash_with_atomic_flush": [
-        CompilerErrorParser,
-        AsanErrorParser,
-        DbCrashErrorParser,
-    ],
-    "asan_crash_with_txn": [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
-    "ubsan": [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
-    "ubsan_crash": [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    "ubsan_crash_with_atomic_flush": [
-        CompilerErrorParser,
-        UbsanErrorParser,
-        DbCrashErrorParser,
-    ],
-    "ubsan_crash_with_txn": [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
-    "valgrind": [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
-    "tsan": [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
-    "format_compatible": [CompilerErrorParser, CompatErrorParser],
-    "run_format_compatible": [CompilerErrorParser, CompatErrorParser],
-    "no_compression": [CompilerErrorParser, GTestErrorParser],
-    "run_no_compression": [CompilerErrorParser, GTestErrorParser],
-    "regression": [CompilerErrorParser],
-    "run_regression": [CompilerErrorParser],
+    'punit': [CompilerErrorParser, GTestErrorParser],
+    'unit': [CompilerErrorParser, GTestErrorParser],
+    'release': [CompilerErrorParser, GTestErrorParser],
+    'unit_481': [CompilerErrorParser, GTestErrorParser],
+    'release_481': [CompilerErrorParser, GTestErrorParser],
+    'clang_unit': [CompilerErrorParser, GTestErrorParser],
+    'clang_release': [CompilerErrorParser, GTestErrorParser],
+    'clang_analyze': [CompilerErrorParser, ScanBuildErrorParser],
+    'code_cov': [CompilerErrorParser, GTestErrorParser],
+    'unity': [CompilerErrorParser, GTestErrorParser],
+    'lite': [CompilerErrorParser],
+    'lite_test': [CompilerErrorParser, GTestErrorParser],
+    'stress_crash': [CompilerErrorParser, DbCrashErrorParser],
+    'write_stress': [CompilerErrorParser, WriteStressErrorParser],
+    'asan': [CompilerErrorParser, GTestErrorParser, AsanErrorParser],
+    'asan_crash': [CompilerErrorParser, AsanErrorParser, DbCrashErrorParser],
+    'ubsan': [CompilerErrorParser, GTestErrorParser, UbsanErrorParser],
+    'ubsan_crash': [CompilerErrorParser, UbsanErrorParser, DbCrashErrorParser],
+    'valgrind': [CompilerErrorParser, GTestErrorParser, ValgrindErrorParser],
+    'tsan': [CompilerErrorParser, GTestErrorParser, TsanErrorParser],
+    'format_compatible': [CompilerErrorParser, CompatErrorParser],
+    'run_format_compatible': [CompilerErrorParser, CompatErrorParser],
+    'no_compression': [CompilerErrorParser, GTestErrorParser],
+    'run_no_compression': [CompilerErrorParser, GTestErrorParser],
+    'regression': [CompilerErrorParser],
+    'run_regression': [CompilerErrorParser],
 }
 
 
 def main():
     if len(sys.argv) != 2:
-        return "Usage: %s <test name>" % sys.argv[0]
+        return 'Usage: %s <test name>' % sys.argv[0]
     test_name = sys.argv[1]
     if test_name not in _TEST_NAME_TO_PARSERS:
-        return "Unknown test name: %s" % test_name
+        return 'Unknown test name: %s' % test_name
 
     error_parsers = []
     for parser_cls in _TEST_NAME_TO_PARSERS[test_name]:
@@ -176,5 +163,5 @@ def main():
                 print(error_msg)
 
 
-if __name__ == "__main__":
+if __name__ == '__main__':
     sys.exit(main())

build_tools/fb_compile_mongo.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+
 # fail early
 set -e
 

build_tools/fbcode_config.sh

@@ -1,5 +1,4 @@
 #!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 #
 # Set environment variables so that we can compile rocksdb using
 # fbcode settings.  It uses the latest g++ and clang compilers and also
@@ -21,47 +20,33 @@ LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
 GLIBC_INCLUDE="$GLIBC_BASE/include"
 GLIBC_LIBS=" -L $GLIBC_BASE/lib"
 
-if ! test $ROCKSDB_DISABLE_SNAPPY; then
-  # snappy
-  SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-  if test -z $PIC_BUILD; then
-    SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
-  else
-    SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
-  fi
-  CFLAGS+=" -DSNAPPY"
+# snappy
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
+if test -z $PIC_BUILD; then
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+else
+  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy_pic.a"
 fi
+CFLAGS+=" -DSNAPPY"
 
 if test -z $PIC_BUILD; then
-  if ! test $ROCKSDB_DISABLE_ZLIB; then
-    # location of zlib headers and libraries
-    ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-    ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
-    CFLAGS+=" -DZLIB"
-  fi
+  # location of zlib headers and libraries
+  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
+  ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+  CFLAGS+=" -DZLIB"
 
-  if ! test $ROCKSDB_DISABLE_BZIP; then
-    # location of bzip headers and libraries
-    BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-    BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
-    CFLAGS+=" -DBZIP2"
-  fi
+  # location of bzip headers and libraries
+  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
+  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+  CFLAGS+=" -DBZIP2"
 
-  if ! test $ROCKSDB_DISABLE_LZ4; then
-    LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-    LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
-    CFLAGS+=" -DLZ4"
-  fi
-fi
+  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
+  LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+  CFLAGS+=" -DLZ4"
 
-if ! test $ROCKSDB_DISABLE_ZSTD; then
   ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-  if test -z $PIC_BUILD; then
-    ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
-  else
-    ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd_pic.a"
-  fi
-  CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY"
+  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+  CFLAGS+=" -DZSTD"
 fi
 
 # location of gflags headers and libraries
@@ -96,10 +81,9 @@ else
 fi
 CFLAGS+=" -DTBB"
 
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+export PORTABLE=1
 
 BINUTILS="$BINUTILS_BASE/bin"
 AR="$BINUTILS/ar"
@@ -113,13 +97,12 @@ CLANG_LIB="$CLANG_BASE/lib"
 CLANG_SRC="$CLANG_BASE/../../src"
 
 CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_BIN/scan-build
+CLANG_SCAN_BUILD="$CLANG_SRC/llvm/tools/clang/tools/scan-build/bin/scan-build"
 
 if [ -z "$USE_CLANG" ]; then
   # gcc
   CC="$GCC_BASE/bin/gcc"
   CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
 
   CFLAGS+=" -B$BINUTILS/gold"
   CFLAGS+=" -isystem $GLIBC_INCLUDE"
@@ -130,7 +113,6 @@ else
   CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
   CC="$CLANG_BIN/clang"
   CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
 
   KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
 
@@ -147,7 +129,7 @@ else
 fi
 
 CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
 CXXFLAGS+=" $CFLAGS"
 
 EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
@@ -164,4 +146,12 @@ EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GF
 
 VALGRIND_VER="$VALGRIND_BASE/bin/"
 
-export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD
+LUA_PATH="$LUA_BASE"
+
+if test -z $PIC_BUILD; then
+  LUA_LIB=" $LUA_PATH/lib/liblua.a"
+else
+  LUA_LIB=" $LUA_PATH/lib/liblua_pic.a"
+fi
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD LUA_PATH LUA_LIB

build_tools/fbcode_config4.8.1.sh

@@ -0,0 +1,116 @@
+#!/bin/sh
+#
+# Set environment variables so that we can compile rocksdb using
+# fbcode settings.  It uses the latest g++ compiler and also
+# uses jemalloc
+
+BASEDIR=`dirname $BASH_SOURCE`
+source "$BASEDIR/dependencies_4.8.1.sh"
+
+# location of libgcc
+LIBGCC_INCLUDE="$LIBGCC_BASE/include"
+LIBGCC_LIBS=" -L $LIBGCC_BASE/lib"
+
+# location of glibc
+GLIBC_INCLUDE="$GLIBC_BASE/include"
+GLIBC_LIBS=" -L $GLIBC_BASE/lib"
+
+# location of snappy headers and libraries
+SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include"
+SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy.a"
+
+# location of zlib headers and libraries
+ZLIB_INCLUDE=" -I $ZLIB_BASE/include"
+ZLIB_LIBS=" $ZLIB_BASE/lib/libz.a"
+
+# location of bzip headers and libraries
+BZIP2_INCLUDE=" -I $BZIP2_BASE/include/"
+BZIP2_LIBS=" $BZIP2_BASE/lib/libbz2.a"
+
+LZ4_INCLUDE=" -I $LZ4_BASE/include"
+LZ4_LIBS=" $LZ4_BASE/lib/liblz4.a"
+
+ZSTD_INCLUDE=" -I $ZSTD_BASE/include"
+ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd.a"
+
+# location of gflags headers and libraries
+GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
+GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags.a"
+
+# location of jemalloc
+JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include"
+JEMALLOC_LIB="$JEMALLOC_BASE/lib/libjemalloc.a"
+
+# location of numa
+NUMA_INCLUDE=" -I $NUMA_BASE/include/"
+NUMA_LIB=" $NUMA_BASE/lib/libnuma.a"
+
+# location of libunwind
+LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind.a"
+
+# location of tbb
+TBB_INCLUDE=" -isystem $TBB_BASE/include/"
+TBB_LIBS="$TBB_BASE/lib/libtbb.a"
+
+# use Intel SSE support for checksum calculations
+export USE_SSE=1
+export PORTABLE=1
+
+BINUTILS="$BINUTILS_BASE/bin"
+AR="$BINUTILS/ar"
+
+DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP2_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE"
+
+STDLIBS="-L $GCC_BASE/lib64"
+
+if [ -z "$USE_CLANG" ]; then
+  # gcc
+  CC="$GCC_BASE/bin/gcc"
+  CXX="$GCC_BASE/bin/g++"
+
+  CFLAGS="-B$BINUTILS/gold -m64 -mtune=generic"
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  JEMALLOC=1
+else
+  # clang
+  CLANG_BIN="$CLANG_BASE/bin"
+  CLANG_LIB="$CLANG_BASE/lib"
+  CLANG_INCLUDE="$CLANG_LIB/clang/*/include"
+  CC="$CLANG_BIN/clang"
+  CXX="$CLANG_BIN/clang++"
+
+  KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include/"
+
+  CFLAGS="-B$BINUTILS/gold -nostdinc -nostdlib"
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1 "
+  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/4.8.1/x86_64-facebook-linux "
+  CFLAGS+=" -isystem $GLIBC_INCLUDE"
+  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
+  CFLAGS+=" -isystem $CLANG_INCLUDE"
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
+  CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
+  CXXFLAGS="-nostdinc++"
+fi
+
+CFLAGS+=" $DEPS_INCLUDE"
+CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_SUPPORT_THREAD_LOCAL -DHAVE_SSE42"
+CFLAGS+=" -DSNAPPY -DGFLAGS=google -DZLIB -DBZIP2 -DLZ4 -DZSTD -DNUMA -DTBB"
+CXXFLAGS+=" $CFLAGS"
+
+EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS"
+EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib/ld.so"
+EXEC_LDFLAGS+=" $LIBUNWIND"
+EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/gcc-4.8.1-glibc-2.17/lib"
+# required by libtbb
+EXEC_LDFLAGS+=" -ldl"
+
+PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
+
+EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP2_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS"
+
+VALGRIND_VER="$VALGRIND_BASE/bin/"
+
+LUA_PATH="$LUA_BASE"
+
+export CC CXX AR CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE LUA_PATH

build_tools/fbcode_config_platform010.sh

@@ -1,175 +0,0 @@
-#!/bin/sh
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-#
-# Set environment variables so that we can compile rocksdb using
-# fbcode settings.  It uses the latest g++ and clang compilers and also
-# uses jemalloc
-# Environment variables that change the behavior of this script:
-# PIC_BUILD -- if true, it will only take pic versions of libraries from fbcode. libraries that don't have pic variant will not be included
-
-
-BASEDIR=`dirname $BASH_SOURCE`
-source "$BASEDIR/dependencies_platform010.sh"
-
-# Disallow using libraries from default locations as they might not be compatible with platform010 libraries.
-CFLAGS=" --sysroot=/DOES/NOT/EXIST"
-
-# libgcc
-LIBGCC_INCLUDE="$LIBGCC_BASE/include/c++/trunk"
-LIBGCC_LIBS=" -L $LIBGCC_BASE/lib -B$LIBGCC_BASE/lib/gcc/x86_64-facebook-linux/trunk/"
-
-# glibc
-GLIBC_INCLUDE="$GLIBC_BASE/include"
-GLIBC_LIBS=" -L $GLIBC_BASE/lib"
-GLIBC_LIBS+=" -B$GLIBC_BASE/lib"
-
-if test -z $PIC_BUILD; then
-  MAYBE_PIC=
-else
-  MAYBE_PIC=_pic
-fi
-
-if ! test $ROCKSDB_DISABLE_SNAPPY; then
-  # snappy
-  SNAPPY_INCLUDE=" -I $SNAPPY_BASE/include/"
-  SNAPPY_LIBS=" $SNAPPY_BASE/lib/libsnappy${MAYBE_PIC}.a"
-  CFLAGS+=" -DSNAPPY"
-fi
-
-if ! test $ROCKSDB_DISABLE_ZLIB; then
-  # location of zlib headers and libraries
-  ZLIB_INCLUDE=" -I $ZLIB_BASE/include/"
-  ZLIB_LIBS=" $ZLIB_BASE/lib/libz${MAYBE_PIC}.a"
-  CFLAGS+=" -DZLIB"
-fi
-
-if ! test $ROCKSDB_DISABLE_BZIP; then
-  # location of bzip headers and libraries
-  BZIP_INCLUDE=" -I $BZIP2_BASE/include/"
-  BZIP_LIBS=" $BZIP2_BASE/lib/libbz2${MAYBE_PIC}.a"
-  CFLAGS+=" -DBZIP2"
-fi
-
-if ! test $ROCKSDB_DISABLE_LZ4; then
-  LZ4_INCLUDE=" -I $LZ4_BASE/include/"
-  LZ4_LIBS=" $LZ4_BASE/lib/liblz4${MAYBE_PIC}.a"
-  CFLAGS+=" -DLZ4"
-fi
-
-if ! test $ROCKSDB_DISABLE_ZSTD; then
-  ZSTD_INCLUDE=" -I $ZSTD_BASE/include/"
-  ZSTD_LIBS=" $ZSTD_BASE/lib/libzstd${MAYBE_PIC}.a"
-  CFLAGS+=" -DZSTD -DZSTD_STATIC_LINKING_ONLY"
-fi
-
-# location of gflags headers and libraries
-GFLAGS_INCLUDE=" -I $GFLAGS_BASE/include/"
-GFLAGS_LIBS=" $GFLAGS_BASE/lib/libgflags${MAYBE_PIC}.a"
-CFLAGS+=" -DGFLAGS=gflags"
-
-BENCHMARK_INCLUDE=" -I $BENCHMARK_BASE/include/"
-BENCHMARK_LIBS=" $BENCHMARK_BASE/lib/libbenchmark${MAYBE_PIC}.a"
-
-# location of jemalloc
-JEMALLOC_INCLUDE=" -I $JEMALLOC_BASE/include/"
-JEMALLOC_LIB=" $JEMALLOC_BASE/lib/libjemalloc${MAYBE_PIC}.a"
-
-# location of numa
-NUMA_INCLUDE=" -I $NUMA_BASE/include/"
-NUMA_LIB=" $NUMA_BASE/lib/libnuma${MAYBE_PIC}.a"
-CFLAGS+=" -DNUMA"
-
-# location of libunwind
-LIBUNWIND="$LIBUNWIND_BASE/lib/libunwind${MAYBE_PIC}.a"
-
-# location of TBB
-TBB_INCLUDE=" -isystem $TBB_BASE/include/"
-TBB_LIBS="$TBB_BASE/lib/libtbb${MAYBE_PIC}.a"
-CFLAGS+=" -DTBB"
-
-# location of LIBURING
-LIBURING_INCLUDE=" -isystem $LIBURING_BASE/include/"
-LIBURING_LIBS="$LIBURING_BASE/lib/liburing${MAYBE_PIC}.a"
-CFLAGS+=" -DLIBURING"
-
-test "$USE_SSE" || USE_SSE=1
-export USE_SSE
-test "$PORTABLE" || PORTABLE=1
-export PORTABLE
-
-BINUTILS="$BINUTILS_BASE/bin"
-AR="$BINUTILS/ar"
-AS="$BINUTILS/as"
-
-DEPS_INCLUDE="$SNAPPY_INCLUDE $ZLIB_INCLUDE $BZIP_INCLUDE $LZ4_INCLUDE $ZSTD_INCLUDE $GFLAGS_INCLUDE $NUMA_INCLUDE $TBB_INCLUDE $LIBURING_INCLUDE $BENCHMARK_INCLUDE"
-
-STDLIBS="-L $GCC_BASE/lib64"
-
-CLANG_BIN="$CLANG_BASE/bin"
-CLANG_LIB="$CLANG_BASE/lib"
-CLANG_SRC="$CLANG_BASE/../../src"
-
-CLANG_ANALYZER="$CLANG_BIN/clang++"
-CLANG_SCAN_BUILD="$CLANG_BIN/scan-build"
-
-if [ -z "$USE_CLANG" ]; then
-  # gcc
-  CC="$GCC_BASE/bin/gcc"
-  CXX="$GCC_BASE/bin/g++"
-  AR="$GCC_BASE/bin/gcc-ar"
-
-  CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
-  CFLAGS+=" -I$GCC_BASE/include"
-  CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/include"
-  CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/install-tools/include"
-  CFLAGS+=" -isystem $GCC_BASE/lib/gcc/x86_64-redhat-linux-gnu/11.2.1/include-fixed/"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -I$GLIBC_INCLUDE"
-  CFLAGS+=" -I$LIBGCC_BASE/include"
-  CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/"
-  CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/x86_64-facebook-linux/"
-  CFLAGS+=" -I$LIBGCC_BASE/include/c++/11.x/backward"
-  CFLAGS+=" -isystem $GLIBC_INCLUDE -I$GLIBC_INCLUDE"
-  JEMALLOC=1
-else
-  # clang
-  CLANG_INCLUDE="$CLANG_LIB/clang/stable/include"
-  CC="$CLANG_BIN/clang"
-  CXX="$CLANG_BIN/clang++"
-  AR="$CLANG_BIN/llvm-ar"
-
-  CFLAGS+=" -B$BINUTILS -nostdinc -nostdlib"
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/trunk "
-  CFLAGS+=" -isystem $LIBGCC_BASE/include/c++/trunk/x86_64-facebook-linux "
-  CFLAGS+=" -isystem $GLIBC_INCLUDE"
-  CFLAGS+=" -isystem $LIBGCC_INCLUDE"
-  CFLAGS+=" -isystem $CLANG_INCLUDE"
-  CFLAGS+=" -Wno-expansion-to-defined "
-  CXXFLAGS="-nostdinc++"
-fi
-
-KERNEL_HEADERS_INCLUDE="$KERNEL_HEADERS_BASE/include"
-CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE/linux "
-CFLAGS+=" -isystem $KERNEL_HEADERS_INCLUDE "
-
-CFLAGS+=" $DEPS_INCLUDE"
-CFLAGS+=" -DROCKSDB_PLATFORM_POSIX -DROCKSDB_LIB_IO_POSIX -DROCKSDB_FALLOCATE_PRESENT -DROCKSDB_MALLOC_USABLE_SIZE -DROCKSDB_RANGESYNC_PRESENT -DROCKSDB_SCHED_GETCPU_PRESENT -DROCKSDB_IOURING_PRESENT"
-CXXFLAGS+=" $CFLAGS"
-
-EXEC_LDFLAGS=" $SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $NUMA_LIB $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS"
-EXEC_LDFLAGS+=" -Wl,--dynamic-linker,/usr/local/fbcode/platform010/lib/ld.so"
-EXEC_LDFLAGS+=" $LIBUNWIND"
-EXEC_LDFLAGS+=" -Wl,-rpath=/usr/local/fbcode/platform010/lib"
-EXEC_LDFLAGS+=" -Wl,-rpath=$GCC_BASE/lib64"
-# required by libtbb
-EXEC_LDFLAGS+=" -ldl"
-
-PLATFORM_LDFLAGS="$LIBGCC_LIBS $GLIBC_LIBS $STDLIBS -lgcc -lstdc++"
-PLATFORM_LDFLAGS+=" -B$BINUTILS"
-
-EXEC_LDFLAGS_SHARED="$SNAPPY_LIBS $ZLIB_LIBS $BZIP_LIBS $LZ4_LIBS $ZSTD_LIBS $GFLAGS_LIBS $TBB_LIBS $LIBURING_LIBS $BENCHMARK_LIBS"
-
-VALGRIND_VER="$VALGRIND_BASE/bin/"
-
-export CC CXX AR AS CFLAGS CXXFLAGS EXEC_LDFLAGS EXEC_LDFLAGS_SHARED VALGRIND_VER JEMALLOC_LIB JEMALLOC_INCLUDE CLANG_ANALYZER CLANG_SCAN_BUILD

build_tools/format-diff.sh

@@ -1,106 +1,36 @@
 #!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # If clang_format_diff.py command is not specfied, we assume we are able to
 # access directly without any path.
+if [ -z $CLANG_FORMAT_DIFF ]
+then
+CLANG_FORMAT_DIFF="clang-format-diff.py"
+fi
 
-print_usage () {
-  echo "Usage:"
-  echo "format-diff.sh [OPTIONS]"
-  echo "-c: check only."
-  echo "-y: auto-apply formatting without prompts (non-interactive mode)."
-  echo "-h: print this message."
-}
+# Check clang-format-diff.py
+if ! which $CLANG_FORMAT_DIFF &> /dev/null
+then
+  echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
+  echo "You can download clang-format-diff.py by running: "
+  echo "    curl --location http://goo.gl/iUW1u2 -o ${CLANG_FORMAT_DIFF}"
+  echo "You can download clang-format by running: "
+  echo "    brew install clang-format"
+  echo "Then, move both files (i.e. ${CLANG_FORMAT_DIFF} and clang-format) to some directory within PATH=${PATH}"
+  exit 128
+fi
 
-while getopts ':cyh' OPTION; do
-  case "$OPTION" in
-    c)
-      CHECK_ONLY=1
-      ;;
-    y)
-      AUTO_APPLY=1
-      ;;
-    h)
-      print_usage
-      exit 1
-      ;;
-    ?)
-      print_usage
-      exit 1
-      ;;
-  esac
-done
+# Check argparse, a library that clang-format-diff.py requires.
+python 2>/dev/null << EOF
+import argparse
+EOF
 
-REPO_ROOT="$(git rev-parse --show-toplevel)"
-
-if [ "$CLANG_FORMAT_DIFF" ]; then
-  echo "Note: CLANG_FORMAT_DIFF='$CLANG_FORMAT_DIFF'"
-  # Dry run to confirm dependencies like argparse
-  if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-    true #Good
-  else
-    exit 128
-  fi
-else
-  # First try directly executing the possibilities
-  if clang-format-diff --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=clang-format-diff
-  elif clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=clang-format-diff.py
-  elif $REPO_ROOT/clang-format-diff.py --help &> /dev/null < /dev/null; then
-    CLANG_FORMAT_DIFF=$REPO_ROOT/clang-format-diff.py
-  else
-    # This probably means we need to directly invoke the interpreter.
-    # But first find clang-format-diff.py
-    if [ -f "$REPO_ROOT/clang-format-diff.py" ]; then
-      CFD_PATH="$REPO_ROOT/clang-format-diff.py"
-    elif which clang-format-diff.py &> /dev/null; then
-      CFD_PATH="$(which clang-format-diff.py)"
-    else
-      echo "You didn't have clang-format-diff.py and/or clang-format available in your computer!"
-      echo "You can download clang-format-diff.py by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      echo "You can download clang-format by running:"
-      echo "    brew install clang-format"
-      echo "  Or"
-      echo "    apt install clang-format"
-      echo "  This might work too:"
-      echo "    yum install git-clang-format"
-      echo "Then make sure clang-format is available and executable from \$PATH:"
-      echo "    clang-format --version"
-      exit 128
-    fi
-    # Check argparse pre-req on interpreter, or it will fail
-    if echo import argparse | ${PYTHON:-python3}; then
-      true # Good
-    else
-      echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
-      echo "installed. You can try either of the follow ways to install it:"
-      echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
-      echo "  2. easy_install argparse (if you have easy_install)"
-      echo "  3. pip install argparse (if you have pip)"
-      exit 129
-    fi
-    # Unfortunately, some machines have a Python2 clang-format-diff.py
-    # installed but only a Python3 interpreter installed. Unfortunately,
-    # automatic 2to3 migration is insufficient, so suggest downloading latest.
-    if grep -q "print '" "$CFD_PATH" && \
-       ${PYTHON:-python3} --version | grep -q 'ython 3'; then
-      echo "You have clang-format-diff.py for Python 2 but are using a Python 3"
-      echo "interpreter (${PYTHON:-python3})."
-      echo "You can download clang-format-diff.py for Python 3 by running: "
-      echo "    curl --location https://raw.githubusercontent.com/llvm/llvm-project/main/clang/tools/clang-format/clang-format-diff.py -o ${REPO_ROOT}/clang-format-diff.py"
-      echo "You should make sure the downloaded script is not compromised."
-      exit 130
-    fi
-    CLANG_FORMAT_DIFF="${PYTHON:-python3} $CFD_PATH"
-    # This had better work after all those checks
-    if $CLANG_FORMAT_DIFF --help >/dev/null < /dev/null; then
-      true #Good
-    else
-      exit 128
-    fi
-  fi
+if [ "$?" != 0 ]
+then
+  echo "To run clang-format-diff.py, we'll need the library "argparse" to be"
+  echo "installed. You can try either of the follow ways to install it:"
+  echo "  1. Manually download argparse: https://pypi.python.org/pypi/argparse"
+  echo "  2. easy_install argparse (if you have easy_install)"
+  echo "  3. pip install argparse (if you have pip)"
+  exit 129
 fi
 
 # TODO(kailiu) following work is not complete since we still need to figure
@@ -122,139 +52,49 @@ fi
 # fi
 set -e
 
-# Exclude third-party from formatting
-EXCLUDE=':!third-party/'
-
 uncommitted_code=`git diff HEAD`
+LAST_MASTER=`git merge-base master HEAD`
 
 # If there's no uncommitted changes, we assume user are doing post-commit
-# format check, in which case we'll try to check the modified lines vs. the
-# facebook/rocksdb.git main branch. Otherwise, we'll check format of the
-# uncommitted code only.
+# format check, in which case we'll check the modified lines since last commit
+# from master. Otherwise, we'll check format of the uncommitted code only.
 if [ -z "$uncommitted_code" ]
 then
-  # Attempt to get name of facebook/rocksdb.git remote.
-  [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE="$(LC_ALL=POSIX LANG=POSIX git remote -v | grep 'facebook/rocksdb.git' | head -n 1 | cut -f 1)"
-  # Fall back on 'origin' if that fails
-  [ "$FORMAT_REMOTE" ] || FORMAT_REMOTE=origin
-  # Use main branch from that remote
-  [ "$FORMAT_UPSTREAM" ] || FORMAT_UPSTREAM="$FORMAT_REMOTE/$(LC_ALL=POSIX LANG=POSIX git remote show $FORMAT_REMOTE | sed -n '/HEAD branch/s/.*: //p')"
-  # Get the common ancestor with that remote branch. Everything after that
-  # common ancestor would be considered the contents of a pull request, so
-  # should be relevant for formatting fixes.
-  FORMAT_UPSTREAM_MERGE_BASE="$(git merge-base "$FORMAT_UPSTREAM" HEAD)"
-  # Get the differences
-  diffs=$(git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" -- $EXCLUDE | $CLANG_FORMAT_DIFF -p 1) || true
-  echo "Checking format of changes not yet in $FORMAT_UPSTREAM..."
+  # Check the format of last commit
+  diffs=$(git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -p 1)
 else
   # Check the format of uncommitted lines,
-  diffs=$(git diff -U0 HEAD -- $EXCLUDE | $CLANG_FORMAT_DIFF -p 1) || true
-  echo "Checking format of uncommitted changes..."
-fi
-
-# Check for missing copyright in new files
-echo "Checking for copyright headers in new files..."
-
-# Get list of new files (added, not just modified)
-if [ -z "$uncommitted_code" ]; then
-  # Post-commit: check files added since merge base
-  new_files=$(git diff --name-only --diff-filter=A "$FORMAT_UPSTREAM_MERGE_BASE" -- '*.h' '*.cc' '*.py' $EXCLUDE)
-else
-  # Pre-commit: check staged new files
-  new_files=$(git diff --name-only --diff-filter=A --cached HEAD -- '*.h' '*.cc' '*.py' $EXCLUDE)
-fi
-
-if [ -n "$new_files" ]; then
-  files_missing_copyright=""
-
-  for file in $new_files; do
-    if [ -f "$file" ]; then
-      # Check if file is missing copyright
-      # For .py files, check for Python-style comment
-      # For .h and .cc files, check for C++-style comment
-      if [[ "$file" == *.py ]]; then
-        if ! grep -q "Copyright (c) Meta Platforms, Inc. and affiliates" "$file"; then
-          files_missing_copyright="$files_missing_copyright $file"
-          # Add copyright header to Python file
-          temp_file=$(mktemp)
-          {
-            echo "#  Copyright (c) Meta Platforms, Inc. and affiliates."
-            echo "#  This source code is licensed under both the GPLv2 (found in the COPYING file in the root directory)"
-            echo "#  and the Apache 2.0 License (found in the LICENSE.Apache file in the root directory)."
-            echo
-            cat "$file"
-          } > "$temp_file"
-          mv "$temp_file" "$file"
-          echo "Added copyright header to $file"
-        fi
-      elif [[ "$file" == *.h ]] || [[ "$file" == *.cc ]]; then
-        if ! grep -q "Copyright (c) Meta Platforms, Inc. and affiliates" "$file"; then
-          files_missing_copyright="$files_missing_copyright $file"
-          # Add copyright header to C++ file
-          temp_file=$(mktemp)
-          {
-            echo "//  Copyright (c) Meta Platforms, Inc. and affiliates. "
-            echo "//  This source code is licensed under both the GPLv2 (found in the "
-            echo "//  COPYING file in the root directory) and Apache 2.0 License "
-            echo "//  (found in the LICENSE.Apache file in the root directory)."
-            echo
-            cat "$file"
-          } > "$temp_file"
-          mv "$temp_file" "$file"
-          echo "Added copyright header to $file"
-        fi
-      fi
-    fi
-  done
-
-  if [ -n "$files_missing_copyright" ]; then
-    echo "Copyright headers were added to new files."
-  else
-    echo "All new files have copyright headers."
-  fi
-else
-  echo "No new files to check for copyright headers."
+  diffs=$(git diff -U0 HEAD | $CLANG_FORMAT_DIFF -p 1)
 fi
 
 if [ -z "$diffs" ]
 then
   echo "Nothing needs to be reformatted!"
   exit 0
-elif [ $? -ne 1 ]; then
-  # CLANG_FORMAT_DIFF will exit on 1 while there is suggested changes.
-  exit $?
-elif [ $CHECK_ONLY ]
-then
-  echo "Your change has unformatted code. Please run make format!"
-  if [ $VERBOSE_CHECK ]; then
-    clang-format --version
-    echo "$diffs"
-  fi
-  exit 1
 fi
 
 # Highlight the insertion/deletion from the clang-format-diff.py's output
 COLOR_END="\033[0m"
-COLOR_RED="\033[0;31m"
-COLOR_GREEN="\033[0;32m"
+COLOR_RED="\033[0;31m" 
+COLOR_GREEN="\033[0;32m" 
 
 echo -e "Detect lines that doesn't follow the format rules:\r"
 # Add the color to the diff. lines added will be green; lines removed will be red.
-echo "$diffs" |
+echo "$diffs" | 
   sed -e "s/\(^-.*$\)/`echo -e \"$COLOR_RED\1$COLOR_END\"`/" |
   sed -e "s/\(^+.*$\)/`echo -e \"$COLOR_GREEN\1$COLOR_END\"`/"
 
-# Handle auto-apply mode (non-interactive)
-if [ "$AUTO_APPLY" ]; then
-  to_fix="y"
-else
-  echo -e "Would you like to fix the format automatically (y/n): \c"
-
-  # Make sure under any mode, we can read user input.
-  exec < /dev/tty
-  read to_fix
+if [[ "$OPT" == *"-DTRAVIS"* ]]
+then
+  exit 1
 fi
 
+echo -e "Would you like to fix the format automatically (y/n): \c"
+
+# Make sure under any mode, we can read user input.
+exec < /dev/tty
+read to_fix
+
 if [ "$to_fix" != "y" ]
 then
   exit 1
@@ -263,15 +103,14 @@ fi
 # Do in-place format adjustment.
 if [ -z "$uncommitted_code" ]
 then
-  git diff -U0 "$FORMAT_UPSTREAM_MERGE_BASE" -- $EXCLUDE | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 $LAST_MASTER^ | $CLANG_FORMAT_DIFF -i -p 1
 else
-  git diff -U0 HEAD -- $EXCLUDE | $CLANG_FORMAT_DIFF -i -p 1
+  git diff -U0 HEAD^ | $CLANG_FORMAT_DIFF -i -p 1
 fi
 echo "Files reformatted!"
 
 # Amend to last commit if user do the post-commit format check
-# Skip amend prompt in auto-apply mode (user can amend manually if desired)
-if [ -z "$uncommitted_code" ] && [ -z "$AUTO_APPLY" ]; then
+if [ -z "$uncommitted_code" ]; then
   echo -e "Would you like to amend the changes to last commit (`git log HEAD --oneline | head -1`)? (y/n): \c"
   read to_amend
 

build_tools/getdeps_fallback_mirror.py

@@ -1,127 +0,0 @@
-#  Copyright (c) Meta Platforms, Inc. and affiliates.
-#  This source code is licensed under both the GPLv2 (found in the COPYING file in the root directory)
-#  and the Apache 2.0 License (found in the LICENSE.Apache file in the root directory).
-
-#!/usr/bin/env python3
-"""
-Pre-download packages with unreliable mirrors using fallback mirrors.
-Reads package info from folly's getdeps manifest files.
-"""
-import sys
-import os
-import hashlib
-import subprocess
-import configparser
-
-def sha256_file(path):
-    """Calculate SHA256 hash of a file."""
-    h = hashlib.sha256()
-    try:
-        with open(path, 'rb') as f:
-            for chunk in iter(lambda: f.read(65536), b''):
-                h.update(chunk)
-        return h.hexdigest()
-    except Exception:
-        return None
-
-def parse_manifest(manifest_path):
-    """Parse a getdeps manifest file to extract download info."""
-    config = configparser.ConfigParser()
-    try:
-        config.read(manifest_path)
-        if 'download' in config:
-            return {
-                'url': config['download'].get('url', ''),
-                'sha256': config['download'].get('sha256', ''),
-            }
-    except Exception:
-        pass
-    return None
-
-def get_fallback_mirrors(url):
-    """Get fallback mirror URLs for a given URL."""
-    # Fallback mirror patterns for known unreliable hosts
-    mirror_fallbacks = {
-        "ftp.gnu.org/gnu/": [
-            "https://mirrors.kernel.org/gnu/",
-            "https://ftpmirror.gnu.org/gnu/",
-            "https://ftp.gnu.org/gnu/",
-        ],
-        "ftpmirror.gnu.org/gnu/": [
-            "https://mirrors.kernel.org/gnu/",
-            "https://ftpmirror.gnu.org/gnu/",
-            "https://ftp.gnu.org/gnu/",
-        ],
-    }
-
-    for pattern, mirrors in mirror_fallbacks.items():
-        if pattern in url:
-            # Extract the path after the pattern
-            path_start = url.find(pattern) + len(pattern)
-            path = url[path_start:]
-            return [mirror + path for mirror in mirrors]
-    return [url]  # No fallback, use original
-
-def main():
-    if len(sys.argv) != 4:
-        print(f"Usage: {sys.argv[0]} <download_dir> <cache_dir> <manifests_dir>")
-        sys.exit(1)
-
-    download_dir, cache_dir, manifests_dir = sys.argv[1], sys.argv[2], sys.argv[3]
-
-    # Packages known to have unreliable mirrors
-    packages_to_check = ["autoconf", "automake", "libtool"]
-
-    for package in packages_to_check:
-        manifest_path = os.path.join(manifests_dir, package)
-        if not os.path.exists(manifest_path):
-            continue
-
-        info = parse_manifest(manifest_path)
-        if not info or not info['url'] or not info['sha256']:
-            continue
-
-        # Determine filename from URL
-        url = info['url']
-        expected_sha256 = info['sha256']
-        url_filename = os.path.basename(url)
-
-        # getdeps uses format: {package}-{filename}
-        filename = f"{package}-{url_filename}"
-        filepath = os.path.join(download_dir, filename)
-        cache_path = os.path.join(cache_dir, filename)
-
-        # Check if already valid
-        if os.path.exists(filepath) and sha256_file(filepath) == expected_sha256:
-            print(f"  {filename}: OK (already downloaded)")
-            continue
-
-        # Check cache
-        if os.path.exists(cache_path) and sha256_file(cache_path) == expected_sha256:
-            print(f"  {filename}: OK (from cache)")
-            subprocess.run(['cp', cache_path, filepath], check=True)
-            continue
-
-        # Try fallback mirrors
-        mirrors = get_fallback_mirrors(url)
-        downloaded = False
-        for mirror_url in mirrors:
-            print(f"  {filename}: trying {mirror_url}...")
-            try:
-                subprocess.run(['wget', '-q', '-O', filepath, mirror_url], check=True, timeout=120)
-                if sha256_file(filepath) == expected_sha256:
-                    print(f"  {filename}: OK (downloaded)")
-                    subprocess.run(['cp', filepath, cache_path], check=False)
-                    downloaded = True
-                    break
-                else:
-                    os.remove(filepath)
-            except Exception:
-                if os.path.exists(filepath):
-                    os.remove(filepath)
-
-        if not downloaded:
-            print(f"  {filename}: WARNING - all mirrors failed")
-
-if __name__ == "__main__":
-    main()

build_tools/gnu_parallel

@@ -1170,7 +1170,7 @@ sub parse_env_var {
     # Check if any variables contain \n
     if(my @v = map { s/BASH_FUNC_(.*)\(\)/$1/; $_ } grep { $ENV{$_}=~/\n/ } @vars) {
 	# \n is bad for csh and will cause it to fail.
-	$Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | grep -E "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn;
+	$Global::envwarn = ::shell_quote_scalar(q{echo $SHELL | egrep "/t?csh" > /dev/null && echo CSH/TCSH DO NOT SUPPORT newlines IN VARIABLES/FUNCTIONS. Unset }."@v".q{ && exec false;}."\n\n") . $Global::envwarn;
     }
 
     if(not @qcsh) { push @qcsh, "true"; }
@@ -1561,7 +1561,6 @@ sub save_stdin_stdout_stderr {
 	::die_bug("Can't dup STDERR: $!");
     open $Global::original_stdin, "<&", "STDIN" or
 	::die_bug("Can't dup STDIN: $!");
-    $Global::is_terminal = (-t $Global::original_stderr) && !$ENV{'CIRCLECI'}&& !$ENV{'GITHUB_ACTIONS'}  && !$ENV{'TRAVIS'};
 }
 
 sub enough_file_handles {
@@ -1841,17 +1840,12 @@ sub start_another_job {
 }
 }
 
-$opt::min_progress_interval = 0;
-
 sub init_progress {
     # Uses:
     #   $opt::bar
     # Returns:
     #   list of computers for progress output
     $|=1;
-    if (not $Global::is_terminal) {
-      $opt::min_progress_interval = 30;
-    }
     if($opt::bar) {
 	return("","");
     }
@@ -1876,9 +1870,6 @@ sub drain_job_queue {
     }
     my $last_header="";
     my $sleep = 0.2;
-    my $last_left = 1000000000;
-    my $last_progress_time = 0;
-    my $ps_reported = 0;
     do {
         while($Global::total_running > 0) {
             debug($Global::total_running, "==", scalar
@@ -1889,39 +1880,14 @@ sub drain_job_queue {
 		    close $job->fh(0,"w");
 		}
 	    }
-            # When not connected to terminal, assume CI (e.g. CircleCI). In
-            # that case we want occasional progress output to prevent abort
-            # due to timeout with no output, but we also need to stop sending
-            # progress output if there has been no actual progress, so that
-            # the job can time out appropriately (CirecleCI: 10m) in case of
-            # a hung test. But without special output, it is extremely
-            # annoying to diagnose which test is hung, so we add that using
-            # `ps` below.
-            if($opt::progress and
-               ($Global::is_terminal or (time() - $last_progress_time) >= 30)) {
+            if($opt::progress) {
                 my %progress = progress();
                 if($last_header ne $progress{'header'}) {
                     print $Global::original_stderr "\n", $progress{'header'}, "\n";
                     $last_header = $progress{'header'};
                 }
-                if ($Global::is_terminal) {
-                    print $Global::original_stderr "\r",$progress{'status'};
-                }
-                if ($last_left > $Global::left) {
-                    if (not $Global::is_terminal) {
-                        print $Global::original_stderr $progress{'status'},"\n";
-                    }
-                    $last_progress_time = time();
-                    $ps_reported = 0;
-                } elsif (not $ps_reported and (time() - $last_progress_time) >= 300) {
-                    # No progress in at least 60 seconds: run ps
-                    print $Global::original_stderr "\n";
-                    my $script_dir = ::dirname($0);
-                    system("$script_dir/ps_with_stack || ps -wwf");
-                    $ps_reported = 1;
-                }
-                $last_left = $Global::left;
-                flush $Global::original_stderr;
+                print $Global::original_stderr "\r",$progress{'status'};
+		flush $Global::original_stderr;
             }
 	    if($Global::total_running < $Global::max_jobs_running
 	       and not $Global::JobQueue->empty()) {
@@ -1955,7 +1921,7 @@ sub drain_job_queue {
 	     not $Global::start_no_new_jobs and not $Global::JobQueue->empty());
     if($opt::progress) {
 	my %progress = progress();
-	print $Global::original_stderr $opt::progress_sep, $progress{'status'}, "\n";
+	print $Global::original_stderr "\r", $progress{'status'}, "\n";
 	flush $Global::original_stderr;
     }
 }
@@ -1988,11 +1954,10 @@ sub progress {
     my $eta = "";
     my ($status,$header)=("","");
     if($opt::eta) {
-      my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
-        compute_eta();
-      $eta = sprintf("ETA: %ds Left: %d AVG: %.2fs  ",
-        $this_eta, $left, $avgtime);
-      $Global::left = $left;
+	my($total, $completed, $left, $pctcomplete, $avgtime, $this_eta) =
+	    compute_eta();
+	$eta = sprintf("ETA: %ds Left: %d AVG: %.2fs  ",
+		       $this_eta, $left, $avgtime);
     }
     my $termcols = terminal_columns();
     my @workers = sort keys %Global::host;
@@ -5117,8 +5082,8 @@ sub openoutputfiles {
 	# Set reading FD if using --group (--ungroup does not need)
 	for my $fdno (1,2) {
 	    # Re-open the file for reading
-	    # so fdw can be closed separately
-	    # and fdr can be seeked separately (for --line-buffer)
+	    # so fdw can be closed seperately
+	    # and fdr can be seeked seperately (for --line-buffer)
 	    open(my $fdr,"<", $self->fh($fdno,'name')) ||
 		::die_bug("fdr: Cannot open ".$self->fh($fdno,'name'));
 	    $self->set_fh($fdno,'r',$fdr);
@@ -5836,7 +5801,7 @@ sub workdir {
 		    . "-" . $self->seq();
 	    } else {
 		$workdir = $opt::workdir;
-		# Rsync treats /./ special. We don't want that
+		# Rsync treats /./ special. We dont want that
 		$workdir =~ s:/\./:/:g; # Remove /./
 		$workdir =~ s:/+$::; # Remove ending / if any
 		$workdir =~ s:^\./::g; # Remove starting ./ if any

build_tools/make_package.sh

@@ -1,6 +1,5 @@
 # shellcheck disable=SC1113
 #/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 set -e
 
@@ -103,26 +102,31 @@ function main() {
   gem_install fpm
 
   make static_lib
-  LIBDIR=/usr/lib
-  if [[ $FPM_OUTPUT = "rpm" ]]; then
-      LIBDIR=$(rpm --eval '%_libdir')
-  fi
+  make install INSTALL_PATH=package
 
-  rm -rf package
-  make install DESTDIR=package PREFIX=/usr LIBDIR=$LIBDIR
+  cd package
+
+  LIB_DIR=lib
+  if [[ -z "$ARCH" ]]; then
+      ARCH=$(getconf LONG_BIT)
+  fi
+  if [[ ("$FPM_OUTPUT" = "rpm") && ($ARCH -eq 64) ]]; then
+      mv lib lib64
+      LIB_DIR=lib64
+  fi
 
   fpm \
     -s dir \
     -t $FPM_OUTPUT \
-    -C package \
     -n rocksdb \
     -v $1 \
+    --prefix /usr \
     --url http://rocksdb.org/ \
     -m rocksdb@fb.com \
     --license BSD \
     --vendor Facebook \
     --description "RocksDB is an embeddable persistent key-value store for fast storage." \
-    usr
+    include $LIB_DIR
 }
 
 # shellcheck disable=SC2068

build_tools/precommit_checker.py

@@ -0,0 +1,208 @@
+#!/usr/local/fbcode/gcc-4.9-glibc-2.20-fb/bin/python2.7
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+import argparse
+import commands
+import subprocess
+import sys
+import re
+import os
+import time
+
+
+#
+# Simple logger
+#
+
+class Log:
+
+    def __init__(self, filename):
+        self.filename = filename
+        self.f = open(self.filename, 'w+', 0)
+
+    def caption(self, str):
+        line = "\n##### %s #####\n" % str
+        if self.f:
+            self.f.write("%s \n" % line)
+        else:
+            print(line)
+
+    def error(self, str):
+        data = "\n\n##### ERROR ##### %s" % str
+        if self.f:
+            self.f.write("%s \n" % data)
+        else:
+            print(data)
+
+    def log(self, str):
+        if self.f:
+            self.f.write("%s \n" % str)
+        else:
+            print(str)
+
+#
+# Shell Environment
+#
+
+
+class Env(object):
+
+    def __init__(self, logfile, tests):
+        self.tests = tests
+        self.log = Log(logfile)
+
+    def shell(self, cmd, path=os.getcwd()):
+        if path:
+            os.chdir(path)
+
+        self.log.log("==== shell session ===========================")
+        self.log.log("%s> %s" % (path, cmd))
+        status = subprocess.call("cd %s; %s" % (path, cmd), shell=True,
+                                 stdout=self.log.f, stderr=self.log.f)
+        self.log.log("status = %s" % status)
+        self.log.log("============================================== \n\n")
+        return status
+
+    def GetOutput(self, cmd, path=os.getcwd()):
+        if path:
+            os.chdir(path)
+
+        self.log.log("==== shell session ===========================")
+        self.log.log("%s> %s" % (path, cmd))
+        status, out = commands.getstatusoutput(cmd)
+        self.log.log("status = %s" % status)
+        self.log.log("out = %s" % out)
+        self.log.log("============================================== \n\n")
+        return status, out
+
+#
+# Pre-commit checker
+#
+
+
+class PreCommitChecker(Env):
+
+    def __init__(self, args):
+        Env.__init__(self, args.logfile, args.tests)
+        self.ignore_failure = args.ignore_failure
+
+    #
+    #   Get commands for a given job from the determinator file
+    #
+    def get_commands(self, test):
+        status, out = self.GetOutput(
+            "RATIO=1 build_tools/rocksdb-lego-determinator %s" % test, ".")
+        return status, out
+
+    #
+    # Run a specific CI job
+    #
+    def run_test(self, test):
+        self.log.caption("Running test %s locally" % test)
+
+        # get commands for the CI job determinator
+        status, cmds = self.get_commands(test)
+        if status != 0:
+            self.log.error("Error getting commands for test %s" % test)
+            return False
+
+        # Parse the JSON to extract the commands to run
+        cmds = re.findall("'shell':'([^\']*)'", cmds)
+
+        if len(cmds) == 0:
+            self.log.log("No commands found")
+            return False
+
+        # Run commands
+        for cmd in cmds:
+            # Replace J=<..> with the local environment variable
+            if "J" in os.environ:
+                cmd = cmd.replace("J=1", "J=%s" % os.environ["J"])
+                cmd = cmd.replace("make ", "make -j%s " % os.environ["J"])
+            # Run the command
+            status = self.shell(cmd, ".")
+            if status != 0:
+                self.log.error("Error running command %s for test %s"
+                               % (cmd, test))
+                return False
+
+        return True
+
+    #
+    # Run specified CI jobs
+    #
+    def run_tests(self):
+        if not self.tests:
+            self.log.error("Invalid args. Please provide tests")
+            return False
+
+        self.print_separator()
+        self.print_row("TEST", "RESULT")
+        self.print_separator()
+
+        result = True
+        for test in self.tests:
+            start_time = time.time()
+            self.print_test(test)
+            result = self.run_test(test)
+            elapsed_min = (time.time() - start_time) / 60
+            if not result:
+                self.log.error("Error running test %s" % test)
+                self.print_result("FAIL (%dm)" % elapsed_min)
+                if not self.ignore_failure:
+                    return False
+                result = False
+            else:
+                self.print_result("PASS (%dm)" % elapsed_min)
+
+        self.print_separator()
+        return result
+
+    #
+    # Print a line
+    #
+    def print_separator(self):
+        print("".ljust(60, "-"))
+
+    #
+    # Print two colums
+    #
+    def print_row(self, c0, c1):
+        print("%s%s" % (c0.ljust(40), c1.ljust(20)))
+
+    def print_test(self, test):
+        print(test.ljust(40), end="")
+        sys.stdout.flush()
+
+    def print_result(self, result):
+        print(result.ljust(20))
+
+#
+# Main
+#
+parser = argparse.ArgumentParser(description='RocksDB pre-commit checker.')
+
+# --log <logfile>
+parser.add_argument('--logfile', default='/tmp/precommit-check.log',
+                    help='Log file. Default is /tmp/precommit-check.log')
+# --ignore_failure
+parser.add_argument('--ignore_failure', action='store_true', default=False,
+                    help='Stop when an error occurs')
+# <test ....>
+parser.add_argument('tests', nargs='+',
+                    help='CI test(s) to run. e.g: unit punit asan tsan ubsan')
+
+args = parser.parse_args()
+checker = PreCommitChecker(args)
+
+print("Please follow log %s" % checker.log.filename)
+
+if not checker.run_tests():
+    print("Error running tests. Please check log file %s"
+          % checker.log.filename)
+    sys.exit(1)
+
+sys.exit(0)

build_tools/ps_with_stack

@@ -1,38 +0,0 @@
-#!/usr/bin/env perl
-
-use strict;
-
-open(my $ps, "-|", "ps -wwf");
-my $cols_known = 0;
-my $cmd_col = 0;
-my $pid_col = 0;
-while (<$ps>) {
-  print;
-  my @cols = split(/\s+/);
-
-  if (!$cols_known && /CMD/) {
-    # Parse relevant ps column headers
-    for (my $i = 0; $i <= $#cols; $i++) {
-      if ($cols[$i] eq "CMD") {
-        $cmd_col = $i;
-      }
-      if ($cols[$i] eq "PID") {
-        $pid_col = $i;
-      }
-    }
-    $cols_known = 1;
-  } else {
-    my $pid = $cols[$pid_col];
-    my $cmd = $cols[$cmd_col];
-    # Match numeric PID and relative path command
-    # -> The intention is only to dump stack traces for hangs in code under
-    # test, which means we probably just built it and are executing by
-    # relative path (e.g. ./my_test or foo/bar_test) rather then by absolute
-    # path (e.g. /usr/bin/time) or PATH search (e.g. grep).
-    if ($pid =~ /^[0-9]+$/ && $cmd =~ /^[^\/ ]+[\/]/) {
-      print "Dumping stacks for $pid...\n";
-      system("pstack $pid || gdb -batch -p $pid -ex 'thread apply all bt'");
-    }
-  }
-}
-close $ps;

build_tools/regression_build_test.sh

@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
 set -e
 
@@ -20,11 +19,26 @@ STAT_FILE=${STAT_FILE:-$(mktemp -t -u rocksdb_test_stats_XXXX)}
 
 function cleanup {
   rm -rf $DATA_DIR
-  rm -f $STAT_FILE.*
+  rm -f $STAT_FILE.fillseq
+  rm -f $STAT_FILE.readrandom
+  rm -f $STAT_FILE.overwrite
+  rm -f $STAT_FILE.memtablefillreadrandom
 }
 
 trap cleanup EXIT
 
+if [ -z $GIT_BRANCH ]; then
+  git_br=`git rev-parse --abbrev-ref HEAD`
+else
+  git_br=$(basename $GIT_BRANCH)
+fi
+
+if [ $git_br == "master" ]; then
+  git_br=""
+else
+  git_br="."$git_br
+fi
+
 make release
 
 # measure fillseq + fill up the DB for overwrite benchmark
@@ -258,6 +272,7 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --value_size=100 \
     --compression_type=none \
     --compression_ratio=1 \
+    --hard_rate_limit=2 \
     --write_buffer_size=134217728 \
     --max_write_buffer_number=4 \
     --level0_file_num_compaction_trigger=8 \
@@ -270,10 +285,12 @@ common_in_mem_args="--db=/dev/shm/rocksdb \
     --sync=0 \
     --verify_checksum=1 \
     --delete_obsolete_files_period_micros=314572800 \
+    --max_grandparent_overlap_factor=10 \
     --use_plain_table=1 \
     --open_files=-1 \
     --mmap_read=1 \
     --mmap_write=0 \
+    --memtablerep=prefix_hash \
     --bloom_bits=10 \
     --bloom_locality=1 \
     --perf_level=0"
@@ -360,7 +377,7 @@ function send_to_ods {
     echo >&2 "ERROR: Key $key doesn't have a value."
     return
   fi
-  curl --silent "https://www.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=$key&value=$value" \
+  curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build$git_br&key=$key&value=$value" \
     --connect-timeout 60
 }
 

build_tools/rocksdb-lego-determinator

@@ -0,0 +1,801 @@
+#!/usr/bin/env bash
+# This script is executed by Sandcastle
+# to determine next steps to run
+
+# Usage:
+# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name>
+#
+# Input         Value
+# -------------------------------------------------------------------------
+# EMAIL         Email address to report on trigger conditions
+# ONCALL        Email address to raise a task on failure
+# TRIGGER       Trigger conditions for email. Valid values are fail, warn, all
+# SUBSCRIBER    Email addresss to add as subscriber for task
+#
+
+#
+# Report configuration
+#
+REPORT_EMAIL=
+if [ ! -z $EMAIL ]; then
+  if [ -z $TRIGGER ]; then
+    TRIGGER="fail"
+  fi
+
+  REPORT_EMAIL="
+  {
+      'type':'email',
+      'triggers': [ '$TRIGGER' ],
+      'emails':['$EMAIL']
+  },"
+fi
+
+CREATE_TASK=
+if [ ! -z $ONCALL ]; then
+  CREATE_TASK="
+  {
+      'type':'task',
+      'triggers':[ 'fail' ],
+      'priority':0,
+      'subscribers':[ '$SUBSCRIBER' ],
+      'tags':[ 'rocksdb', 'ci' ],
+  },"
+fi
+
+# For now, create the tasks using only the dedicated task creation tool.
+CREATE_TASK=
+
+REPORT=
+if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then
+  REPORT="'report': [
+    $REPORT_EMAIL
+    $CREATE_TASK
+  ]"
+fi
+
+#
+# Helper variables
+#
+CLEANUP_ENV="
+{
+    'name':'Cleanup environment',
+    'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true)  && make clean',
+    'user':'root'
+}"
+
+# We will eventually set the RATIO to 1, but we want do this
+# in steps. RATIO=$(nproc) will make it work as J=1
+if [ -z $RATIO ]; then
+  RATIO=$(nproc)
+fi
+
+if [ -z $PARALLEL_J ]; then
+  PARALLEL_J="J=$(expr $(nproc) / ${RATIO})"
+fi
+
+if [ -z $PARALLEL_j ]; then
+  PARALLEL_j="-j$(expr $(nproc) / ${RATIO})"
+fi
+
+PARALLELISM="$PARALLEL_J $PARALLEL_j"
+
+DEBUG="OPT=-g"
+SHM="TEST_TMPDIR=/dev/shm/rocksdb"
+NON_SHM="TMPD=/tmp/rocksdb_test_tmp"
+GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1"
+ASAN="COMPILE_WITH_ASAN=1"
+CLANG="USE_CLANG=1"
+LITE="OPT=\"-DROCKSDB_LITE -g\""
+TSAN="COMPILE_WITH_TSAN=1"
+UBSAN="COMPILE_WITH_UBSAN=1"
+DISABLE_JEMALLOC="DISABLE_JEMALLOC=1"
+HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080"
+SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH"
+PARSER="'parser':'python build_tools/error_filter.py $1'"
+
+CONTRUN_NAME="ROCKSDB_CONTRUN_NAME"
+
+# This code is getting called under various scenarios. What we care about is to
+# understand when it's called from nightly contruns because in that case we'll
+# create tasks for any failures. To follow the existing pattern, we'll check
+# the value of $ONCALL. If it's a diff then just call `false` to make sure
+# that errors will be properly propagated to the caller.
+if [ ! -z $ONCALL ]; then
+  TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL"
+else
+  TASK_CREATION_TOOL="false"
+fi
+
+ARTIFACTS=" 'artifacts': [
+    {
+        'name':'database',
+        'paths':[ '/dev/shm/rocksdb' ],
+    }
+]"
+
+#
+# A mechanism to disable tests temporarily
+#
+DISABLE_COMMANDS="[
+    {
+        'name':'Disable test',
+        'oncall':'$ONCALL',
+        'steps': [
+            {
+              'name':'Job disabled. Please contact test owner',
+              'shell':'exit 1',
+              'user':'root'
+            },
+        ],
+    }
+]"
+
+#
+# RocksDB unit test
+#
+UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB unit test not under /dev/shm
+#
+UNIT_TEST_NON_SHM_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'timeout': 86400,
+                'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB release build and unit tests
+#
+RELEASE_BUILD_COMMANDS="[
+    {
+        'name':'Rocksdb Release Build',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB release',
+                'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB unit test on gcc-4.8.1
+#
+UNIT_TEST_COMMANDS_481="[
+    {
+        'name':'Rocksdb Unit Test on GCC 4.8.1',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug version',
+                'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB release build and unit tests
+#
+RELEASE_BUILD_COMMANDS_481="[
+    {
+        'name':'Rocksdb Release on GCC 4.8.1',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB release on GCC 4.8.1',
+                'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB unit test with CLANG
+#
+CLANG_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and test RocksDB debug',
+                'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB release build with CLANG
+#
+CLANG_RELEASE_BUILD_COMMANDS="[
+    {
+        'name':'Rocksdb CLANG Release Build',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB release',
+                'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB analyze
+#
+CLANG_ANALYZE_COMMANDS="[
+    {
+        'name':'Rocksdb analyze',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'RocksDB build and analyze',
+                'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB code coverage
+#
+CODE_COV_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test Code Coverage',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build, test and collect code coverage info',
+                'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB unity
+#
+UNITY_COMMANDS="[
+    {
+        'name':'Rocksdb Unity',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build, test unity test',
+                'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# Build RocksDB lite
+#
+LITE_BUILD_COMMANDS="[
+    {
+        'name':'Rocksdb Lite build',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB debug version',
+                'shell':'$LITE make J=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# Report RocksDB lite binary size to scuba
+REPORT_LITE_BINARY_SIZE_COMMANDS="[
+    {
+        'name':'Rocksdb Lite Binary Size',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Report RocksDB Lite binary size to scuba',
+                'shell':'tools/report_lite_binary_size.sh',
+                'user':'root',
+            },
+        ],
+]"
+
+#
+# RocksDB stress/crash test
+#
+STRESS_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Stress/Crash Test',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug stress tests',
+                'shell':'$SHM $DEBUG make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+            {
+                'name':'Build and run RocksDB debug crash tests',
+                'timeout': 86400,
+                'shell':'$SHM $DEBUG make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            }
+        ],
+        $ARTIFACTS,
+        $REPORT
+    }
+]"
+
+# RocksDB write stress test.
+# We run on disk device on purpose (i.e. no $SHM)
+# because we want to add some randomness to fsync commands
+WRITE_STRESS_COMMANDS="[
+    {
+        'name':'Rocksdb Write Stress Test',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB write stress tests',
+                'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            }
+        ],
+        'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}],
+        $REPORT
+    }
+]"
+
+
+#
+# RocksDB test under address sanitizer
+#
+ASAN_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under ASAN',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Test RocksDB debug under ASAN',
+'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d',
+                'user':'root',
+                $PARSER
+            }
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB crash testing under address sanitizer
+#
+ASAN_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb crash test under ASAN',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug asan_crash_test',
+                'timeout': 86400,
+                'shell':'$SHM $DEBUG make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB test under undefined behavior sanitizer
+#
+UBSAN_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under UBSAN',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Test RocksDB debug under UBSAN',
+                'shell':'set -o pipefail && $SHM $UBSAN $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            }
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB crash testing under udnefined behavior sanitizer
+#
+UBSAN_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb crash test under UBSAN',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build and run RocksDB debug ubsan_crash_test',
+                'timeout': 86400,
+                'shell':'$SHM $DEBUG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB unit test under valgrind
+#
+VALGRIND_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under valgrind',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Run RocksDB debug unit tests',
+                'timeout': 86400,
+                'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB test under TSAN
+#
+TSAN_UNIT_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Unit Test under TSAN',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Run RocksDB debug unit test',
+                'timeout': 86400,
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB crash test under TSAN
+#
+TSAN_CRASH_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Crash Test under TSAN',
+        'oncall':'$ONCALL',
+        'timeout': 86400,
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Compile and run',
+                'timeout': 86400,
+                'shell':'set -o pipefail && $SHM $DEBUG $TSAN CRASH_TEST_KILL_ODD=1887 CRASH_TEST_EXT_ARGS=--log2_keys_per_lock=22  make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB format compatible
+#
+
+run_format_compatible()
+{
+  export TEST_TMPDIR=/dev/shm/rocksdb
+  rm -rf /dev/shm/rocksdb
+  mkdir /dev/shm/rocksdb
+
+  tools/check_format_compatible.sh
+}
+
+FORMAT_COMPATIBLE_COMMANDS="[
+    {
+        'name':'Rocksdb Format Compatible tests',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Run RocksDB debug unit test',
+                'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB no compression
+#
+run_no_compression()
+{
+  export TEST_TMPDIR=/dev/shm/rocksdb
+  rm -rf /dev/shm/rocksdb
+  mkdir /dev/shm/rocksdb
+  make clean
+  cat build_tools/fbcode_config.sh | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh
+  mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh
+  cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile
+  mv .tmp.Makefile Makefile
+  make $DEBUG J=1 check
+}
+
+NO_COMPRESSION_COMMANDS="[
+    {
+        'name':'Rocksdb No Compression tests',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Run RocksDB debug unit test',
+                'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB regression
+#
+run_regression()
+{
+  time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d  $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX)
+
+  # ======= report size to ODS ========
+
+  # parameters: $1 -- key, $2 -- value
+  function send_size_to_ods {
+    curl -s "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \
+      --connect-timeout 60
+  }
+
+  # === normal build ===
+  make clean
+  make -j$(nproc) static_lib
+  send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a)
+  strip librocksdb.a
+  send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a)
+
+  make -j$(nproc) shared_lib
+  send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`)
+  strip `readlink -f librocksdb.so`
+  send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
+
+  # === lite build ===
+  make clean
+  OPT=-DROCKSDB_LITE make -j$(nproc) static_lib
+  send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a)
+  strip librocksdb.a
+  send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a)
+
+  OPT=-DROCKSDB_LITE make -j$(nproc) shared_lib
+  send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`)
+  strip `readlink -f librocksdb.so`
+  send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`)
+}
+
+REGRESSION_COMMANDS="[
+    {
+        'name':'Rocksdb regression commands',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Make and run script',
+                'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+#
+# RocksDB Java build
+#
+JAVA_BUILD_TEST_COMMANDS="[
+    {
+        'name':'Rocksdb Java Build',
+        'oncall':'$ONCALL',
+        'steps': [
+            $CLEANUP_ENV,
+            {
+                'name':'Build RocksDB for Java',
+                'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL',
+                'user':'root',
+                $PARSER
+            },
+        ],
+        $REPORT
+    }
+]"
+
+
+case $1 in
+  unit)
+    echo $UNIT_TEST_COMMANDS
+    ;;
+  unit_non_shm)
+    echo $UNIT_TEST_NON_SHM_COMMANDS
+    ;;
+  release)
+    echo $RELEASE_BUILD_COMMANDS
+    ;;
+  unit_481)
+    echo $UNIT_TEST_COMMANDS_481
+    ;;
+  release_481)
+    echo $RELEASE_BUILD_COMMANDS_481
+    ;;
+  clang_unit)
+    echo $CLANG_UNIT_TEST_COMMANDS
+    ;;
+  clang_release)
+    echo $CLANG_RELEASE_BUILD_COMMANDS
+    ;;
+  clang_analyze)
+    echo $CLANG_ANALYZE_COMMANDS
+    ;;
+  code_cov)
+    echo $CODE_COV_COMMANDS
+    ;;
+  unity)
+    echo $UNITY_COMMANDS
+    ;;
+  lite)
+    echo $LITE_BUILD_COMMANDS
+    ;;
+  report_lite_binary_size)
+    echo $REPORT_LITE_BINARY_SIZE_COMMANDS
+    ;;
+  stress_crash)
+    echo $STRESS_CRASH_TEST_COMMANDS
+    ;;
+  write_stress)
+    echo $WRITE_STRESS_COMMANDS
+    ;;
+  asan)
+    echo $ASAN_TEST_COMMANDS
+    ;;
+  asan_crash)
+    echo $ASAN_CRASH_TEST_COMMANDS
+    ;;
+  ubsan)
+    echo $UBSAN_TEST_COMMANDS
+    ;;
+  ubsan_crash)
+    echo $UBSAN_CRASH_TEST_COMMANDS
+    ;;
+  valgrind)
+    echo $VALGRIND_TEST_COMMANDS
+    ;;
+  tsan)
+    echo $TSAN_UNIT_TEST_COMMANDS
+    ;;
+  tsan_crash)
+    echo $TSAN_CRASH_TEST_COMMANDS
+    ;;
+  format_compatible)
+    echo $FORMAT_COMPATIBLE_COMMANDS
+    ;;
+  run_format_compatible)
+    run_format_compatible
+    ;;
+  no_compression)
+    echo $NO_COMPRESSION_COMMANDS
+    ;;
+  run_no_compression)
+    run_no_compression
+    ;;
+  regression)
+    echo $REGRESSION_COMMANDS
+    ;;
+  run_regression)
+    run_regression
+    ;;
+  java_build)
+    echo $JAVA_BUILD_TEST_COMMANDS
+    ;;
+  *)
+    echo "Invalid determinator command"
+    ;;
+esac

build_tools/run_ci_db_test.ps1

@@ -1,4 +1,3 @@
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 # This script enables you running RocksDB tests by running
 # All the tests concurrently and utilizing all the cores
 Param(
@@ -42,7 +41,7 @@ $RunOnly.Add("c_test") | Out-Null
 $RunOnly.Add("compact_on_deletion_collector_test") | Out-Null
 $RunOnly.Add("merge_test") | Out-Null
 $RunOnly.Add("stringappend_test") | Out-Null # Apparently incorrectly written
-$RunOnly.Add("backup_engine_test") | Out-Null # Disabled
+$RunOnly.Add("backupable_db_test") | Out-Null # Disabled
 $RunOnly.Add("timer_queue_test") | Out-Null # Not a gtest
 
 if($RunAll -and $SuiteRun -ne "") {
@@ -68,7 +67,7 @@ $BinariesFolder = -Join($RootFolder, "\build\Debug\")
 
 if($WorkFolder -eq "") {
 
-    # If TEST_TMPDIR is set use it
+    # If TEST_TMPDIR is set use it    
     [string]$var = $Env:TEST_TMPDIR
     if($var -eq "") {
         $WorkFolder = -Join($RootFolder, "\db_tests\")
@@ -93,7 +92,7 @@ $ExcludeCasesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeCases -ne "") {
     Write-Host "ExcludeCases: $ExcludeCases"
     $l = $ExcludeCases -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeCasesSet.Add($t) | Out-Null
     }
 }
@@ -102,7 +101,7 @@ $ExcludeExesSet = New-Object System.Collections.Generic.HashSet[string]
 if($ExcludeExes -ne "") {
     Write-Host "ExcludeExe: $ExcludeExes"
     $l = $ExcludeExes -split ' '
-    ForEach($t in $l) {
+    ForEach($t in $l) { 
       $ExcludeExesSet.Add($t) | Out-Null
     }
 }
@@ -118,10 +117,6 @@ if($ExcludeExes -ne "") {
 #   MultiThreaded/MultiThreadedDBTest.
 #     MultiThreaded/0  # GetParam() = 0
 #     MultiThreaded/1  # GetParam() = 1
-#   RibbonTypeParamTest/0.  # TypeParam = struct DefaultTypesAndSettings
-#     CompactnessAndBacktrackAndFpRate
-#     Extremes
-#     FindOccupancyForSuccessRate
 #
 # into this:
 #
@@ -129,9 +124,6 @@ if($ExcludeExes -ne "") {
 #   DBTest.WriteEmptyBatch
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/0
 #   MultiThreaded/MultiThreadedDBTest.MultiThreaded/1
-#   RibbonTypeParamTest/0.CompactnessAndBacktrackAndFpRate
-#   RibbonTypeParamTest/0.Extremes
-#   RibbonTypeParamTest/0.FindOccupancyForSuccessRate
 #
 # Output into the parameter in a form TestName -> Log File Name
 function ExtractTestCases([string]$GTestExe, $HashTable) {
@@ -145,8 +137,6 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
 
     ForEach( $l in $Tests) {
 
-      # remove trailing comment if any
-      $l = $l -replace '\s+\#.*',''
       # Leading whitespace is fine
       $l = $l -replace '^\s+',''
       # Trailing dot is a test group but no whitespace
@@ -155,7 +145,8 @@ function ExtractTestCases([string]$GTestExe, $HashTable) {
       }  else {
         # Otherwise it is a test name, remove leading space
         $test = $l
-        # create a log name
+        # remove trailing comment if any and create a log name
+        $test = $test -replace '\s+\#.*',''
         $test = "$Group$test"
 
         if($ExcludeCasesSet.Contains($test)) {
@@ -261,7 +252,7 @@ if($Run -ne "") {
 
   $DiscoveredExe = @()
   dir -Path $search_path | ForEach-Object {
-     $DiscoveredExe += ($_.Name)
+     $DiscoveredExe += ($_.Name)     
   }
 
   # Remove exclusions
@@ -301,7 +292,7 @@ if($SuiteRun -ne "") {
 
   $ListOfExe = @()
   dir -Path $search_path | ForEach-Object {
-     $ListOfExe += ($_.Name)
+     $ListOfExe += ($_.Name)     
   }
 
   # Exclude those in RunOnly from running as suites
@@ -356,7 +347,7 @@ function RunJobs($Suites, $TestCmds, [int]$ConcurrencyVal)
 
     # Wait for all to finish and get the results
     while(($JobToLog.Count -gt 0) -or
-          ($TestCmds.Count -gt 0) -or
+          ($TestCmds.Count -gt 0) -or 
            ($Suites.Count -gt 0)) {
 
         # Make sure we have maximum concurrent jobs running if anything
@@ -476,8 +467,8 @@ RunJobs -Suites $CasesToRun -TestCmds $TestExes -ConcurrencyVal $Concurrency
 
 $EndDate = (Get-Date)
 
-New-TimeSpan -Start $StartDate -End $EndDate |
-  ForEach-Object {
+New-TimeSpan -Start $StartDate -End $EndDate | 
+  ForEach-Object { 
     "Elapsed time: {0:g}" -f $_
   }
 
@@ -491,3 +482,5 @@ if(!$script:success) {
  }
 
  exit 0
+
+ 

build_tools/setup_centos7.sh

@@ -1,9 +1,8 @@
 #!/bin/bash
-# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-set -ex
+set -e
 
-ROCKSDB_VERSION="6.7.3"
-ZSTD_VERSION="1.4.4"
+ROCKSDB_VERSION="5.10.3"
+ZSTD_VERSION="1.1.3"
 
 echo "This script configures CentOS with everything needed to build and run RocksDB"
 
@@ -40,6 +39,5 @@ cd /usr/local/rocksdb
 chown -R vagrant:vagrant /usr/local/rocksdb/
 sudo -u vagrant make static_lib
 cd examples/
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ make all
-sudo -u vagrant LD_LIBRARY_PATH=/usr/local/lib/ ./c_simple_example
-
+sudo -u vagrant make all
+sudo -u vagrant ./c_simple_example

build_tools/ubuntu20_image/Dockerfile

@@ -1,57 +0,0 @@
-# from official ubuntu 20.04
-FROM ubuntu:20.04
-# update system
-RUN apt-get update && apt-get upgrade -y
-# install basic tools
-RUN apt-get install -y vim wget curl
-# install tzdata noninteractive
-RUN DEBIAN_FRONTEND=noninteractive TZ=Etc/UTC apt-get -y install tzdata
-# install git and default compilers
-RUN apt-get install -y git gcc g++ clang clang-tools
-# install basic package
-RUN apt-get install -y lsb-release software-properties-common gnupg
-# install gflags, tbb
-RUN apt-get install -y libgflags-dev libtbb-dev
-# install compression libs
-RUN apt-get install -y libsnappy-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
-# install cmake
-RUN apt-get install -y cmake
-RUN apt-get install -y libssl-dev
-# install clang-13
-WORKDIR /root
-RUN wget https://apt.llvm.org/llvm.sh
-RUN chmod +x llvm.sh
-RUN ./llvm.sh 13 all
-# install gcc-7, 8, 10, 11, default is 9
-RUN apt-get install -y gcc-7 g++-7
-RUN apt-get install -y gcc-8 g++-8
-RUN apt-get install -y gcc-10 g++-10
-RUN add-apt-repository -y ppa:ubuntu-toolchain-r/test
-RUN apt-get install -y gcc-11 g++-11
-# install apt-get install -y valgrind
-RUN apt-get install -y valgrind
-# install folly depencencies
-RUN apt-get install -y libgoogle-glog-dev
-# install openjdk 8
-RUN apt-get install -y openjdk-8-jdk
-ENV JAVA_HOME /usr/lib/jvm/java-1.8.0-openjdk-amd64
-# install mingw
-RUN apt-get install -y mingw-w64
-
-# install gtest-parallel package
-RUN git clone --single-branch --branch master --depth 1 https://github.com/google/gtest-parallel.git ~/gtest-parallel
-ENV PATH $PATH:/root/gtest-parallel
-
-# install libprotobuf for fuzzers test
-RUN apt-get install -y ninja-build binutils liblzma-dev libz-dev pkg-config autoconf libtool
-RUN git clone --branch v1.0 https://github.com/google/libprotobuf-mutator.git ~/libprotobuf-mutator && cd ~/libprotobuf-mutator && git checkout ffd86a32874e5c08a143019aad1aaf0907294c9f && mkdir build && cd build && cmake .. -GNinja -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang++-13 -DCMAKE_BUILD_TYPE=Release -DLIB_PROTO_MUTATOR_DOWNLOAD_PROTOBUF=ON && ninja && ninja install
-ENV PKG_CONFIG_PATH /usr/local/OFF/:/root/libprotobuf-mutator/build/external.protobuf/lib/pkgconfig/
-ENV PROTOC_BIN /root/libprotobuf-mutator/build/external.protobuf/bin/protoc
-
-# install the latest google benchmark
-RUN git clone --depth 1 --branch v1.7.0 https://github.com/google/benchmark.git ~/benchmark
-RUN cd ~/benchmark && mkdir build && cd build && cmake .. -GNinja -DCMAKE_BUILD_TYPE=Release -DBENCHMARK_ENABLE_GTEST_TESTS=0 && ninja && ninja install
-
-# clean up
-RUN rm -rf /var/lib/apt/lists/*
-RUN rm -rf /root/benchmark