jabberwocky238/rocksdb
1
0
Fork 0
mirror of https://github.com/facebook/rocksdb.git synced 2026-07-07 14:47:40 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
0f15da1c2db1d5583d0c7200208b6465357f95fc
rocksdb/folly.mk
T
xingbowang 638354e766 Fix getdeps fallback mirror downloads (#14763) 
Summary:
- parse folly getdeps manifests with bare package entries so fallback prefetching actually runs
- validate and remove bad cached/downloaded archives before trying fallback mirrors
- download through temporary files and include libiberty in the GNU toolchain fallback set

Context:

Nightly test failed with dependency download failure in folly.

```
Assessing autoconf...
Download with https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz -> /tmp/fbcode_builder_getdeps-Z__wZrocksdbZrocksdbZthird-partyZfollyZbuildZfbcode_builder-root/downloads/autoconf-autoconf-2.69.tar.gz ...
 [Complete in 136.616022 seconds]
    raise Exception(
Exception: https://ftpmirror.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz: expected sha256 954bd69b391edc12d6a4a51a2dd1476543da5c6bbf05a95b59dc0dd6fd4c2969 but got e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
make: *** [folly.mk:152: build_folly] Error 1
##[error]Process completed with exit code 2.
```

Pull Request resolved: https://github.com/facebook/rocksdb/pull/14763

Test Plan:
1. Connection failure:
      - Forced first mirror to http://127.0.0.1:1/...
      - It logged connection refused.
      - It then tried https://mirrors.kernel.org/gnu/...
      - Download succeeded, size 1927468, SHA matched 954bd69b...
  2. Empty file / bad hash:
      - Ran a local HTTP server returning a zero-byte autoconf-2.69.tar.gz
      - Script logged mismatch with actual=e3b0c442... size=0
      - It removed the bad download and fell back to mirrors.kernel.org
      - Download succeeded with the expected SHA.
  3. Existing zero-byte cache:
      - Seeded cache with an empty tarball.
      - Script removed invalid cache and downloaded a verified copy.

Reviewed By: mszeszko-meta

Differential Revision: D105859558

Pulled By: xingbowang

fbshipit-source-id: ff1f20f87debad561610271ce99b8b8de2d4264f
2026-05-28 07:19:20 -07:00

177 lines
9.0 KiB
Makefile
Raw Blame History

# This file contains the vast majority of folly-related build configuration
# for the checkout_folly and build_folly targets, so that this file can be
# hashed for purposes of caching folly builds and not hitting that cache when
# something here changes.
# This provides a Makefile simulation of a Meta-internal folly integration.
# It is not validated for general use.
#
# USE_FOLLY links the build targets with libfolly.a. The latter could be
# built using 'make build_folly', or built externally and specified in
# the CXXFLAGS and EXTRA_LDFLAGS env variables. The build_detect_platform
# script tries to detect if an external folly dependency has been specified.
# If not, it exports FOLLY_PATH to the path of the installed Folly and
# dependency libraries.
#
# USE_FOLLY_LITE cherry picks source files from Folly to include in the
# RocksDB library. Its faster and has fewer dependencies on 3rd party
# libraries, but with limited functionality. For example, coroutine
# functionality is not available.
ifeq ($(USE_FOLLY),1)
ifeq ($(USE_FOLLY_LITE),1)
$(error Please specify only one of USE_FOLLY and USE_FOLLY_LITE)
endif
ifneq ($(strip $(FOLLY_PATH)),)
BOOST_PATH = $(shell (ls -d $(FOLLY_PATH)/../boost*))
DBL_CONV_PATH = $(shell (ls -d $(FOLLY_PATH)/../double-conversion*))
GFLAGS_PATH = $(shell (ls -d $(FOLLY_PATH)/../gflags*))
GLOG_PATH = $(shell (ls -d $(FOLLY_PATH)/../glog*))
LIBEVENT_PATH = $(shell (ls -d $(FOLLY_PATH)/../libevent*))
XZ_PATH = $(shell (ls -d $(FOLLY_PATH)/../xz*))
LIBSODIUM_PATH = $(shell (ls -d $(FOLLY_PATH)/../libsodium*))
FMT_PATH = $(shell (ls -d $(FOLLY_PATH)/../fmt*))
# For some reason, glog and fmt libraries are under either lib or lib64
GLOG_LIB_PATH = $(shell (ls -d $(GLOG_PATH)/lib*))
FMT_LIB_PATH = $(shell (ls -d $(FMT_PATH)/lib*))
# AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include
PLATFORM_CXXFLAGS += -I$(BOOST_PATH)/include -I$(DBL_CONV_PATH)/include -I$(GLOG_PATH)/include -I$(LIBEVENT_PATH)/include -I$(XZ_PATH)/include -I$(LIBSODIUM_PATH)/include -I$(FOLLY_PATH)/include -I$(FMT_PATH)/include
else
PLATFORM_CCFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include
PLATFORM_CXXFLAGS += -isystem $(BOOST_PATH)/include -isystem $(DBL_CONV_PATH)/include -isystem $(GLOG_PATH)/include -isystem $(LIBEVENT_PATH)/include -isystem $(XZ_PATH)/include -isystem $(LIBSODIUM_PATH)/include -isystem $(FOLLY_PATH)/include -isystem $(FMT_PATH)/include
endif
# Add -ldl at the end as gcc resolves a symbol in a library by searching only in libraries specified later
# in the command line
PLATFORM_LDFLAGS += $(FOLLY_PATH)/lib/libfolly.a $(BOOST_PATH)/lib/libboost_context.a $(BOOST_PATH)/lib/libboost_filesystem.a $(BOOST_PATH)/lib/libboost_atomic.a $(BOOST_PATH)/lib/libboost_program_options.a $(BOOST_PATH)/lib/libboost_regex.a $(BOOST_PATH)/lib/libboost_system.a $(BOOST_PATH)/lib/libboost_thread.a $(DBL_CONV_PATH)/lib/libdouble-conversion.a $(LIBEVENT_PATH)/lib/libevent.a $(LIBSODIUM_PATH)/lib/libsodium.a -ldl
ifneq ($(DEBUG_LEVEL),0)
PLATFORM_LDFLAGS += $(FMT_LIB_PATH)/libfmtd.a $(GLOG_LIB_PATH)/libglogd.so $(GFLAGS_PATH)/lib/libgflags_debug.so.2.2
else
PLATFORM_LDFLAGS += $(FMT_LIB_PATH)/libfmt.a $(GLOG_LIB_PATH)/libglog.so $(GFLAGS_PATH)/lib/libgflags.so.2.2
endif
PLATFORM_LDFLAGS += -Wl,-rpath=$(GLOG_LIB_PATH) -Wl,-rpath=$(GFLAGS_PATH)/lib
endif
PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
endif
ifeq ($(USE_FOLLY_LITE),1)
# Path to the Folly source code and include files
FOLLY_DIR = ./third-party/folly
ifneq ($(strip $(BOOST_SOURCE_PATH)),)
BOOST_INCLUDE = $(shell (ls -d $(BOOST_SOURCE_PATH)/boost*/))
# AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(BOOST_INCLUDE)
PLATFORM_CXXFLAGS += -I$(BOOST_INCLUDE)
else
PLATFORM_CCFLAGS += -isystem $(BOOST_INCLUDE)
PLATFORM_CXXFLAGS += -isystem $(BOOST_INCLUDE)
endif
endif # BOOST_SOURCE_PATH
ifneq ($(strip $(FMT_SOURCE_PATH)),)
FMT_INCLUDE = $(shell (ls -d $(FMT_SOURCE_PATH)/fmt*/include/))
# AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(FMT_INCLUDE)
PLATFORM_CXXFLAGS += -I$(FMT_INCLUDE)
else
PLATFORM_CCFLAGS += -isystem $(FMT_INCLUDE)
PLATFORM_CXXFLAGS += -isystem $(FMT_INCLUDE)
endif
endif # FMT_SOURCE_PATH
# AIX: pre-defined system headers are surrounded by an extern "C" block
ifeq ($(PLATFORM), OS_AIX)
PLATFORM_CCFLAGS += -I$(FOLLY_DIR)
PLATFORM_CXXFLAGS += -I$(FOLLY_DIR)
else
PLATFORM_CCFLAGS += -isystem $(FOLLY_DIR)
PLATFORM_CXXFLAGS += -isystem $(FOLLY_DIR)
endif
PLATFORM_CCFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
PLATFORM_CXXFLAGS += -DUSE_FOLLY -DFOLLY_NO_CONFIG
# TODO: fix linking with fbcode compiler config
PLATFORM_LDFLAGS += -lglog
endif
FOLLY_COMMIT_HASH = 548b16da0b3cc887d69cdb6ae06496ad8a2a9276
FOLLY_GETDEPS_CACHE_DIR = /tmp/rocksdb-getdeps-cache
define restore_folly_getdeps_downloads
@cd third-party/folly && \
DOWNLOAD_DIR=`$(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir | sed 's|/installed/.*|/downloads|'` && \
mkdir -p "$$DOWNLOAD_DIR" && \
CACHE_DIR="$(FOLLY_GETDEPS_CACHE_DIR)" && \
mkdir -p "$$CACHE_DIR" && \
echo "Restoring cached downloads..." && \
if ls "$$CACHE_DIR"/*.tar.gz "$$CACHE_DIR"/*.tar.xz "$$CACHE_DIR"/*.zip >/dev/null 2>&1; then \
cp -n "$$CACHE_DIR"/*.tar.gz "$$CACHE_DIR"/*.tar.xz "$$CACHE_DIR"/*.zip "$$DOWNLOAD_DIR/" 2>/dev/null || true; \
fi && \
echo "Handling known unreliable downloads with fallback mirrors..." && \
$(PYTHON) ../../build_tools/getdeps_fallback_mirror.py "$$DOWNLOAD_DIR" "$$CACHE_DIR" build/fbcode_builder/manifests
endef
define cache_folly_getdeps_downloads
@cd third-party/folly && \
DOWNLOAD_DIR=`$(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir | sed 's|/installed/.*|/downloads|'` && \
CACHE_DIR="$(FOLLY_GETDEPS_CACHE_DIR)" && \
if ls "$$DOWNLOAD_DIR"/*.tar.gz "$$DOWNLOAD_DIR"/*.tar.xz "$$DOWNLOAD_DIR"/*.zip >/dev/null 2>&1; then \
cp -n "$$DOWNLOAD_DIR"/*.tar.gz "$$DOWNLOAD_DIR"/*.tar.xz "$$DOWNLOAD_DIR"/*.zip "$$CACHE_DIR/" 2>/dev/null || true; \
fi
endef
# For public CI runs, checkout folly in a way that can build with RocksDB.
# This is mostly intended as a test-only simulation of Meta-internal folly
# integration.
checkout_folly:
if [ -e third-party/folly ]; then \
cd third-party/folly && ${GIT_COMMAND} fetch origin; \
else \
cd third-party && ${GIT_COMMAND} clone https://github.com/facebook/folly.git; \
fi
@# Pin to a particular version for public CI, so that PR authors don't
@# need to worry about folly breaking our integration. Update periodically
cd third-party/folly && git reset --hard $(FOLLY_COMMIT_HASH)
@# Apparently missing include
perl -pi -e 's/(#include <atomic>)/$$1\n#include <cstring>/' third-party/folly/folly/lang/Exception.h
@# const mismatch
perl -pi -e 's/: environ/: (const char**)(environ)/' third-party/folly/folly/Subprocess.cpp
@# Restore cached downloads and handle unreliable mirrors with fallback
$(restore_folly_getdeps_downloads)
@# NOTE: boost and fmt source will be needed for any build including `USE_FOLLY_LITE` builds as those depend on those headers
cd third-party/folly && GETDEPS_USE_WGET=1 $(PYTHON) build/fbcode_builder/getdeps.py fetch boost && GETDEPS_USE_WGET=1 $(PYTHON) build/fbcode_builder/getdeps.py fetch fmt
@# Update cache with any new downloads
$(cache_folly_getdeps_downloads)
CXX_M_FLAGS = $(filter -m%, $(CXXFLAGS))
FOLLY_BUILD_FLAGS = --no-tests
# NOTE: To avoid ODR violations, we must build folly in debug mode iff
# building RocksDB in debug mode.
ifneq ($(DEBUG_LEVEL),0)
FOLLY_BUILD_FLAGS += --build-type Debug
endif
build_folly:
FOLLY_INST_PATH=`cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir`; \
if [ "$$FOLLY_INST_PATH" ]; then \
rm -rf $${FOLLY_INST_PATH}/../../*; \
else \
echo "Please run checkout_folly first"; \
false; \
fi
@# Restore fallback archives after the cleanup above removes downloads.
$(restore_folly_getdeps_downloads)
cd third-party/folly && \
CXXFLAGS=" $(CXX_M_FLAGS) -DHAVE_CXX11_ATOMIC " GETDEPS_USE_WGET=1 $(PYTHON) build/fbcode_builder/getdeps.py build $(FOLLY_BUILD_FLAGS)
@# In the folly build, glog and gflags are only built as dynamic libraries,
@# not static. This patchelf command is needed to reliably have the glog
@# library find its dependency gflags, because apparently the rpath of the
@# final binary is not used in resolving that transitive dependency.
FOLLY_INST_PATH=`cd third-party/folly && $(PYTHON) build/fbcode_builder/getdeps.py show-inst-dir`; \
cd "$$FOLLY_INST_PATH" && patchelf --add-rpath $$PWD/../gflags-*/lib ../glog-*/lib*/libglog*.so.*.*.*
Reference in New Issue View Git Blame Copy Permalink