Merge bitcoin/bitcoin#30201: depends: remove FORCE_USE_SYSTEM_CLANG

7cbfd7a7ce45ac68d6041f42f468862f5c193d8c refactor: rename (macho) ld64 to lld (fanquake)
d851451705aa67175cbafb06f455e6969131f90f ci: update deps for macOS cross build (fanquake)
9ebdd5e9e04be3355309d3096da00db37b4d540c depends: update install docs for macOS cross compilation (fanquake)
fb74fd66cb2b016f07e47dc58d81b33034ce851b depends: remove no-longer used llvm_* vars from macOS build (fanquake)
9043f12425d869d8bf7f2be1bb12368e41d09c5d depends: no-longer pass -B to clang in macOS cross-compile (fanquake)
f9994b025e1d72b4c0f5267de54caa2bfae136a3 depends: remove native LLVM package (fanquake)
e9a44faf14854c5b05a297df182d10706f438a83 depends: remove FORCE_USE_SYSTEM_CLANG (fanquake)
9946618f6174422d9ba93bf96c278269bebacf39 guix: use clang-toolchain-18 for macOS build (fanquake)

Pull request description:

  Remove `FORCE_USE_SYSTEM_CLANG` in favour of always using the system Clang and lld for macOS cross-compilation; rather than downloading precompiled blobs.

  For example, anyone using Ubuntu 24.04 should be able to `apt install clang llvm lld .. etc`, and then cross-compile for macOS using:
  ```bash
  # clang --version
  Ubuntu clang version 18.1.3 (1)
  make -C depends HOST=arm64-apple-darwin FORCE_USE_SYSTEM_CLANG=1
  ./autogen.sh
  CONFIG_SITE=/path/to/depends/arm64-apple-darwin/share/config.site ./configure
  make
  # file src/qt/bitcoin-qt
  src/qt/bitcoin-qt: Mach-O 64-bit arm64 executable, flags:<NOUNDEFS|DYLDLINK|TWOLEVEL|WEAK_DEFINES|BINDS_TO_WEAK|PIE|HAS_TLV_DESCRIPTORS>
  ```

  Note that the minimum supported version of Clang we will support for macOS cross-compilation will likely be more recent than our other minimum supported version of Clang, due to compiler/linker option usage.

ACKs for top commit:
  Sjors:
    tACK 7cbfd7a7ce45ac68d6041f42f468862f5c193d8c
  theuni:
    ACK 7cbfd7a7ce45ac68d6041f42f468862f5c193d8c
  TheCharlatan:
    Nice, ACK 7cbfd7a7ce45ac68d6041f42f468862f5c193d8c

Tree-SHA512: 1499e29b3b238c5c85278c38e8fb6bb5e7883db3443f62b6bf397c5d761bedbc054962be645a9defce15266f0a969bb4b3ccd28b6e4dd874472857b928f185d1

ci/test/00_setup_env_mac_cross.sh

@@ -9,9 +9,9 @@ export LC_ALL=C.UTF-8
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 
 export CONTAINER_NAME=ci_macos_cross
-export CI_IMAGE_NAME_TAG="docker.io/ubuntu:22.04"
+export CI_IMAGE_NAME_TAG="docker.io/ubuntu:24.04"
 export HOST=x86_64-apple-darwin
-export PACKAGES="zip"
+export PACKAGES="clang lld llvm zip"
 export XCODE_VERSION=15.0
 export XCODE_BUILD_ID=15A240d
 export RUN_UNIT_TESTS=false

contrib/devtools/symbol-check.py

@@ -239,8 +239,8 @@ def check_MACHO_sdk(binary) -> bool:
         return True
     return False
 
-def check_MACHO_ld64(binary) -> bool:
+def check_MACHO_lld(binary) -> bool:
-    if binary.build_version.tools[0].version == [17, 0, 6]:
+    if binary.build_version.tools[0].version == [18, 1, 6]:
         return True
     return False
 
@@ -282,7 +282,7 @@ lief.EXE_FORMATS.MACHO: [
     ('DYNAMIC_LIBRARIES', check_MACHO_libraries),
     ('MIN_OS', check_MACHO_min_os),
     ('SDK', check_MACHO_sdk),
-    ('LD64', check_MACHO_ld64),
+    ('LLD', check_MACHO_lld),
 ],
 lief.EXE_FORMATS.PE: [
     ('DYNAMIC_LIBRARIES', check_PE_libraries),

contrib/guix/libexec/build.sh

@@ -178,8 +178,7 @@ make -C depends --jobs="$JOBS" HOST="$HOST" \
                                    x86_64_linux_AR=x86_64-linux-gnu-gcc-ar \
                                    x86_64_linux_RANLIB=x86_64-linux-gnu-gcc-ranlib \
                                    x86_64_linux_NM=x86_64-linux-gnu-gcc-nm \
-                                   x86_64_linux_STRIP=x86_64-linux-gnu-strip \
+                                   x86_64_linux_STRIP=x86_64-linux-gnu-strip
-                                   FORCE_USE_SYSTEM_CLANG=1
 
 
 ###########################

contrib/guix/manifest.scm

@@ -532,9 +532,9 @@ inspecting signatures in Mach-O binaries.")
           ((string-contains target "darwin")
            (list ;; Native GCC 11 toolchain
                  gcc-toolchain-11
-                 clang-toolchain-17
+                 clang-toolchain-18
-                 lld-17
+                 lld-18
-                 (make-lld-wrapper lld-17 #:lld-as-ld? #t)
+                 (make-lld-wrapper lld-18 #:lld-as-ld? #t)
                  python-signapple
                  zip))
           (else '())))))

depends/Makefile

@@ -181,8 +181,6 @@ all_packages = $(packages) $(native_packages)
 
 meta_depends = Makefile config.guess config.sub funcs.mk builders/default.mk hosts/default.mk hosts/$(host_os).mk builders/$(build_os).mk
 
-$(host_arch)_$(host_os)_native_toolchain?=$($(host_os)_native_toolchain)
-
 include funcs.mk
 
 final_build_id_long+=$(shell $(build_SHA256SUM) config.site.in)

depends/README.md

@@ -49,11 +49,11 @@ The paths are automatically configured and no other options are needed.
 
 #### For macOS cross compilation
 
-    sudo apt-get install g++ zip
+    apt install clang lld llvm g++ zip
 
-Note: You must obtain the macOS SDK before proceeding with a cross-compile.
+Clang 18 or later is required. You must also obtain the macOS SDK before
-Under the depends directory, create a subdirectory named `SDKs`.
+proceeding with a cross-compile. Under the depends directory, create a
-Then, place the extracted SDK under this new directory.
+subdirectory named `SDKs`. Then, place the extracted SDK under this new directory.
 For more information, see [SDK Extraction](../contrib/macdeploy/README.md#sdk-extraction).
 
 #### For Win64 cross compilation
@@ -119,9 +119,6 @@ The following can be set when running make: `make FOO=bar`
 - `DEBUG`: Disable some optimizations and enable more runtime checking
 - `HOST_ID_SALT`: Optional salt to use when generating host package ids
 - `BUILD_ID_SALT`: Optional salt to use when generating build package ids
-- `FORCE_USE_SYSTEM_CLANG`: (EXPERTS ONLY) When cross-compiling for macOS, use Clang found in the
-  system's `$PATH` rather than the default prebuilt release of Clang
-  from llvm.org. Clang 8 or later is required
 - `LOG`: Use file-based logging for individual packages. During a package build its log file
   resides in the `depends` directory, and the log file is printed out automatically in case
   of build error. After successful build log files are moved along with package archives

depends/builders/darwin.mk

@@ -18,7 +18,6 @@ darwin_STRIP:=$(shell xcrun -f strip)
 darwin_OBJDUMP:=$(shell xcrun -f objdump)
 darwin_NM:=$(shell xcrun -f nm)
 darwin_DSYMUTIL:=$(shell xcrun -f dsymutil)
-darwin_native_toolchain=
 
 x86_64_darwin_CFLAGS += -arch x86_64
 x86_64_darwin_CXXFLAGS += -arch x86_64

depends/funcs.mk

@@ -46,7 +46,7 @@ endef
 
 define int_get_build_id
 $(eval $(1)_dependencies += $($(1)_$(host_arch)_$(host_os)_dependencies) $($(1)_$(host_os)_dependencies))
-$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($($(1)_type)_native_toolchain) $($(1)_dependencies)))
+$(eval $(1)_all_dependencies:=$(call int_get_all_dependencies,$(1),$($(1)_dependencies)))
 $(foreach dep,$($(1)_all_dependencies),$(eval $(1)_build_id_deps+=$(dep)-$($(dep)_version)-$($(dep)_recipe_hash)))
 $(eval $(1)_build_id_long:=$(1)-$($(1)_version)-$($(1)_recipe_hash)-$(release_type) $($(1)_build_id_deps) $($($(1)_type)_id))
 $(eval $(1)_build_id:=$(shell echo -n "$($(1)_build_id_long)" | $(build_SHA256SUM) | cut -c-$(HASH_LENGTH)))
@@ -297,6 +297,3 @@ $(foreach package,$(all_packages),$(eval $(call int_config_attach_build_config,$
 
 #create build targets
 $(foreach package,$(all_packages),$(eval $(call int_add_cmds,$(package))))
-
-#special exception: if a toolchain package exists, all non-native packages depend on it
-$(foreach package,$(packages),$(eval $($(package)_extracted): |$($($(host_arch)_$(host_os)_native_toolchain)_cached) ))

depends/hosts/darwin.mk

@@ -2,39 +2,10 @@ OSX_MIN_VERSION=11.0
 OSX_SDK_VERSION=14.0
 XCODE_VERSION=15.0
 XCODE_BUILD_ID=15A240d
-LD64_VERSION=711
+LLD_VERSION=711
 
 OSX_SDK=$(SDK_PATH)/Xcode-$(XCODE_VERSION)-$(XCODE_BUILD_ID)-extracted-SDK-with-libcxx-headers
 
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-# FORCE_USE_SYSTEM_CLANG is empty, so we use our depends-managed, pinned LLVM
-# from llvm.org
-
-darwin_native_toolchain=native_llvm
-
-clang_prog=$(build_prefix)/bin/clang
-clangxx_prog=$(clang_prog)++
-llvm_config_prog=$(build_prefix)/bin/llvm-config
-
-llvm_TOOLS=AR NM OBJDUMP RANLIB STRIP
-
-# Make-only lowercase function
-lc = $(subst A,a,$(subst B,b,$(subst C,c,$(subst D,d,$(subst E,e,$(subst F,f,$(subst G,g,$(subst H,h,$(subst I,i,$(subst J,j,$(subst K,k,$(subst L,l,$(subst M,m,$(subst N,n,$(subst O,o,$(subst P,p,$(subst Q,q,$(subst R,r,$(subst S,s,$(subst T,t,$(subst U,u,$(subst V,v,$(subst W,w,$(subst X,x,$(subst Y,y,$(subst Z,z,$1))))))))))))))))))))))))))
-
-# For well-known tools provided by LLVM, make sure that their well-known
-# variable is set to the full path of the tool, just like how AC_PATH_{TOO,PROG}
-# would.
-$(foreach TOOL,$(llvm_TOOLS),$(eval darwin_$(TOOL) = $$(build_prefix)/bin/llvm-$(call lc,$(TOOL))))
-
-# Clang expects dsymutil to be called dsymutil
-darwin_DSYMUTIL=$(build_prefix)/bin/dsymutil
-
-else
-# FORCE_USE_SYSTEM_CLANG is non-empty, so we use the clang from the user's
-# system
-
-darwin_native_toolchain=
-
 # We can't just use $(shell command -v clang) because GNU Make handles builtins
 # in a special way and doesn't know that `command` is a POSIX-standard builtin
 # prior to 1af314465e5dfe3e8baa839a32a72e83c04f26ef, first released in v4.2.90.
@@ -44,9 +15,6 @@ darwin_native_toolchain=
 # Source: https://lists.gnu.org/archive/html/bug-make/2017-11/msg00017.html
 clang_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang")
 clangxx_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v clang++")
-llvm_config_prog=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-config")
-
-llvm_lib_dir=$(shell $(llvm_config_prog) --libdir)
 
 darwin_AR=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ar")
 darwin_DSYMUTIL=$(shell $(SHELL) $(.SHELLFLAGS) "command -v dsymutil")
@@ -54,7 +22,6 @@ darwin_NM=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-nm")
 darwin_OBJDUMP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-objdump")
 darwin_RANLIB=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-ranlib")
 darwin_STRIP=$(shell $(SHELL) $(.SHELLFLAGS) "command -v llvm-strip")
-endif
 
 # Flag explanations:
 #
@@ -63,11 +30,6 @@ endif
 #         Ensures that modern linker features are enabled. See here for more
 #         details: https://github.com/bitcoin/bitcoin/pull/19407.
 #
-#     -B$(build_prefix)/bin
-#
-#         Explicitly point to our binaries so that they are
-#         ensured to be found and preferred over other possibilities.
-#
 #     -isysroot$(OSX_SDK) -nostdlibinc
 #
 #         Disable default include paths built into the compiler as well as
@@ -92,7 +54,6 @@ darwin_CC=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
               -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
               -u LIBRARY_PATH \
               $(clang_prog) --target=$(host) \
-              -B$(build_prefix)/bin \
               -isysroot$(OSX_SDK) -nostdlibinc \
               -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks
 
@@ -100,7 +61,6 @@ darwin_CXX=env -u C_INCLUDE_PATH -u CPLUS_INCLUDE_PATH \
                -u OBJC_INCLUDE_PATH -u OBJCPLUS_INCLUDE_PATH -u CPATH \
                -u LIBRARY_PATH \
                $(clangxx_prog) --target=$(host) \
-               -B$(build_prefix)/bin \
                -isysroot$(OSX_SDK) -nostdlibinc \
                -iwithsysroot/usr/include/c++/v1 \
                -iwithsysroot/usr/include -iframeworkwithsysroot/System/Library/Frameworks
@@ -110,8 +70,8 @@ darwin_CXXFLAGS=-pipe -std=$(CXX_STANDARD) -mmacosx-version-min=$(OSX_MIN_VERSIO
 darwin_LDFLAGS=-Wl,-platform_version,macos,$(OSX_MIN_VERSION),$(OSX_SDK_VERSION)
 
 ifneq ($(build_os),darwin)
-darwin_CFLAGS += -mlinker-version=$(LD64_VERSION)
+darwin_CFLAGS += -mlinker-version=$(LLD_VERSION)
-darwin_CXXFLAGS += -mlinker-version=$(LD64_VERSION)
+darwin_CXXFLAGS += -mlinker-version=$(LLD_VERSION)
 darwin_LDFLAGS += -Wl,-no_adhoc_codesign -fuse-ld=lld
 endif
 

depends/packages/native_llvm.mk

@@ -1,31 +0,0 @@
-package=native_llvm
-$(package)_version=17.0.6
-$(package)_major_version=$(firstword $(subst ., ,$($(package)_version)))
-$(package)_download_path=https://github.com/llvm/llvm-project/releases/download/llvmorg-$($(package)_version)
-ifneq (,$(findstring aarch64,$(BUILD)))
-$(package)_file_name=clang+llvm-$($(package)_version)-aarch64-linux-gnu.tar.xz
-$(package)_sha256_hash=6dd62762285326f223f40b8e4f2864b5c372de3f7de0731cb7cd55ca5287b75a
-else
-$(package)_file_name=clang+llvm-$($(package)_version)-x86_64-linux-gnu-ubuntu-22.04.tar.xz
-$(package)_sha256_hash=884ee67d647d77e58740c1e645649e29ae9e8a6fe87c1376be0f3a30f3cc9ab3
-endif
-
-define $(package)_stage_cmds
-  mkdir -p $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include && \
-  mkdir -p $($(package)_staging_prefix_dir)/bin && \
-  mkdir -p $($(package)_staging_prefix_dir)/include/llvm-c && \
-  cp bin/clang $($(package)_staging_prefix_dir)/bin/ && \
-  cp -P bin/clang++ $($(package)_staging_prefix_dir)/bin/ && \
-  cp bin/dsymutil $($(package)_staging_prefix_dir)/bin/dsymutil && \
-  cp bin/ld64.lld $($(package)_staging_prefix_dir)/bin/ld64.lld && \
-  cp bin/llvm-ar $($(package)_staging_prefix_dir)/bin/llvm-ar && \
-  cp bin/llvm-config $($(package)_staging_prefix_dir)/bin/ && \
-  cp bin/llvm-nm $($(package)_staging_prefix_dir)/bin/llvm-nm && \
-  cp bin/llvm-objdump $($(package)_staging_prefix_dir)/bin/llvm-objdump && \
-  cp bin/llvm-ranlib $($(package)_staging_prefix_dir)/bin/llvm-ranlib && \
-  cp bin/llvm-strip $($(package)_staging_prefix_dir)/bin/llvm-strip && \
-  cp include/llvm-c/ExternC.h $($(package)_staging_prefix_dir)/include/llvm-c && \
-  cp include/llvm-c/lto.h $($(package)_staging_prefix_dir)/include/llvm-c && \
-  cp lib/libLTO.so $($(package)_staging_prefix_dir)/lib/ && \
-  cp -r lib/clang/$($(package)_major_version)/include/* $($(package)_staging_prefix_dir)/lib/clang/$($(package)_major_version)/include/
-endef

depends/packages/packages.mk

@@ -24,13 +24,3 @@ multiprocess_packages = libmultiprocess capnp
 multiprocess_native_packages = native_libmultiprocess native_capnp
 
 usdt_linux_packages=systemtap
-
-darwin_native_packages =
-
-ifneq ($(build_os),darwin)
-
-ifeq ($(strip $(FORCE_USE_SYSTEM_CLANG)),)
-darwin_native_packages+= native_llvm
-endif
-
-endif