Merge bitcoin/bitcoin#33079: ci: limit max stack size to 512 KiB

3b23f95e34 ci: limit max stack size to 512 KiB (dergoegge)
2931a87477 ci: limit stack size to 512kb in native macOS jobs (fanquake)

Pull request description:

  Picks up #31367.
  Closes #29840.

  Limit adjustment is moved until after compilation, otherwise compilation might not succeed.
  I've used compilation flags to limit the stack size in the native macOS jobs, because trying to use `ulimit` resulted in:
  ```bash
  + '[' -n 1 ']'
  + ulimit -s 262144
  /Users/runner/work/bitcoin/bitcoin/ci/test/03_test_script.sh: line 17: ulimit: stack size: cannot modify limit: Operation not permitted
  ```

  See example failures (`ulimit -s 64`) here: https://github.com/bitcoin/bitcoin/runs/46861548042.

ACKs for top commit:
  dergoegge:
    utACK 3b23f95e34

Tree-SHA512: 7e00626f3ca9e860d79a301af2427008ce27c329b618e24f95e7a55b284459a446216d2859c2e63be50abb9d4f0d343c12ff50445231652d354f225477928a35

ci/test/00_setup_env_arm.sh

@@ -16,6 +16,7 @@ export USE_BUSY_BOX=true
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
+export CI_LIMIT_STACK_SIZE=1
 # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1"
 # This could be removed once the ABI change warning does not show up by default
 export BITCOIN_CONFIG="-DREDUCE_EXPORTS=ON -DCMAKE_CXX_FLAGS='-Wno-psabi -Wno-error=maybe-uninitialized'"

ci/test/00_setup_env_mac_native.sh

@@ -11,7 +11,7 @@ export LC_ALL=C.UTF-8
 export PIP_PACKAGES="--break-system-packages zmq"
 export GOAL="install deploy"
 export CMAKE_GENERATOR="Ninja"
-export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DREDUCE_EXPORTS=ON"
+export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DREDUCE_EXPORTS=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""

ci/test/00_setup_env_mac_native_fuzz.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C.UTF-8
 
 export CMAKE_GENERATOR="Ninja"
-export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON"
+export BITCOIN_CONFIG="-DBUILD_FOR_FUZZING=ON -DCMAKE_EXE_LINKER_FLAGS='-Wl,-stack_size -Wl,0x80000'"
 export CI_OS_NAME="macos"
 export NO_DEPENDS=1
 export OSX_SDK=""

ci/test/00_setup_env_native_asan.sh

@@ -23,6 +23,7 @@ export APT_LLVM_V="20"
 export PACKAGES="systemtap-sdt-dev clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev python3-zmq qt6-base-dev qt6-tools-dev qt6-l10n-tools libevent-dev libboost-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}"
 export NO_DEPENDS=1
 export GOAL="install"
+export CI_LIMIT_STACK_SIZE=1
 export BITCOIN_CONFIG="\
  -DWITH_USDT=ON -DWITH_ZMQ=ON -DBUILD_GUI=ON \
  -DSANITIZERS=address,float-divide-by-zero,integer,undefined \

ci/test/00_setup_env_native_msan.sh

@@ -17,6 +17,7 @@ export CONTAINER_NAME="ci_native_msan"
 export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} llvm-${APT_LLVM_V}-dev libclang-${APT_LLVM_V}-dev libclang-rt-${APT_LLVM_V}-dev ninja-build"
 export DEP_OPTS="DEBUG=1 NO_QT=1 CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'"
 export GOAL="install"
+export CI_LIMIT_STACK_SIZE=1
 # Setting CMAKE_{C,CXX}_FLAGS_DEBUG flags to an empty string ensures that the flags set in MSAN_FLAGS remain unaltered.
 # _FORTIFY_SOURCE is not compatible with MSAN.
 export BITCOIN_CONFIG="\

ci/test/00_setup_env_native_previous_releases.sh

@@ -15,6 +15,7 @@ export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude fe
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
+export CI_LIMIT_STACK_SIZE=1
 export DOWNLOAD_PREVIOUS_RELEASES="true"
 export BITCOIN_CONFIG="\
  -DWITH_ZMQ=ON -DBUILD_GUI=ON -DREDUCE_EXPORTS=ON \

ci/test/00_setup_env_native_tsan.sh

@@ -12,5 +12,6 @@ export APT_LLVM_V="20"
 export PACKAGES="clang-${APT_LLVM_V} llvm-${APT_LLVM_V} libclang-rt-${APT_LLVM_V}-dev libc++abi-${APT_LLVM_V}-dev libc++-${APT_LLVM_V}-dev python3-zmq"
 export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'"
 export GOAL="install"
+export CI_LIMIT_STACK_SIZE=1
 export BITCOIN_CONFIG="-DWITH_ZMQ=ON -DSANITIZERS=thread \
 -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION -D_LIBCPP_REMOVE_TRANSITIVE_INCLUDES'"

ci/test/03_test_script.sh

@@ -140,6 +140,10 @@ bash -c "${PRINT_CCACHE_STATISTICS}"
 du -sh "${DEPENDS_DIR}"/*/
 du -sh "${PREVIOUS_RELEASES_DIR}"
 
+if [ -n "${CI_LIMIT_STACK_SIZE}" ]; then
+  ulimit -s 512
+fi
+
 if [ -n "$USE_VALGRIND" ]; then
   "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh"
 fi