From 5408e8514575bfbb04474ae7b9a7d2f919f2dea3 Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 6 Mar 2025 16:02:12 +0000 Subject: [PATCH] macdeploy: disable compression in macOS gen-sdk script Starting with Python 3.11, Pythons gzip might delegate to zlib. Depending on the OS, i.e Ubuntu vs Fedora, the underlying zlib implementation might differ, resulting in different output. For now, or until a better solution exists, disable compression. This results in the SDK increasing in size to ~157mb. Which is not unreasonable, to regain determinism (and would be significantly worse without the previous commit). See: https://docs.python.org/3/library/gzip.html#gzip.compress Co-authored-by: stickies-v Github-Pull: #32009 Rebased-From: c1213a35abed01a97a9c52954919158f91f974d2 --- ci/test/01_base_install.sh | 2 +- contrib/guix/README.md | 2 +- contrib/macdeploy/README.md | 4 ++-- contrib/macdeploy/gen-sdk | 29 +++++++++++------------------ 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index ce460987901..d26bda3d23b 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -89,7 +89,7 @@ mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then - OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar" OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" if [ ! -f "$OSX_SDK_PATH" ]; then ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 7f6b8232bba..aadc231e3be 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -37,7 +37,7 @@ You can then either point to the SDK using the `SDK_PATH` environment variable: ```sh # Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers -tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar # Indicate where to locate the SDK tarball export SDK_PATH=/path/to/parent/dir/of/extracted/SDK diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 3cbf5b506be..a47238616b7 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -51,8 +51,8 @@ path to `Xcode.app` (extracted in the previous stage) as the first argument. ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` -The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`. -The `sha256sum` should be `5aa41897b7f00abdaf1ece242dde3eb96a395746c09638b3a59720694712387d`. +The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar`. +The `sha256sum` should be `95b00dc41fa090747dc0a7907a5031a2fcb2d7f95c9584ba6bccdb99b6e3d498`. ## Deterministic macOS App Notes diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index f269369110d..cf37929287e 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -2,9 +2,7 @@ import argparse import plistlib import pathlib -import sys import tarfile -import gzip import os import contextlib @@ -22,12 +20,12 @@ def run(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) - parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + parser.add_argument('xcode_app', metavar='XCODEAPP', type=pathlib.Path) + parser.add_argument("-o", metavar='OUTSDKTAR', dest='out_sdkt', type=pathlib.Path, required=False) args = parser.parse_args() - xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + xcode_app = args.xcode_app.resolve() assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) xcode_app_plist = xcode_app.joinpath("Contents/version.plist") @@ -47,11 +45,7 @@ def run(): out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) - if args.out_sdktgz: - out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) - else: - # Construct our own out_sdktgz if not specified on the command line - out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + out_sdkt_path = args.out_sdkt or pathlib.Path("./{}.tar".format(out_name)) def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): """Add all files in dir_to_add to tarfp, but prepend alt_base_dir to the files' @@ -87,14 +81,13 @@ def run(): tarfp.add("./usr/lib", recursive=True, filter=change_tarinfo_base) tarfp.add("./System/Library/Frameworks", recursive=True, filter=change_tarinfo_base) - print("Creating output .tar.gz file...") - with out_sdktgz_path.open("wb") as fp: - with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: - with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: - print("Adding MacOSX SDK {} files...".format(sdk_version)) - tarfp_add_with_base_change(tarfp, sdk_dir, out_name) - print("Done! Find the resulting gzipped tarball at:") - print(out_sdktgz_path.resolve()) + print("Creating output .tar file...") + with out_sdkt_path.open("wb") as fp: + with tarfile.open(mode="w", fileobj=fp, format=tarfile.PAX_FORMAT) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Done! Find the resulting tarball at:") + print(out_sdkt_path.resolve()) if __name__ == '__main__': run()