From a33d03454508187abed764e55351ffcececc4c6e Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 6 Mar 2025 15:56:09 +0000 Subject: [PATCH 1/3] contrib: more selectively pick files for macOS SDK Only include what we really need. Skip 100s of mb of manpages, swiftmodules, modulemaps. Note that System/Library is only needed for the Qt build. --- contrib/macdeploy/README.md | 2 +- contrib/macdeploy/gen-sdk | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index d47ee6774e0..3cbf5b506be 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -52,7 +52,7 @@ path to `Xcode.app` (extracted in the previous stage) as the first argument. ``` The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`. -The `sha256sum` should be `c0c2e7bb92c1fee0c4e9f3a485e4530786732d6c6dd9e9f418c282aa6892f55d`. +The `sha256sum` should be `5aa41897b7f00abdaf1ece242dde3eb96a395746c09638b3a59720694712387d`. ## Deterministic macOS App Notes diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index f0bbabf81f9..f269369110d 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -68,6 +68,8 @@ def run(): """ def change_tarinfo_base(tarinfo): + if tarinfo.name and tarinfo.name.endswith((".swiftmodule", ".modulemap")): + return None if tarinfo.name and tarinfo.name.startswith("./"): tarinfo.name = str(pathlib.Path(alt_base_dir, tarinfo.name)) if tarinfo.linkname and tarinfo.linkname.startswith("./"): @@ -81,7 +83,9 @@ def run(): return tarinfo with cd(dir_to_add): # recursion already adds entries in sorted order - tarfp.add(".", recursive=True, filter=change_tarinfo_base) + tarfp.add("./usr/include", recursive=True, filter=change_tarinfo_base) + tarfp.add("./usr/lib", recursive=True, filter=change_tarinfo_base) + tarfp.add("./System/Library/Frameworks", recursive=True, filter=change_tarinfo_base) print("Creating output .tar.gz file...") with out_sdktgz_path.open("wb") as fp: From c1213a35abed01a97a9c52954919158f91f974d2 Mon Sep 17 00:00:00 2001 From: fanquake Date: Thu, 6 Mar 2025 16:02:12 +0000 Subject: [PATCH 2/3] macdeploy: disable compression in macOS gen-sdk script Starting with Python 3.11, Pythons gzip might delegate to zlib. Depending on the OS, i.e Ubuntu vs Fedora, the underlying zlib implementation might differ, resulting in different output. For now, or until a better solution exists, disable compression. This results in the SDK increasing in size to ~157mb. Which is not unreasonable, to regain determinism (and would be significantly worse without the previous commit). See: https://docs.python.org/3/library/gzip.html#gzip.compress Co-authored-by: stickies-v --- ci/test/01_base_install.sh | 2 +- contrib/guix/README.md | 2 +- contrib/macdeploy/README.md | 4 ++-- contrib/macdeploy/gen-sdk | 29 +++++++++++------------------ 4 files changed, 15 insertions(+), 22 deletions(-) diff --git a/ci/test/01_base_install.sh b/ci/test/01_base_install.sh index 9bb67aa5664..a0f4164bbc5 100755 --- a/ci/test/01_base_install.sh +++ b/ci/test/01_base_install.sh @@ -90,7 +90,7 @@ mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers" if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then - OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz" + OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar" OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}" if [ ! -f "$OSX_SDK_PATH" ]; then ${CI_RETRY_EXE} curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH" diff --git a/contrib/guix/README.md b/contrib/guix/README.md index 7f6b8232bba..aadc231e3be 100644 --- a/contrib/guix/README.md +++ b/contrib/guix/README.md @@ -37,7 +37,7 @@ You can then either point to the SDK using the `SDK_PATH` environment variable: ```sh # Extract the SDK tarball to /path/to/parent/dir/of/extracted/SDK/Xcode---extracted-SDK-with-libcxx-headers -tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar.gz +tar -C /path/to/parent/dir/of/extracted/SDK -xaf /path/to/Xcode---extracted-SDK-with-libcxx-headers.tar # Indicate where to locate the SDK tarball export SDK_PATH=/path/to/parent/dir/of/extracted/SDK diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index 3cbf5b506be..a47238616b7 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -51,8 +51,8 @@ path to `Xcode.app` (extracted in the previous stage) as the first argument. ./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' ``` -The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar.gz`. -The `sha256sum` should be `5aa41897b7f00abdaf1ece242dde3eb96a395746c09638b3a59720694712387d`. +The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar`. +The `sha256sum` should be `95b00dc41fa090747dc0a7907a5031a2fcb2d7f95c9584ba6bccdb99b6e3d498`. ## Deterministic macOS App Notes diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk index f269369110d..cf37929287e 100755 --- a/contrib/macdeploy/gen-sdk +++ b/contrib/macdeploy/gen-sdk @@ -2,9 +2,7 @@ import argparse import plistlib import pathlib -import sys import tarfile -import gzip import os import contextlib @@ -22,12 +20,12 @@ def run(): parser = argparse.ArgumentParser( description=__doc__, formatter_class=argparse.RawTextHelpFormatter) - parser.add_argument('xcode_app', metavar='XCODEAPP', nargs=1) - parser.add_argument("-o", metavar='OUTSDKTGZ', nargs=1, dest='out_sdktgz', required=False) + parser.add_argument('xcode_app', metavar='XCODEAPP', type=pathlib.Path) + parser.add_argument("-o", metavar='OUTSDKTAR', dest='out_sdkt', type=pathlib.Path, required=False) args = parser.parse_args() - xcode_app = pathlib.Path(args.xcode_app[0]).resolve() + xcode_app = args.xcode_app.resolve() assert xcode_app.is_dir(), "The supplied Xcode.app path '{}' either does not exist or is not a directory".format(xcode_app) xcode_app_plist = xcode_app.joinpath("Contents/version.plist") @@ -47,11 +45,7 @@ def run(): out_name = "Xcode-{xcode_version}-{xcode_build_id}-extracted-SDK-with-libcxx-headers".format(xcode_version=xcode_version, xcode_build_id=xcode_build_id) - if args.out_sdktgz: - out_sdktgz_path = pathlib.Path(args.out_sdktgz_path) - else: - # Construct our own out_sdktgz if not specified on the command line - out_sdktgz_path = pathlib.Path("./{}.tar.gz".format(out_name)) + out_sdkt_path = args.out_sdkt or pathlib.Path("./{}.tar".format(out_name)) def tarfp_add_with_base_change(tarfp, dir_to_add, alt_base_dir): """Add all files in dir_to_add to tarfp, but prepend alt_base_dir to the files' @@ -87,14 +81,13 @@ def run(): tarfp.add("./usr/lib", recursive=True, filter=change_tarinfo_base) tarfp.add("./System/Library/Frameworks", recursive=True, filter=change_tarinfo_base) - print("Creating output .tar.gz file...") - with out_sdktgz_path.open("wb") as fp: - with gzip.GzipFile(fileobj=fp, mode='wb', compresslevel=9, mtime=0) as gzf: - with tarfile.open(mode="w", fileobj=gzf, format=tarfile.GNU_FORMAT) as tarfp: - print("Adding MacOSX SDK {} files...".format(sdk_version)) - tarfp_add_with_base_change(tarfp, sdk_dir, out_name) - print("Done! Find the resulting gzipped tarball at:") - print(out_sdktgz_path.resolve()) + print("Creating output .tar file...") + with out_sdkt_path.open("wb") as fp: + with tarfile.open(mode="w", fileobj=fp, format=tarfile.PAX_FORMAT) as tarfp: + print("Adding MacOSX SDK {} files...".format(sdk_version)) + tarfp_add_with_base_change(tarfp, sdk_dir, out_name) + print("Done! Find the resulting tarball at:") + print(out_sdkt_path.resolve()) if __name__ == '__main__': run() From 3e01b5d0e7be3dabe7f52d70e577f03f31505ad9 Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 18 Nov 2025 14:34:48 +0000 Subject: [PATCH 3/3] contrib: rename gen-sdk to gen-sdk.py This puts it in scope for the Python linters. --- contrib/macdeploy/README.md | 4 ++-- contrib/macdeploy/{gen-sdk => gen-sdk.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename contrib/macdeploy/{gen-sdk => gen-sdk.py} (100%) diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index a47238616b7..1763c6cb513 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -44,11 +44,11 @@ xip -x Xcode_15.xip ### Step 2: Generating the SDK tarball from `Xcode.app` -To generate the SDK, run the script [`gen-sdk`](./gen-sdk) with the +To generate the SDK, run the script [`gen-sdk.py`](./gen-sdk.py) with the path to `Xcode.app` (extracted in the previous stage) as the first argument. ```bash -./contrib/macdeploy/gen-sdk '/path/to/Xcode.app' +./contrib/macdeploy/gen-sdk.py '/path/to/Xcode.app' ``` The generated archive should be: `Xcode-15.0-15A240d-extracted-SDK-with-libcxx-headers.tar`. diff --git a/contrib/macdeploy/gen-sdk b/contrib/macdeploy/gen-sdk.py similarity index 100% rename from contrib/macdeploy/gen-sdk rename to contrib/macdeploy/gen-sdk.py