From ec52964471a8cd904cb643126cb5cb21b61ccc40 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 15 Aug 2024 16:31:08 -0700 Subject: [PATCH 1/5] iOS: Add dSYM binaries to without_entitlements.txt In flutter/engine#54414, we added dSYM files for physical and simulator binaries in both regular and extension-safe framework builds, but did not add the dSYMs to the without_entitlements.txt list. This passed all engine pre/post-submit tests, as well as framework tests, but failed during release codesigning in Cocoon in a test here: https://github.com/flutter/cocoon/blob/d849b14bab90e0f90e2f7667e37c9f9a5696b918/cipd_packages/codesign/lib/src/file_codesign_visitor.dart#L305-L313 This adds the missing files to without_entitlements.txt, which fixes a codesigning error as seen in this build log: https://ci.chromium.org/ui/p/dart-internal/builders/flutter/Mac%20Production%20Engine%20Drone/13590/overview A corresponding change was landed on the flutter-3.24-candidate.1 branch: https://github.com/flutter/engine/pull/54573 The build associated with that patch correctly completed code signing in this build: https://ci.chromium.org/ui/p/dart-internal/builders/flutter/Mac%20engine_release_builder/688/overview And more specifically, this sub-build: https://ci.chromium.org/ui/p/dart-internal/builders/flutter/Mac%20Production%20Engine%20Drone/13650/overview And even more specifically, this build step: https://logs.chromium.org/logs/dart-internal/buildbucket/cr-buildbucket/8739493904842446705/+/u/Global_generators/Codesign__Volumes_Work_s_w_ir_cache_builder_src_out_release_unsigned_artifacts.zip/codesign_Apple_engine_binaries/stdout Additionally, this patch adds `sky_utils.assert_valid_codesign_config()` which fails the script (and thus the build) with an error message if any file in scope for code signing (i.e. Mach-O binaries) is not listed in the code-signing config (entitlements.txt, without_entitlements.txt), or any listed file is not found on disk. Issue: https://github.com/flutter/flutter/issues/116493 --- sky/tools/create_ios_framework.py | 49 ++++++++++++++++----------- sky/tools/sky_utils.py | 56 +++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 20 deletions(-) diff --git a/sky/tools/create_ios_framework.py b/sky/tools/create_ios_framework.py index 4a40e15a71367..27e9deb05be08 100644 --- a/sky/tools/create_ios_framework.py +++ b/sky/tools/create_ios_framework.py @@ -164,26 +164,35 @@ def create_framework( # pylint: disable=too-many-arguments def zip_archive(dst): - sky_utils.write_codesign_config(os.path.join(dst, 'entitlements.txt'), ['gen_snapshot_arm64']) - - sky_utils.write_codesign_config( - os.path.join(dst, 'without_entitlements.txt'), [ - 'Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', - 'Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', - 'extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', - 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter' - ] - ) - - sky_utils.create_zip( - dst, 'artifacts.zip', [ - 'gen_snapshot_arm64', - 'Flutter.xcframework', - 'entitlements.txt', - 'without_entitlements.txt', - 'extension_safe/Flutter.xcframework', - ] - ) + # pylint: disable=line-too-long + with_entitlements = ['gen_snapshot_arm64'] + with_entitlements_file = os.path.join(dst, 'entitlements.txt') + sky_utils.write_codesign_config(with_entitlements_file, with_entitlements) + + without_entitlements = [ + 'Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + 'Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + ] + without_entitlements_file = os.path.join(dst, 'without_entitlements.txt') + sky_utils.write_codesign_config(without_entitlements_file, without_entitlements) + # pylint: enable=line-too-long + + zip_contents = [ + 'gen_snapshot_arm64', + 'Flutter.xcframework', + 'entitlements.txt', + 'without_entitlements.txt', + 'extension_safe/Flutter.xcframework', + ] + sky_utils.assert_valid_codesign_config(dst, zip_contents, with_entitlements, without_entitlements) + sky_utils.create_zip(dst, 'artifacts.zip', zip_contents) def process_framework(args, dst, framework_binary, dsym): diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index 5e6a762fa7b1c..c152f2b32a3c1 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -25,6 +25,62 @@ def assert_file(path, what): sys.exit(os.EX_NOINPUT) +def assert_valid_codesign_config(framework_dir, zip_contents, entitlements, without_entitlements): + """Exits with exit code 1 if the codesign configuration contents are incorrect. + All Mach-O binaries found within zip_contents exactly must be listed in + either entitlements or without_entitlements.""" + binaries = set() + for zip_content_path in zip_contents: + # If file, check if Mach-O binary. + if _is_macho_binary(os.path.join(framework_dir, zip_content_path)): + binaries.add(zip_content_path) + # If directory, check transitive closure of files for Mach-O binaries. + for root, _, files in os.walk(os.path.join(framework_dir, zip_content_path)): + for file in [os.path.join(root, f) for f in files]: + if _is_macho_binary(file): + binaries.add(os.path.relpath(file, framework_dir)) + + # Verify that all Mach-O binaries are listed in either entitlements or without_entitlements. + listed_binaries = set(entitlements + without_entitlements) + if listed_binaries != binaries: + log_error( + 'ERROR: binaries listed in entitlements.txt and without_entitlements.txt do not ' + 'match the set of binaries in the files to be zipped' + ) + log_error('Binaries found in files to be zipped:') + for file in sorted(binaries): + log_error(' ' + file) + + not_listed = sorted(binaries - listed_binaries) + if not_listed: + log_error('Binaries NOT LISTED in entitlements.txt/without_entitlements.txt:') + for file in not_listed: + log_error(' ' + file) + + not_found = sorted(listed_binaries - binaries) + if not_found: + log_error('Binaries listed in entitlements.txt/without_entitlements.txt but NOT FOUND:') + for file in not_found: + log_error(' ' + file) + sys.exit(1) + + +def _is_macho_binary(filename): + """Returns True if the specified path is file and a Mach-O binary.""" + if not os.path.isfile(filename): + return False + + with open(filename, 'rb') as file: + chunk = file.read(4) + return ( + chunk == b'\xca\xfe\xba\xbe' or # Mach-O Universal Big Endian + chunk == b'\xce\xfa\xed\xfe' or # Mach-O Little Endian (32-bit) + chunk == b'\xcf\xfa\xed\xfe' or # Mach-O Little Endian (64-bit) + chunk == b'\xfe\xed\xfa\xce' or # Mach-O Big Endian (32-bit) + chunk == b'\xfe\xed\xfa\xcf' # Mach-O Big Endian (64-bit) + ) + + def buildroot_relative_path(path): """Returns the absolute path to the specified buildroot-relative path.""" buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..', '..')) From b6dc292388955ae6e6937499b1197e055b373d63 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 15 Aug 2024 21:11:37 -0700 Subject: [PATCH 2/5] Fail if duplicates found in entitlements lists Eliminate duplicate value found in without_entitlements.txt --- sky/tools/create_ios_framework.py | 1 - sky/tools/sky_utils.py | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sky/tools/create_ios_framework.py b/sky/tools/create_ios_framework.py index 27e9deb05be08..6b43fbd282cc7 100644 --- a/sky/tools/create_ios_framework.py +++ b/sky/tools/create_ios_framework.py @@ -175,7 +175,6 @@ def zip_archive(dst): 'Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', 'Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', 'extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', - 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', 'extension_safe/Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index c152f2b32a3c1..fc358d0b2b998 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -29,6 +29,18 @@ def assert_valid_codesign_config(framework_dir, zip_contents, entitlements, with """Exits with exit code 1 if the codesign configuration contents are incorrect. All Mach-O binaries found within zip_contents exactly must be listed in either entitlements or without_entitlements.""" + if _contains_duplicates(entitlements): + log_error('ERROR: duplicate value(s) found in entitlements.txt') + sys.exit(1) + + if _contains_duplicates(without_entitlements): + log_error('ERROR: duplicate value(s) found in without_entitlements.txt') + sys.exit(1) + + if _contains_duplicates(entitlements + without_entitlements): + log_error('ERROR: value(s) found in both entitlements and without_entitlements.txt') + sys.exit(1) + binaries = set() for zip_content_path in zip_contents: # If file, check if Mach-O binary. @@ -65,6 +77,11 @@ def assert_valid_codesign_config(framework_dir, zip_contents, entitlements, with sys.exit(1) +def _contains_duplicates(strings): + """Returns true if the list of strings contains a duplicate value.""" + return len(strings) != len(set(strings)) + + def _is_macho_binary(filename): """Returns True if the specified path is file and a Mach-O binary.""" if not os.path.isfile(filename): From b834ef83c7a0e730b46037c5c80f07bae4046489 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 15 Aug 2024 21:25:45 -0700 Subject: [PATCH 3/5] Only add dSYMs to entitlements if --dsym passed --- sky/tools/create_ios_framework.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/sky/tools/create_ios_framework.py b/sky/tools/create_ios_framework.py index 6b43fbd282cc7..fab8604388b14 100644 --- a/sky/tools/create_ios_framework.py +++ b/sky/tools/create_ios_framework.py @@ -91,7 +91,7 @@ def main(): gen_snapshot = os.path.join(x64_out_dir, 'gen_snapshot_x64') sky_utils.copy_binary(gen_snapshot, os.path.join(dst, 'gen_snapshot_x64')) - zip_archive(dst) + zip_archive(dst, args) return 0 @@ -163,7 +163,7 @@ def create_framework( # pylint: disable=too-many-arguments return 0 -def zip_archive(dst): +def zip_archive(dst, args): # pylint: disable=line-too-long with_entitlements = ['gen_snapshot_arm64'] with_entitlements_file = os.path.join(dst, 'entitlements.txt') @@ -171,14 +171,18 @@ def zip_archive(dst): without_entitlements = [ 'Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', - 'Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', 'Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', - 'Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', 'extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', - 'extension_safe/Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', - 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', ] + if args.dsym: + without_entitlements.extend([ + 'Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + 'Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + 'extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/dSYMs/Flutter.framework.dSYM/Contents/Resources/DWARF/Flutter', + ]) + without_entitlements_file = os.path.join(dst, 'without_entitlements.txt') sky_utils.write_codesign_config(without_entitlements_file, without_entitlements) # pylint: enable=line-too-long From ea4994f91b88c44bc75a10913017a8d044d69048 Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 15 Aug 2024 21:53:10 -0700 Subject: [PATCH 4/5] PYLIIIIIIINT!! --- sky/tools/sky_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index fc358d0b2b998..e1453e5ddaff1 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -89,12 +89,12 @@ def _is_macho_binary(filename): with open(filename, 'rb') as file: chunk = file.read(4) - return ( - chunk == b'\xca\xfe\xba\xbe' or # Mach-O Universal Big Endian - chunk == b'\xce\xfa\xed\xfe' or # Mach-O Little Endian (32-bit) - chunk == b'\xcf\xfa\xed\xfe' or # Mach-O Little Endian (64-bit) - chunk == b'\xfe\xed\xfa\xce' or # Mach-O Big Endian (32-bit) - chunk == b'\xfe\xed\xfa\xcf' # Mach-O Big Endian (64-bit) + return chunk in ( + b'\xca\xfe\xba\xbe', # Mach-O Universal Big Endian + b'\xce\xfa\xed\xfe', # Mach-O Little Endian (32-bit) + b'\xcf\xfa\xed\xfe', # Mach-O Little Endian (64-bit) + b'\xfe\xed\xfa\xce', # Mach-O Big Endian (32-bit) + b'\xfe\xed\xfa\xcf', # Mach-O Big Endian (64-bit) ) From e33d20304773fb67ae9aeb2df781a199b0bd5b9a Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Thu, 15 Aug 2024 21:59:36 -0700 Subject: [PATCH 5/5] More descriptive exit status while I'm at it --- sky/tools/sky_utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sky/tools/sky_utils.py b/sky/tools/sky_utils.py index e1453e5ddaff1..30c3eb429a3a3 100644 --- a/sky/tools/sky_utils.py +++ b/sky/tools/sky_utils.py @@ -31,15 +31,15 @@ def assert_valid_codesign_config(framework_dir, zip_contents, entitlements, with either entitlements or without_entitlements.""" if _contains_duplicates(entitlements): log_error('ERROR: duplicate value(s) found in entitlements.txt') - sys.exit(1) + sys.exit(os.EX_DATAERR) if _contains_duplicates(without_entitlements): log_error('ERROR: duplicate value(s) found in without_entitlements.txt') - sys.exit(1) + sys.exit(os.EX_DATAERR) if _contains_duplicates(entitlements + without_entitlements): log_error('ERROR: value(s) found in both entitlements and without_entitlements.txt') - sys.exit(1) + sys.exit(os.EX_DATAERR) binaries = set() for zip_content_path in zip_contents: @@ -74,7 +74,7 @@ def assert_valid_codesign_config(framework_dir, zip_contents, entitlements, with log_error('Binaries listed in entitlements.txt/without_entitlements.txt but NOT FOUND:') for file in not_found: log_error(' ' + file) - sys.exit(1) + sys.exit(os.EX_NOINPUT) def _contains_duplicates(strings):