From c73bffe74709c5df228447a6ddf3c8eb811d36bf Mon Sep 17 00:00:00 2001 From: godofredoc Date: Thu, 21 Dec 2023 17:46:11 -0800 Subject: [PATCH] Migrate verify_codesigned. (#139328) This is part of the migration of adhoc tests to shard tests. Bug: https://github.com/flutter/flutter/issues/139153 --- .ci.yaml | 10 +- dev/bots/test.dart | 318 +++++++++++++++++++++++++++++++ dev/bots/test/codesign_test.dart | 311 ++++++++++++++++++++++++++++++ dev/bots/test/common.dart | 10 + 4 files changed, 643 insertions(+), 6 deletions(-) create mode 100644 dev/bots/test/codesign_test.dart diff --git a/.ci.yaml b/.ci.yaml index e56e2e05fa6..d22f2b20826 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -3714,26 +3714,24 @@ targets: - name: Mac_x64 verify_binaries_codesigned enabled_branches: - flutter-\d+\.\d+-candidate\.\d+ - recipe: flutter/flutter + recipe: flutter/flutter_drone presubmit: false timeout: 60 properties: tags: > ["framework", "hostonly", "shard", "mac"] - validation: verify_binaries_codesigned - validation_name: Verify x64 binaries codesigned + shard: verify_binaries_codesigned - name: Mac_arm64 verify_binaries_codesigned enabled_branches: - flutter-\d+\.\d+-candidate\.\d+ - recipe: flutter/flutter + recipe: flutter/flutter_drone presubmit: false timeout: 60 properties: tags: > ["framework", "hostonly", "shard", "mac"] - validation: verify_binaries_codesigned - validation_name: Verify arm64 binaries codesigned + shard: verify_binaries_codesigned - name: Mac web_tool_tests recipe: flutter/flutter_drone diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 295e791303e..4cbc35c0dba 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -52,13 +52,16 @@ import 'dart:core' as system show print; import 'dart:core' hide print; import 'dart:io' as system show exit; import 'dart:io' hide exit; +import 'dart:io' as io; import 'dart:math' as math; import 'dart:typed_data'; import 'package:archive/archive.dart'; import 'package:file/file.dart' as fs; import 'package:file/local.dart'; +import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; +import 'package:process/process.dart'; import 'browser.dart'; import 'run_command.dart'; @@ -252,6 +255,7 @@ Future main(List args) async { 'analyze': _runAnalyze, 'fuchsia_precache': _runFuchsiaPrecache, 'docs': _runDocs, + 'verify_binaries_codesigned': _runVerifyCodesigned, kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. }); } catch (error, stackTrace) { @@ -1644,6 +1648,320 @@ Future _runDocs() async { ); } +// Verifies binaries are codesigned. +Future _runVerifyCodesigned() async { + printProgress('${green}Running binaries codesign verification$reset'); + await runCommand( + 'flutter', + [ + 'precache', + '--android', + '--ios', + '--macos' + ], + workingDirectory: flutterRoot, + ); + + await verifyExist(flutterRoot); + await verifySignatures(flutterRoot); +} + +const List expectedEntitlements = [ + 'com.apple.security.cs.allow-jit', + 'com.apple.security.cs.allow-unsigned-executable-memory', + 'com.apple.security.cs.allow-dyld-environment-variables', + 'com.apple.security.network.client', + 'com.apple.security.network.server', + 'com.apple.security.cs.disable-library-validation', +]; + +/// Binaries that are expected to be codesigned and have entitlements. +/// +/// This list should be kept in sync with the actual contents of Flutter's +/// cache. +Future> binariesWithEntitlements(String flutterRoot) async { + return [ + 'artifacts/engine/android-arm-profile/darwin-x64/gen_snapshot', + 'artifacts/engine/android-arm-release/darwin-x64/gen_snapshot', + 'artifacts/engine/android-arm64-profile/darwin-x64/gen_snapshot', + 'artifacts/engine/android-arm64-release/darwin-x64/gen_snapshot', + 'artifacts/engine/android-x64-profile/darwin-x64/gen_snapshot', + 'artifacts/engine/android-x64-release/darwin-x64/gen_snapshot', + 'artifacts/engine/darwin-x64-profile/gen_snapshot', + 'artifacts/engine/darwin-x64-profile/gen_snapshot_arm64', + 'artifacts/engine/darwin-x64-profile/gen_snapshot_x64', + 'artifacts/engine/darwin-x64-release/gen_snapshot', + 'artifacts/engine/darwin-x64-release/gen_snapshot_arm64', + 'artifacts/engine/darwin-x64-release/gen_snapshot_x64', + 'artifacts/engine/darwin-x64/flutter_tester', + 'artifacts/engine/darwin-x64/gen_snapshot', + 'artifacts/engine/darwin-x64/gen_snapshot_arm64', + 'artifacts/engine/darwin-x64/gen_snapshot_x64', + 'artifacts/engine/ios-profile/gen_snapshot_arm64', + 'artifacts/engine/ios-release/gen_snapshot_arm64', + 'artifacts/engine/ios/gen_snapshot_arm64', + 'artifacts/libimobiledevice/idevicescreenshot', + 'artifacts/libimobiledevice/idevicesyslog', + 'artifacts/libimobiledevice/libimobiledevice-1.0.6.dylib', + 'artifacts/libplist/libplist-2.0.3.dylib', + 'artifacts/openssl/libcrypto.1.1.dylib', + 'artifacts/openssl/libssl.1.1.dylib', + 'artifacts/usbmuxd/iproxy', + 'artifacts/usbmuxd/libusbmuxd-2.0.6.dylib', + 'dart-sdk/bin/dart', + 'dart-sdk/bin/dartaotruntime', + 'dart-sdk/bin/utils/gen_snapshot', + 'dart-sdk/bin/utils/wasm-opt', + ] + .map((String relativePath) => path.join(flutterRoot, 'bin', 'cache', relativePath)).toList(); +} + +/// Binaries that are only expected to be codesigned. +/// +/// This list should be kept in sync with the actual contents of Flutter's +/// cache. +Future> binariesWithoutEntitlements(String flutterRoot) async { + return [ + 'artifacts/engine/darwin-x64-profile/FlutterMacOS.framework/Versions/A/FlutterMacOS', + 'artifacts/engine/darwin-x64-release/FlutterMacOS.framework/Versions/A/FlutterMacOS', + 'artifacts/engine/darwin-x64/FlutterMacOS.framework/Versions/A/FlutterMacOS', + 'artifacts/engine/darwin-x64/font-subset', + 'artifacts/engine/darwin-x64/impellerc', + 'artifacts/engine/darwin-x64/libpath_ops.dylib', + 'artifacts/engine/darwin-x64/libtessellator.dylib', + 'artifacts/engine/ios-profile/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'artifacts/engine/ios-profile/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'artifacts/engine/ios-profile/extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'artifacts/engine/ios-profile/extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'artifacts/engine/ios-release/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'artifacts/engine/ios-release/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'artifacts/engine/ios-release/extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'artifacts/engine/ios-release/extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'artifacts/engine/ios/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'artifacts/engine/ios/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'artifacts/engine/ios/extension_safe/Flutter.xcframework/ios-arm64/Flutter.framework/Flutter', + 'artifacts/engine/ios/extension_safe/Flutter.xcframework/ios-arm64_x86_64-simulator/Flutter.framework/Flutter', + 'artifacts/ios-deploy/ios-deploy', + ] + .map((String relativePath) => path.join(flutterRoot, 'bin', 'cache', relativePath)).toList(); +} + +/// Verify the existence of all expected binaries in cache. +/// +/// This function ignores code signatures and entitlements, and is intended to +/// be run on every commit. It should throw if either new binaries are added +/// to the cache or expected binaries removed. In either case, this class' +/// [binariesWithEntitlements] or [binariesWithoutEntitlements] lists should +/// be updated accordingly. +Future verifyExist( + String flutterRoot, + {@visibleForTesting ProcessManager processManager = const LocalProcessManager() +}) async { + final Set foundFiles = {}; + final String cacheDirectory = path.join(flutterRoot, 'bin', 'cache'); + + + + for (final String binaryPath + in await findBinaryPaths(cacheDirectory, processManager: processManager)) { + if ((await binariesWithEntitlements(flutterRoot)).contains(binaryPath)) { + foundFiles.add(binaryPath); + } else if ((await binariesWithoutEntitlements(flutterRoot)).contains(binaryPath)) { + foundFiles.add(binaryPath); + } else { + throw Exception( + 'Found unexpected binary in cache: $binaryPath'); + } + } + + final List allExpectedFiles = await binariesWithEntitlements(flutterRoot) + await binariesWithoutEntitlements(flutterRoot); + if (foundFiles.length < allExpectedFiles.length) { + final List unfoundFiles = allExpectedFiles + .where( + (String file) => !foundFiles.contains(file), + ) + .toList(); + print( + 'Expected binaries not found in cache:\n\n${unfoundFiles.join('\n')}\n\n' + 'If this commit is removing binaries from the cache, this test should be fixed by\n' + 'removing the relevant entry from either the "binariesWithEntitlements" or\n' + '"binariesWithoutEntitlements" getters in dev/tools/lib/codesign.dart.', + ); + throw Exception('Did not find all expected binaries!'); + } + + print('All expected binaries present.'); +} + +/// Verify code signatures and entitlements of all binaries in the cache. +Future verifySignatures( + String flutterRoot, + {@visibleForTesting ProcessManager processManager = const LocalProcessManager()} +) async { + final List unsignedBinaries = []; + final List wrongEntitlementBinaries = []; + final List unexpectedBinaries = []; + final String cacheDirectory = path.join(flutterRoot, 'bin', 'cache'); + + for (final String binaryPath + in await findBinaryPaths(cacheDirectory, processManager: processManager)) { + bool verifySignature = false; + bool verifyEntitlements = false; + if ((await binariesWithEntitlements(flutterRoot)).contains(binaryPath)) { + verifySignature = true; + verifyEntitlements = true; + } + if ((await binariesWithoutEntitlements(flutterRoot)).contains(binaryPath)) { + verifySignature = true; + } + if (!verifySignature && !verifyEntitlements) { + unexpectedBinaries.add(binaryPath); + print('Unexpected binary $binaryPath found in cache!'); + continue; + } + print('Verifying the code signature of $binaryPath'); + final io.ProcessResult codeSignResult = await processManager.run( + [ + 'codesign', + '-vvv', + binaryPath, + ], + ); + if (codeSignResult.exitCode != 0) { + unsignedBinaries.add(binaryPath); + print( + 'File "$binaryPath" does not appear to be codesigned.\n' + 'The `codesign` command failed with exit code ${codeSignResult.exitCode}:\n' + '${codeSignResult.stderr}\n', + ); + continue; + } + if (verifyEntitlements) { + print('Verifying entitlements of $binaryPath'); + if (!(await hasExpectedEntitlements(binaryPath, flutterRoot, processManager: processManager))) { + wrongEntitlementBinaries.add(binaryPath); + } + } + } + + // First print all deviations from expectations + if (unsignedBinaries.isNotEmpty) { + print('Found ${unsignedBinaries.length} unsigned binaries:'); + unsignedBinaries.forEach(print); + } + + if (wrongEntitlementBinaries.isNotEmpty) { + print('Found ${wrongEntitlementBinaries.length} binaries with unexpected entitlements:'); + wrongEntitlementBinaries.forEach(print); + } + + if (unexpectedBinaries.isNotEmpty) { + print('Found ${unexpectedBinaries.length} unexpected binaries in the cache:'); + unexpectedBinaries.forEach(print); + } + + // Finally, exit on any invalid state + if (unsignedBinaries.isNotEmpty) { + throw Exception('Test failed because unsigned binaries detected.'); + } + + if (wrongEntitlementBinaries.isNotEmpty) { + throw Exception( + 'Test failed because files found with the wrong entitlements:\n' + '${wrongEntitlementBinaries.join('\n')}', + ); + } + + if (unexpectedBinaries.isNotEmpty) { + throw Exception('Test failed because unexpected binaries found in the cache.'); + } + print('Verified that binaries are codesigned and have expected entitlements.'); +} + +/// Find every binary file in the given [rootDirectory]. +Future> findBinaryPaths( + String rootDirectory, + {@visibleForTesting ProcessManager processManager = const LocalProcessManager() +}) async { + final List allBinaryPaths = []; + final io.ProcessResult result = await processManager.run( + [ + 'find', + rootDirectory, + '-type', + 'f', + ], + ); + final List allFiles = (result.stdout as String) + .split('\n') + .where((String s) => s.isNotEmpty) + .toList(); + + await Future.forEach(allFiles, (String filePath) async { + if (await isBinary(filePath, processManager: processManager)) { + allBinaryPaths.add(filePath); + print('Found: $filePath\n'); + } + }); + return allBinaryPaths; +} + +/// Check mime-type of file at [filePath] to determine if it is binary. +Future isBinary( + String filePath, + {@visibleForTesting ProcessManager processManager = const LocalProcessManager()} +) async { + final io.ProcessResult result = await processManager.run( + [ + 'file', + '--mime-type', + '-b', // is binary + filePath, + ], + ); + return (result.stdout as String).contains('application/x-mach-binary'); +} + +/// Check if the binary has the expected entitlements. +Future hasExpectedEntitlements( + String binaryPath, + String flutterRoot, + {@visibleForTesting ProcessManager processManager = const LocalProcessManager()} +) async { + final io.ProcessResult entitlementResult = await processManager.run( + [ + 'codesign', + '--display', + '--entitlements', + ':-', + binaryPath, + ], + ); + + if (entitlementResult.exitCode != 0) { + print( + 'The `codesign --entitlements` command failed with exit code ${entitlementResult.exitCode}:\n' + '${entitlementResult.stderr}\n', + ); + return false; + } + + bool passes = true; + final String output = entitlementResult.stdout as String; + for (final String entitlement in expectedEntitlements) { + final bool entitlementExpected = + (await binariesWithEntitlements(flutterRoot)).contains(binaryPath); + if (output.contains(entitlement) != entitlementExpected) { + print( + 'File "$binaryPath" ${entitlementExpected ? 'does not have expected' : 'has unexpected'} ' + 'entitlement $entitlement.', + ); + passes = false; + } + } + return passes; +} + /// Runs the skp_generator from the flutter/tests repo. /// /// See also the customer_tests shard. diff --git a/dev/bots/test/codesign_test.dart b/dev/bots/test/codesign_test.dart new file mode 100644 index 00000000000..7d2e5ff3186 --- /dev/null +++ b/dev/bots/test/codesign_test.dart @@ -0,0 +1,311 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@TestOn('mac-os') +library; + +import '../../../packages/flutter_tools/test/src/fake_process_manager.dart'; +import '../test.dart'; +import './common.dart'; + +void main() async { + const String flutterRoot = '/a/b/c'; + final List allExpectedFiles = await binariesWithEntitlements(flutterRoot) + await binariesWithoutEntitlements(flutterRoot); + final String allFilesStdout = allExpectedFiles.join('\n'); + final List withEntitlements = await binariesWithEntitlements(flutterRoot); + + group('verifyExist', () { + test('Not all files found', () async { + final ProcessManager processManager = FakeProcessManager.list( + [ + const FakeCommand( + command: [ + 'find', + '/a/b/c/bin/cache', + '-type', + 'f', + ], + stdout: '/a/b/c/bin/cache/artifacts/engine/android-arm-profile/darwin-x64/gen_snapshot', + ), + const FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + '/a/b/c/bin/cache/artifacts/engine/android-arm-profile/darwin-x64/gen_snapshot', + ], + stdout: 'application/x-mach-binary', + ), + ], + ); + expect( + () async => verifyExist(flutterRoot, processManager: processManager), + throwsExceptionWith('Did not find all expected binaries!'), + ); + }); + + test('All files found', () async { + final List commandList = []; + final FakeCommand findCmd = FakeCommand( + command: const [ + 'find', + '$flutterRoot/bin/cache', + '-type', + 'f',], + stdout: allFilesStdout, + ); + commandList.add(findCmd); + for (final String expectedFile in allExpectedFiles) { + commandList.add( + FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + expectedFile, + ], + stdout: 'application/x-mach-binary', + ) + ); + } + final ProcessManager processManager = FakeProcessManager.list(commandList); + await expectLater(verifyExist('/a/b/c', processManager: processManager), completes); + }); + }); + + group('findBinaryPaths', () { + test('All files found', () async { + final List commandList = []; + final FakeCommand findCmd = FakeCommand( + command: const [ + 'find', + '$flutterRoot/bin/cache', + '-type', + 'f',], + stdout: allFilesStdout, + ); + commandList.add(findCmd); + for (final String expectedFile in allExpectedFiles) { + commandList.add( + FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + expectedFile, + ], + stdout: 'application/x-mach-binary', + ) + ); + } + final ProcessManager processManager = FakeProcessManager.list(commandList); + final List foundFiles = await findBinaryPaths('$flutterRoot/bin/cache', processManager: processManager); + expect(foundFiles, allExpectedFiles); + }); + + test('Empty file list', () async { + final List commandList = []; + const FakeCommand findCmd = FakeCommand( + command: [ + 'find', + '$flutterRoot/bin/cache', + '-type', + 'f',], + ); + commandList.add(findCmd); + final ProcessManager processManager = FakeProcessManager.list(commandList); + final List foundFiles = await findBinaryPaths('$flutterRoot/bin/cache', processManager: processManager); + expect(foundFiles, []); + }); + + group('isBinary', () { + test('isTrue', () async { + final List commandList = []; + const String fileToCheck = '/a/b/c/one.zip'; + const FakeCommand findCmd = FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + fileToCheck, + ], + stdout: 'application/x-mach-binary', + ); + commandList.add(findCmd); + final ProcessManager processManager = FakeProcessManager.list(commandList); + final bool result = await isBinary(fileToCheck, processManager: processManager); + expect(result, isTrue); + }); + + test('isFalse', () async { + final List commandList = []; + const String fileToCheck = '/a/b/c/one.zip'; + const FakeCommand findCmd = FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + fileToCheck, + ], + stdout: 'text/xml', + ); + commandList.add(findCmd); + final ProcessManager processManager = FakeProcessManager.list(commandList); + final bool result = await isBinary(fileToCheck, processManager: processManager); + expect(result, isFalse); + }); + }); + + group('hasExpectedEntitlements', () { + test('expected entitlements', () async { + final List commandList = []; + const String fileToCheck = '/a/b/c/one.zip'; + const FakeCommand codesignCmd = FakeCommand( + command: [ + 'codesign', + '--display', + '--entitlements', + ':-', + fileToCheck, + ], + ); + commandList.add(codesignCmd); + final ProcessManager processManager = FakeProcessManager.list(commandList); + final bool result = await hasExpectedEntitlements(fileToCheck, flutterRoot, processManager: processManager); + expect(result, isTrue); + }); + + test('unexpected entitlements', () async { + final List commandList = []; + const String fileToCheck = '/a/b/c/one.zip'; + const FakeCommand codesignCmd = FakeCommand( + command: [ + 'codesign', + '--display', + '--entitlements', + ':-', + fileToCheck, + ], + exitCode: 1, + ); + commandList.add(codesignCmd); + final ProcessManager processManager = FakeProcessManager.list(commandList); + final bool result = await hasExpectedEntitlements(fileToCheck, flutterRoot, processManager: processManager); + expect(result, isFalse); + }); + }); + }); + + group('verifySignatures', () { + + test('succeeds if every binary is codesigned and has correct entitlements', () async { + final List commandList = []; + final FakeCommand findCmd = FakeCommand( + command: const [ + 'find', + '$flutterRoot/bin/cache', + '-type', + 'f',], + stdout: allFilesStdout, + ); + commandList.add(findCmd); + for (final String expectedFile in allExpectedFiles) { + commandList.add( + FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + expectedFile, + ], + stdout: 'application/x-mach-binary', + ) + ); + } + for (final String expectedFile in allExpectedFiles) { + commandList.add( + FakeCommand( + command: [ + 'codesign', + '-vvv', + expectedFile, + ], + ) + ); + if (withEntitlements.contains(expectedFile)) { + commandList.add( + FakeCommand( + command: [ + 'codesign', + '--display', + '--entitlements', + ':-', + expectedFile, + ], + stdout: expectedEntitlements.join('\n'), + ) + ); + } + } + final ProcessManager processManager = FakeProcessManager.list(commandList); + await expectLater(verifySignatures(flutterRoot, processManager: processManager), completes); + }); + + test('fails if binaries do not have the right entitlements', () async { + final List commandList = []; + final FakeCommand findCmd = FakeCommand( + command: const [ + 'find', + '$flutterRoot/bin/cache', + '-type', + 'f',], + stdout: allFilesStdout, + ); + commandList.add(findCmd); + for (final String expectedFile in allExpectedFiles) { + commandList.add( + FakeCommand( + command: [ + 'file', + '--mime-type', + '-b', + expectedFile, + ], + stdout: 'application/x-mach-binary', + ) + ); + } + for (final String expectedFile in allExpectedFiles) { + commandList.add( + FakeCommand( + command: [ + 'codesign', + '-vvv', + expectedFile, + ], + ) + ); + if (withEntitlements.contains(expectedFile)) { + commandList.add( + FakeCommand( + command: [ + 'codesign', + '--display', + '--entitlements', + ':-', + expectedFile, + ], + ) + ); + } + } + final ProcessManager processManager = FakeProcessManager.list(commandList); + + expect( + () async => verifySignatures(flutterRoot, processManager: processManager), + throwsExceptionWith('Test failed because files found with the wrong entitlements'), + ); + }); + }); +} diff --git a/dev/bots/test/common.dart b/dev/bots/test/common.dart index 86a3628f09a..876895cb1b2 100644 --- a/dev/bots/test/common.dart +++ b/dev/bots/test/common.dart @@ -21,3 +21,13 @@ void tryToDelete(Directory directory) { print('Failed to delete ${directory.path}: $error'); } } + +Matcher throwsExceptionWith(String messageSubString) { + return throwsA( + isA().having( + (Exception e) => e.toString(), + 'description', + contains(messageSubString), + ), + ); +}