From 062022b950c44498a5f765c95f02218478166e50 Mon Sep 17 00:00:00 2001 From: Jenn Magder Date: Mon, 11 Jan 2021 10:29:06 -0800 Subject: [PATCH] Move ios_content_validation_test to pre-submit tools test (#73577) --- .../tasks/ios_content_validation_test.dart | 170 -------------- dev/devicelab/lib/framework/ios.dart | 36 --- .../ios_content_validation_test.dart | 219 ++++++++++++++++++ .../macos_content_validation_test.dart | 56 +---- .../flutter_tools/test/src/darwin_common.dart | 52 +++++ 5 files changed, 277 insertions(+), 256 deletions(-) create mode 100644 packages/flutter_tools/test/integration.shard/ios_content_validation_test.dart create mode 100644 packages/flutter_tools/test/src/darwin_common.dart diff --git a/dev/devicelab/bin/tasks/ios_content_validation_test.dart b/dev/devicelab/bin/tasks/ios_content_validation_test.dart index e84b346ec01..21009e00494 100644 --- a/dev/devicelab/bin/tasks/ios_content_validation_test.dart +++ b/dev/devicelab/bin/tasks/ios_content_validation_test.dart @@ -2,11 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/apk_utils.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/ios.dart'; import 'package:flutter_devicelab/framework/task_result.dart'; import 'package:flutter_devicelab/framework/utils.dart'; import 'package:path/path.dart' as path; @@ -15,173 +12,6 @@ Future main() async { await task(() async { try { await runProjectTest((FlutterProject flutterProject) async { - section('Build app with with --obfuscate'); - await inDirectory(flutterProject.rootPath, () async { - await flutter('build', options: [ - 'ios', - '--release', - '--obfuscate', - '--split-debug-info=foo/', - '--no-codesign', - ]); - }); - final String buildPath = path.join( - flutterProject.rootPath, - 'build', - 'ios', - 'iphoneos', - ); - final String outputAppPath = path.join( - buildPath, - 'Runner.app', - ); - final Directory outputAppFramework = Directory(path.join( - outputAppPath, - 'Frameworks', - 'App.framework', - )); - - final File outputAppFrameworkBinary = File(path.join( - outputAppFramework.path, - 'App', - )); - - if (!outputAppFrameworkBinary.existsSync()) { - fail('Failed to produce expected output at ${outputAppFrameworkBinary.path}'); - } - - if (await dartObservatoryBonjourServiceFound(outputAppPath)) { - throw TaskResult.failure('Release bundle has unexpected NSBonjourServices'); - } - if (await localNetworkUsageFound(outputAppPath)) { - throw TaskResult.failure('Release bundle has unexpected NSLocalNetworkUsageDescription'); - } - - section('Validate obfuscation'); - - // Verify that an identifier from the Dart project code is not present - // in the compiled binary. - await inDirectory(flutterProject.rootPath, () async { - final String response = await eval( - 'grep', - [flutterProject.name, outputAppFrameworkBinary.path], - canFail: true, - ); - if (response.trim().contains('matches')) { - throw TaskResult.failure('Found project name in obfuscated dart library'); - } - }); - - section('Validate release contents'); - - final Directory outputFlutterFramework = Directory(path.join( - flutterProject.rootPath, - outputAppPath, - 'Frameworks', - 'Flutter.framework', - )); - - checkDirectoryNotExists(path.join(outputFlutterFramework.path, 'Headers')); - checkDirectoryNotExists(path.join(outputFlutterFramework.path, 'Modules')); - final File outputFlutterFrameworkBinary = File(path.join( - outputFlutterFramework.path, - 'Flutter', - )); - - if (!outputFlutterFrameworkBinary.existsSync()) { - fail('Failed to produce expected output at ${outputFlutterFrameworkBinary.path}'); - } - - // Archiving should contain a bitcode blob, but not building in release. - // This mimics Xcode behavior and present a developer from having to install a - // 300+MB app to test devices. - if (await containsBitcode(outputFlutterFrameworkBinary.path)) { - throw TaskResult.failure('Bitcode present in Flutter.framework'); - } - - section('Xcode backend script'); - - outputFlutterFramework.deleteSync(recursive: true); - outputAppFramework.deleteSync(recursive: true); - if (outputFlutterFramework.existsSync() || outputAppFramework.existsSync()) { - fail('Failed to delete embedded frameworks'); - } - - final String xcodeBackendPath = path.join( - flutterDirectory.path, - 'packages', - 'flutter_tools', - 'bin', - 'xcode_backend.sh' - ); - - // Simulate a common Xcode build setting misconfiguration - // where FLUTTER_APPLICATION_PATH is missing - final int result = await exec( - xcodeBackendPath, - ['embed_and_thin'], - environment: { - 'SOURCE_ROOT': flutterProject.iosPath, - 'BUILT_PRODUCTS_DIR': path.join( - flutterProject.rootPath, - 'build', - 'ios', - 'Release-iphoneos', - ), - 'TARGET_BUILD_DIR': buildPath, - 'FRAMEWORKS_FOLDER_PATH': 'Runner.app/Frameworks', - 'VERBOSE_SCRIPT_LOGGING': '1', - 'FLUTTER_BUILD_MODE': 'release', - 'ACTION': 'install', // Skip bitcode stripping since we just checked that above. - }, - ); - - if (result != 0) { - fail('xcode_backend embed_and_thin failed'); - } - - if (!outputFlutterFrameworkBinary.existsSync()) { - fail('Failed to re-embed ${outputFlutterFrameworkBinary.path}'); - } - - if (!outputAppFrameworkBinary.existsSync()) { - fail('Failed to re-embed ${outputAppFrameworkBinary.path}'); - } - - section('Clean build'); - - await inDirectory(flutterProject.rootPath, () async { - await flutter('clean'); - }); - - section('Validate debug contents'); - - await inDirectory(flutterProject.rootPath, () async { - await flutter('build', options: [ - 'ios', - '--debug', - '--no-codesign', - ]); - }); - - // Debug should also not contain bitcode. - if (await containsBitcode(outputFlutterFrameworkBinary.path)) { - throw TaskResult.failure('Bitcode present in Flutter.framework'); - } - - if (!await dartObservatoryBonjourServiceFound(outputAppPath)) { - throw TaskResult.failure('Debug bundle is missing NSBonjourServices'); - } - if (!await localNetworkUsageFound(outputAppPath)) { - throw TaskResult.failure('Debug bundle is missing NSLocalNetworkUsageDescription'); - } - - section('Clean build'); - - await inDirectory(flutterProject.rootPath, () async { - await flutter('clean'); - }); - section('Archive'); await inDirectory(flutterProject.rootPath, () async { diff --git a/dev/devicelab/lib/framework/ios.dart b/dev/devicelab/lib/framework/ios.dart index 0a1759e233e..0cd00440881 100644 --- a/dev/devicelab/lib/framework/ios.dart +++ b/dev/devicelab/lib/framework/ios.dart @@ -4,8 +4,6 @@ import 'dart:convert'; -import 'package:path/path.dart' as path; - import 'utils.dart'; typedef SimulatorFunction = Future Function(String deviceId); @@ -58,40 +56,6 @@ Future containsBitcode(String pathToBinary) async { return !emptyBitcodeMarkerFound; } -Future dartObservatoryBonjourServiceFound(String appBundlePath) async => - (await eval( - 'plutil', - [ - '-extract', - 'NSBonjourServices', - 'xml1', - '-o', - '-', - path.join( - appBundlePath, - 'Info.plist', - ), - ], - canFail: true, - )).contains('_dartobservatory._tcp'); - -Future localNetworkUsageFound(String appBundlePath) async => - await exec( - 'plutil', - [ - '-extract', - 'NSLocalNetworkUsageDescription', - 'xml1', - '-o', - '-', - path.join( - appBundlePath, - 'Info.plist', - ), - ], - canFail: true, - ) == 0; - /// Creates and boots a new simulator, passes the new simulator's identifier to /// `testFunction`. /// diff --git a/packages/flutter_tools/test/integration.shard/ios_content_validation_test.dart b/packages/flutter_tools/test/integration.shard/ios_content_validation_test.dart new file mode 100644 index 00000000000..d769d20d5ce --- /dev/null +++ b/packages/flutter_tools/test/integration.shard/ios_content_validation_test.dart @@ -0,0 +1,219 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:file_testing/file_testing.dart'; +import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/base/io.dart'; +import 'package:flutter_tools/src/build_info.dart'; + +import '../src/common.dart'; +import '../src/darwin_common.dart'; +import 'test_utils.dart'; + +void main() { + for (final BuildMode buildMode in [BuildMode.debug, BuildMode.release]) { + group(buildMode.name, () { + String flutterRoot; + String projectRoot; + String flutterBin; + Directory tempDir; + + Directory buildPath; + Directory outputApp; + Directory outputFlutterFramework; + File outputFlutterFrameworkBinary; + Directory outputAppFramework; + File outputAppFrameworkBinary; + + setUpAll(() { + flutterRoot = getFlutterRoot(); + tempDir = createResolvedTempDirectorySync('ios_content_validation.'); + flutterBin = fileSystem.path.join( + flutterRoot, + 'bin', + 'flutter', + ); + + processManager.runSync([ + flutterBin, + ...getLocalEngineArguments(), + 'create', + '--platforms=ios', + '-i', + 'objc', + 'hello', + ], workingDirectory: tempDir.path); + + projectRoot = tempDir.childDirectory('hello').path; + + processManager.runSync([ + flutterBin, + ...getLocalEngineArguments(), + 'build', + 'ios', + '--verbose', + '--no-codesign', + '--${buildMode.name}', + '--obfuscate', + '--split-debug-info=foo/', + ], workingDirectory: projectRoot); + + buildPath = fileSystem.directory(fileSystem.path.join( + projectRoot, + 'build', + 'ios', + 'iphoneos', + )); + + outputApp = buildPath.childDirectory('Runner.app'); + + outputFlutterFramework = fileSystem.directory( + fileSystem.path.join( + outputApp.path, + 'Frameworks', + 'Flutter.framework', + ), + ); + + outputFlutterFrameworkBinary = outputFlutterFramework.childFile('Flutter'); + + outputAppFramework = fileSystem.directory(fileSystem.path.join( + outputApp.path, + 'Frameworks', + 'App.framework', + )); + + outputAppFrameworkBinary = outputAppFramework.childFile('App'); + }); + + tearDownAll(() { + tryToDelete(tempDir); + }); + + testWithoutContext('flutter build ios builds a valid app', () { + expect(outputAppFramework.childFile('App'), exists); + + final File vmSnapshot = fileSystem.file(fileSystem.path.join( + outputAppFramework.path, + 'flutter_assets', + 'vm_snapshot_data', + )); + + expect(vmSnapshot.existsSync(), buildMode == BuildMode.debug); + + expect(outputFlutterFramework.childDirectory('Headers'), isNot(exists)); + expect(outputFlutterFramework.childDirectory('Modules'), isNot(exists)); + + // Archiving should contain a bitcode blob, but not building. + // This mimics Xcode behavior and prevents a developer from having to install a + // 300+MB app. + expect(containsBitcode(outputFlutterFrameworkBinary.path, processManager), isFalse); + }); + + testWithoutContext('Info.plist dart observatory Bonjour service', () { + final String infoPlistPath = fileSystem.path.join( + outputApp.path, + 'Info.plist', + ); + final ProcessResult bonjourServices = processManager.runSync( + [ + 'plutil', + '-extract', + 'NSBonjourServices', + 'xml1', + '-o', + '-', + infoPlistPath, + ], + ); + final bool bonjourServicesFound = (bonjourServices.stdout as String).contains('_dartobservatory._tcp'); + expect(bonjourServicesFound, buildMode == BuildMode.debug); + + final ProcessResult localNetworkUsage = processManager.runSync( + [ + 'plutil', + '-extract', + 'NSLocalNetworkUsageDescription', + 'xml1', + '-o', + '-', + infoPlistPath, + ], + ); + final bool localNetworkUsageFound = localNetworkUsage.exitCode == 0; + expect(localNetworkUsageFound, buildMode == BuildMode.debug); + }); + + testWithoutContext('check symbols', () { + final ProcessResult symbols = processManager.runSync( + [ + 'nm', + '-g', + outputAppFrameworkBinary.path, + '-arch', + 'arm64', + ], + ); + final bool aotSymbolsFound = (symbols.stdout as String).contains('_kDartVmSnapshot'); + expect(aotSymbolsFound, buildMode != BuildMode.debug); + }); + + testWithoutContext('xcode_backend embed_and_thin', () { + outputFlutterFramework.deleteSync(recursive: true); + outputAppFramework.deleteSync(recursive: true); + expect(outputFlutterFrameworkBinary.existsSync(), isFalse); + expect(outputAppFrameworkBinary.existsSync(), isFalse); + + final String xcodeBackendPath = fileSystem.path.join( + flutterRoot, + 'packages', + 'flutter_tools', + 'bin', + 'xcode_backend.sh', + ); + + // Simulate a common Xcode build setting misconfiguration + // where FLUTTER_APPLICATION_PATH is missing + final ProcessResult xcodeBackendResult = processManager.runSync( + [ + xcodeBackendPath, + 'embed_and_thin', + ], + environment: { + 'SOURCE_ROOT': fileSystem.path.join(projectRoot, 'ios'), + 'BUILT_PRODUCTS_DIR': fileSystem.path.join( + projectRoot, + 'build', + 'ios', + 'Release-iphoneos', + ), + 'TARGET_BUILD_DIR': buildPath.path, + 'FRAMEWORKS_FOLDER_PATH': 'Runner.app/Frameworks', + 'VERBOSE_SCRIPT_LOGGING': '1', + 'FLUTTER_BUILD_MODE': 'release', + 'ACTION': 'install', + // Skip bitcode stripping since we just checked that above. + }, + ); + + expect(xcodeBackendResult.exitCode, 0); + expect(outputFlutterFrameworkBinary.existsSync(), isTrue); + expect(outputAppFrameworkBinary.existsSync(), isTrue); + }, skip: !platform.isMacOS || buildMode != BuildMode.release); + + testWithoutContext('validate obfuscation', () { + final ProcessResult grepResult = processManager.runSync([ + 'grep', + '-i', + 'hello', + outputAppFrameworkBinary.path, + ]); + expect(grepResult.stdout, isNot(contains('matches'))); + }); + }, + skip: !platform.isMacOS, + timeout: const Timeout(Duration(minutes: 5)), + ); + } +} diff --git a/packages/flutter_tools/test/integration.shard/macos_content_validation_test.dart b/packages/flutter_tools/test/integration.shard/macos_content_validation_test.dart index 139edd0abd1..0e1dc6c2e50 100644 --- a/packages/flutter_tools/test/integration.shard/macos_content_validation_test.dart +++ b/packages/flutter_tools/test/integration.shard/macos_content_validation_test.dart @@ -5,15 +5,15 @@ import 'package:file_testing/file_testing.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/io.dart'; -import 'package:flutter_tools/src/convert.dart'; import '../src/common.dart'; +import '../src/darwin_common.dart'; import 'test_utils.dart'; void main() { for (final String buildMode in ['Debug', 'Release']) { final String buildModeLower = buildMode.toLowerCase(); - test('flutter build macos --$buildModeLower builds a valid app', () async { + test('flutter build macos --$buildModeLower builds a valid app', () { final String workingDirectory = fileSystem.path.join( getFlutterRoot(), 'dev', @@ -26,13 +26,13 @@ void main() { 'flutter', ); - await processManager.run([ + processManager.runSync([ flutterBin, ...getLocalEngineArguments(), 'clean', ], workingDirectory: workingDirectory); - final ProcessResult result = await processManager.run([ + final ProcessResult result = processManager.runSync([ flutterBin, ...getLocalEngineArguments(), 'build', @@ -112,11 +112,11 @@ void main() { .childDirectory('A') .childFile('FlutterMacOS'); expect( - await containsBitcode(outputFlutterFrameworkBinary.path), + containsBitcode(outputFlutterFrameworkBinary.path, processManager), isFalse, ); - await processManager.run([ + processManager.runSync([ flutterBin, ...getLocalEngineArguments(), 'clean', @@ -126,47 +126,3 @@ void main() { ); } } - -Future containsBitcode(String pathToBinary) async { - // See: https://stackoverflow.com/questions/32755775/how-to-check-a-static-library-is-built-contain-bitcode - final ProcessResult result = await processManager.run([ - 'otool', - '-l', - '-arch', - 'arm64', - pathToBinary, - ]); - final String loadCommands = result.stdout as String; - if (!loadCommands.contains('__LLVM')) { - return false; - } - // Presence of the section may mean a bitcode marker was embedded (size=1), but there is no content. - if (!loadCommands.contains('size 0x0000000000000001')) { - return true; - } - // Check the false positives: size=1 wasn't referencing the __LLVM section. - - bool emptyBitcodeMarkerFound = false; - // Section - // sectname __bundle - // segname __LLVM - // addr 0x003c4000 - // size 0x0042b633 - // offset 3932160 - // ... - final List lines = LineSplitter.split(loadCommands).toList(); - lines.asMap().forEach((int index, String line) { - if (line.contains('segname __LLVM') && lines.length - index - 1 > 3) { - final String emptyBitcodeMarker = - lines.skip(index - 1).take(3).firstWhere( - (String line) => line.contains(' size 0x0000000000000001'), - orElse: () => null, - ); - if (emptyBitcodeMarker != null) { - emptyBitcodeMarkerFound = true; - return; - } - } - }); - return !emptyBitcodeMarkerFound; -} diff --git a/packages/flutter_tools/test/src/darwin_common.dart b/packages/flutter_tools/test/src/darwin_common.dart new file mode 100644 index 00000000000..e3726ab1e60 --- /dev/null +++ b/packages/flutter_tools/test/src/darwin_common.dart @@ -0,0 +1,52 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:process/process.dart'; +import 'package:flutter_tools/src/base/io.dart'; + +bool containsBitcode(String pathToBinary, ProcessManager processManager) { + // See: https://stackoverflow.com/questions/32755775/how-to-check-a-static-library-is-built-contain-bitcode + final ProcessResult result = processManager.runSync([ + 'otool', + '-l', + '-arch', + 'arm64', + pathToBinary, + ]); + final String loadCommands = result.stdout as String; + if (!loadCommands.contains('__LLVM')) { + return false; + } + // Presence of the section may mean a bitcode marker was embedded (size=1), but there is no content. + if (!loadCommands.contains('size 0x0000000000000001')) { + return true; + } + // Check the false positives: size=1 wasn't referencing the __LLVM section. + + bool emptyBitcodeMarkerFound = false; + // Section + // sectname __bundle + // segname __LLVM + // addr 0x003c4000 + // size 0x0042b633 + // offset 3932160 + // ... + final List lines = LineSplitter.split(loadCommands).toList(); + lines.asMap().forEach((int index, String line) { + if (line.contains('segname __LLVM') && lines.length - index - 1 > 3) { + final String emptyBitcodeMarker = + lines.skip(index - 1).take(3).firstWhere( + (String line) => line.contains(' size 0x0000000000000001'), + orElse: () => null, + ); + if (emptyBitcodeMarker != null) { + emptyBitcodeMarkerFound = true; + return; + } + } + }); + return !emptyBitcodeMarkerFound; +}