From 309a2d78fbcd735b7594e3f65df809f52d46b23e Mon Sep 17 00:00:00 2001 From: Chris Bracken Date: Fri, 11 Aug 2017 12:58:35 -0700 Subject: [PATCH] Extract snapshotting logic to Snapshotter class (#11591) First step in eliminating code duplication between script snapshotting (in FLX build) and AOT, assembly AOT snapshotting. --- .../flutter_tools/lib/src/base/build.dart | 142 ++++++++++++++ packages/flutter_tools/lib/src/flx.dart | 77 +------- .../flutter_tools/test/base/build_test.dart | 183 ++++++++++++++++++ 3 files changed, 328 insertions(+), 74 deletions(-) diff --git a/packages/flutter_tools/lib/src/base/build.dart b/packages/flutter_tools/lib/src/base/build.dart index bad8bce5079..f5f7f8a5c47 100644 --- a/packages/flutter_tools/lib/src/base/build.dart +++ b/packages/flutter_tools/lib/src/base/build.dart @@ -6,8 +6,44 @@ import 'dart:async'; import 'dart:convert' show JSON; import 'package:crypto/crypto.dart' show md5; +import 'package:meta/meta.dart'; +import '../artifacts.dart'; +import '../build_info.dart'; +import '../globals.dart'; +import 'context.dart'; import 'file_system.dart'; +import 'process.dart'; + +GenSnapshot get genSnapshot => context.putIfAbsent(GenSnapshot, () => const GenSnapshot()); + +class GenSnapshot { + const GenSnapshot(); + + Future run({ + @required TargetPlatform targetPlatform, + @required BuildMode buildMode, + @required String packagesPath, + @required String depfilePath, + Iterable additionalArgs: const [], + }) { + final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData); + final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData); + final List args = [ + '--assert_initializer', + '--await_is_keyword', + '--causal_async_stacks', + '--vm_snapshot_data=$vmSnapshotData', + '--isolate_snapshot_data=$isolateSnapshotData', + '--packages=$packagesPath', + '--dependencies=$depfilePath', + '--print_snapshot_sizes', + ]..addAll(additionalArgs); + + final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, targetPlatform, buildMode); + return runCommandAndStreamOutput([snapshotterPath]..addAll(args)); + } +} /// A collection of checksums for a set of input files. /// @@ -68,3 +104,109 @@ Future> readDepfile(String depfilePath) async { .where((String path) => path.isNotEmpty) .toSet(); } + +/// Dart snapshot builder. +/// +/// Builds Dart snapshots in one of three modes: +/// * Script snapshot: architecture-independent snapshot of a Dart script core +/// libraries. +/// * AOT snapshot: architecture-specific ahead-of-time compiled snapshot +/// suitable for loading with `mmap`. +/// * Assembly AOT snapshot: architecture-specific ahead-of-time compile to +/// assembly suitable for compilation as a static or dynamic library. +class Snapshotter { + /// Builds an architecture-independent snapshot of the specified script. + Future buildScriptSnapshot({ + @required String mainPath, + @required String snapshotPath, + @required String depfilePath, + @required String packagesPath + }) async { + final List args = [ + '--snapshot_kind=script', + '--script_snapshot=$snapshotPath', + mainPath, + ]; + + final String checksumsPath = '$depfilePath.checksums'; + final int exitCode = await _build( + outputSnapshotPath: snapshotPath, + packagesPath: packagesPath, + snapshotArgs: args, + depfilePath: depfilePath, + mainPath: mainPath, + checksumsPath: checksumsPath, + ); + if (exitCode != 0) + return exitCode; + await _writeChecksum(snapshotPath, depfilePath, mainPath, checksumsPath); + return exitCode; + } + + /// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script. + Future buildAotSnapshot() async { + throw new UnimplementedError('AOT snapshotting not yet implemented'); + } + + Future _build({ + @required List snapshotArgs, + @required String outputSnapshotPath, + @required String packagesPath, + @required String depfilePath, + @required String mainPath, + @required String checksumsPath, + }) async { + if (!await _isBuildRequired(outputSnapshotPath, depfilePath, mainPath, checksumsPath)) { + printTrace('Skipping snapshot build. Checksums match.'); + return 0; + } + + // Build the snapshot. + final int exitCode = await genSnapshot.run( + targetPlatform: null, + buildMode: BuildMode.debug, + packagesPath: packagesPath, + depfilePath: depfilePath, + additionalArgs: snapshotArgs, + ); + if (exitCode != 0) + return exitCode; + + _writeChecksum(outputSnapshotPath, depfilePath, mainPath, checksumsPath); + return 0; + } + + Future _isBuildRequired(String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async { + final File checksumFile = fs.file(checksumsPath); + final File outputSnapshotFile = fs.file(outputSnapshotPath); + final File depfile = fs.file(depfilePath); + if (!outputSnapshotFile.existsSync() || !depfile.existsSync() || !checksumFile.existsSync()) + return true; + + try { + if (checksumFile.existsSync()) { + final Checksum oldChecksum = new Checksum.fromJson(await checksumFile.readAsString()); + final Set checksumPaths = await readDepfile(depfilePath) + ..addAll([outputSnapshotPath, mainPath]); + final Checksum newChecksum = new Checksum.fromFiles(checksumPaths); + return oldChecksum != newChecksum; + } + } catch (e, s) { + // Log exception and continue, this step is a performance improvement only. + printTrace('Error during snapshot checksum output: $e\n$s'); + } + return true; + } + + Future _writeChecksum(String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async { + try { + final Set checksumPaths = await readDepfile(depfilePath) + ..addAll([outputSnapshotPath, mainPath]); + final Checksum checksum = new Checksum.fromFiles(checksumPaths); + await fs.file(checksumsPath).writeAsString(checksum.toJson()); + } catch (e, s) { + // Log exception and continue, this step is a performance improvement only. + print('Error during snapshot checksum output: $e\n$s'); + } + } +} diff --git a/packages/flutter_tools/lib/src/flx.dart b/packages/flutter_tools/lib/src/flx.dart index d90d4edbb75..2d9abf22295 100644 --- a/packages/flutter_tools/lib/src/flx.dart +++ b/packages/flutter_tools/lib/src/flx.dart @@ -4,14 +4,10 @@ import 'dart:async'; -import 'package:meta/meta.dart' show required; - -import 'artifacts.dart'; import 'asset.dart'; import 'base/build.dart'; import 'base/common.dart'; import 'base/file_system.dart'; -import 'base/process.dart'; import 'build_info.dart'; import 'dart/package_map.dart'; import 'devfs.dart'; @@ -30,74 +26,6 @@ const String _kKernelKey = 'kernel_blob.bin'; const String _kSnapshotKey = 'snapshot_blob.bin'; const String _kDylibKey = 'libapp.so'; -Future _createSnapshot({ - @required String mainPath, - @required String snapshotPath, - @required String depfilePath, - @required String packages -}) async { - assert(mainPath != null); - assert(snapshotPath != null); - assert(depfilePath != null); - assert(packages != null); - final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, null, BuildMode.debug); - final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData); - final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData); - - final List args = [ - snapshotterPath, - '--snapshot_kind=script', - '--vm_snapshot_data=$vmSnapshotData', - '--isolate_snapshot_data=$isolateSnapshotData', - '--packages=$packages', - '--script_snapshot=$snapshotPath', - '--dependencies=$depfilePath', - mainPath, - ]; - - // Write the depfile path to disk. - fs.file(depfilePath).parent.childFile('gen_snapshot.d').writeAsString('$depfilePath: $snapshotterPath\n'); - - final File checksumFile = fs.file('$depfilePath.checksums'); - final File snapshotFile = fs.file(snapshotPath); - final File depfile = fs.file(depfilePath); - if (snapshotFile.existsSync() && depfile.existsSync() && checksumFile.existsSync()) { - try { - final String json = await checksumFile.readAsString(); - final Checksum oldChecksum = new Checksum.fromJson(json); - final Set inputPaths = await readDepfile(depfilePath); - inputPaths.add(snapshotPath); - inputPaths.add(mainPath); - final Checksum newChecksum = new Checksum.fromFiles(inputPaths); - if (oldChecksum == newChecksum) { - printTrace('Skipping snapshot build. Checksums match.'); - return 0; - } - } catch (e, s) { - // Log exception and continue, this step is a performance improvement only. - printTrace('Error during snapshot checksum check: $e\n$s'); - } - } - - // Build the snapshot. - final int exitCode = await runCommandAndStreamOutput(args); - if (exitCode != 0) - return exitCode; - - // Compute and record input file checksums. - try { - final Set inputPaths = await readDepfile(depfilePath); - inputPaths.add(snapshotPath); - inputPaths.add(mainPath); - final Checksum checksum = new Checksum.fromFiles(inputPaths); - await checksumFile.writeAsString(checksum.toJson()); - } catch (e, s) { - // Log exception and continue, this step is a performance improvement only. - printTrace('Error during snapshot checksum output: $e\n$s'); - } - return 0; -} - Future build({ String mainPath: defaultMainPath, String manifestPath: defaultManifestPath, @@ -123,11 +51,12 @@ Future build({ // In a precompiled snapshot, the instruction buffer contains script // content equivalents - final int result = await _createSnapshot( + final Snapshotter snapshotter = new Snapshotter(); + final int result = await snapshotter.buildScriptSnapshot( mainPath: mainPath, snapshotPath: snapshotPath, depfilePath: depfilePath, - packages: packagesPath + packagesPath: packagesPath, ); if (result != 0) throwToolExit('Failed to run the Flutter compiler. Exit code: $result', exitCode: result); diff --git a/packages/flutter_tools/test/base/build_test.dart b/packages/flutter_tools/test/base/build_test.dart index 3971c213bef..c80a8697d8c 100644 --- a/packages/flutter_tools/test/base/build_test.dart +++ b/packages/flutter_tools/test/base/build_test.dart @@ -2,13 +2,54 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:async'; +import 'dart:convert' show JSON; + import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/build.dart'; +import 'package:flutter_tools/src/base/context.dart'; import 'package:flutter_tools/src/base/file_system.dart'; +import 'package:flutter_tools/src/build_info.dart'; import 'package:test/test.dart'; import '../src/context.dart'; +class _FakeGenSnapshot implements GenSnapshot { + _FakeGenSnapshot({ + this.succeed: true, + this.snapshotPath: 'output.snapshot', + this.snapshotContent: '', + this.depfilePath: 'output.snapshot.d', + this.depfileContent: 'output.snapshot.d : main.dart', + }); + + final bool succeed; + final String snapshotPath; + final String snapshotContent; + final String depfilePath; + final String depfileContent; + int _callCount = 0; + + int get callCount => _callCount; + + @override + Future run({ + TargetPlatform targetPlatform, + BuildMode buildMode, + String packagesPath, + String depfilePath, + Iterable additionalArgs, + }) async { + _callCount += 1; + + if (!succeed) + return 1; + await fs.file(snapshotPath).writeAsString(snapshotContent); + await fs.file(depfilePath).writeAsString(depfileContent); + return 0; + } +} + void main() { group('Checksum', () { group('fromFiles', () { @@ -103,4 +144,146 @@ void main() { ])); }, overrides: { FileSystem: () => fs }); }); + + group('Snapshotter', () { + _FakeGenSnapshot genSnapshot; + MemoryFileSystem fs; + Snapshotter snapshotter; + + setUp(() { + fs = new MemoryFileSystem(); + genSnapshot = new _FakeGenSnapshot(); + snapshotter = new Snapshotter(); + }); + + testUsingContext('builds snapshot and checksums when no checksums are present', () async { + await fs.file('main.dart').writeAsString('void main() {}'); + await fs.file('output.snapshot').create(); + await fs.file('output.snapshot.d').writeAsString('snapshot : main.dart'); + await snapshotter.buildScriptSnapshot( + mainPath: 'main.dart', + snapshotPath: 'output.snapshot', + depfilePath: 'output.snapshot.d', + packagesPath: '.packages', + ); + + expect(genSnapshot.callCount, 1); + + final Map json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString()); + expect(json, hasLength(2)); + expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e'); + expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e'); + }, overrides: { + FileSystem: () => fs, + GenSnapshot: () => genSnapshot, + }); + + testUsingContext('builds snapshot and checksums when checksums differ', () async { + await fs.file('main.dart').writeAsString('void main() {}'); + await fs.file('output.snapshot').create(); + await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart'); + await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode({ + 'main.dart': '27f5ebf0f8c559b2af9419d190299a5e', + 'output.snapshot': 'deadbeef01234567890abcdef0123456', + })); + await snapshotter.buildScriptSnapshot( + mainPath: 'main.dart', + snapshotPath: 'output.snapshot', + depfilePath: 'output.snapshot.d', + packagesPath: '.packages', + ); + + expect(genSnapshot.callCount, 1); + + final Map json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString()); + expect(json, hasLength(2)); + expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e'); + expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e'); + }, overrides: { + FileSystem: () => fs, + GenSnapshot: () => genSnapshot, + }); + + testUsingContext('builds snapshot and checksums when checksums match but previous snapshot not present', () async { + await fs.file('main.dart').writeAsString('void main() {}'); + await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart'); + await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode({ + 'main.dart': '27f5ebf0f8c559b2af9419d190299a5e', + 'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e', + })); + await snapshotter.buildScriptSnapshot( + mainPath: 'main.dart', + snapshotPath: 'output.snapshot', + depfilePath: 'output.snapshot.d', + packagesPath: '.packages', + ); + + expect(genSnapshot.callCount, 1); + + final Map json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString()); + expect(json, hasLength(2)); + expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e'); + expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e'); + }, overrides: { + FileSystem: () => fs, + GenSnapshot: () => genSnapshot, + }); + + testUsingContext('builds snapshot and checksums when main entry point changes', () async { + final _FakeGenSnapshot genSnapshot = new _FakeGenSnapshot( + snapshotPath: 'output.snapshot', + depfilePath: 'output.snapshot.d', + depfileContent: 'output.snapshot : other.dart', + ); + context.setVariable(GenSnapshot, genSnapshot); + + await fs.file('main.dart').writeAsString('void main() {}'); + await fs.file('other.dart').writeAsString('void main() { print("Kanpai ima kimi wa jinsei no ookina ookina butai ni tachi"); }'); + await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart'); + await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode({ + 'main.dart': '27f5ebf0f8c559b2af9419d190299a5e', + 'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e', + })); + await snapshotter.buildScriptSnapshot( + mainPath: 'other.dart', + snapshotPath: 'output.snapshot', + depfilePath: 'output.snapshot.d', + packagesPath: '.packages', + ); + + expect(genSnapshot.callCount, 1); + final Map json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString()); + expect(json, hasLength(2)); + expect(json['other.dart'], '3238d0ae341339b1731d3c2e195ad177'); + expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e'); + }, overrides: { + FileSystem: () => fs, + }); + + testUsingContext('skips snapshot when checksums match and previous snapshot is present', () async { + await fs.file('main.dart').writeAsString('void main() {}'); + await fs.file('output.snapshot').create(); + await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart'); + await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode({ + 'main.dart': '27f5ebf0f8c559b2af9419d190299a5e', + 'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e', + })); + await snapshotter.buildScriptSnapshot( + mainPath: 'main.dart', + snapshotPath: 'output.snapshot', + depfilePath: 'output.snapshot.d', + packagesPath: '.packages', + ); + + expect(genSnapshot.callCount, 0); + + final Map json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString()); + expect(json, hasLength(2)); + expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e'); + expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e'); + }, overrides: { + FileSystem: () => fs, + GenSnapshot: () => genSnapshot, + }); + }); }