github/flutter
7
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2025-06-03 00:51:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Extract snapshotting logic to Snapshotter class (#11591)

Browse Source 
First step in eliminating code duplication between script snapshotting
(in FLX build) and AOT, assembly AOT snapshotting.
This commit is contained in:
Chris Bracken 2017-08-11 12:58:35 -07:00 committed by GitHub
parent f1f5d4f3a8
commit 309a2d78fb
3 changed files with 328 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

142
packages/flutter_tools/lib/src/base/build.dart
View File

@ -6,8 +6,44 @@ import 'dart:async';
import 'dart:convert' show JSON;
import 'package:crypto/crypto.dart' show md5;
import 'package:meta/meta.dart';
import '../artifacts.dart';
import '../build_info.dart';
import '../globals.dart';
import 'context.dart';
import 'file_system.dart';
import 'process.dart';
GenSnapshot get genSnapshot => context.putIfAbsent(GenSnapshot, () => const GenSnapshot());
class GenSnapshot {
const GenSnapshot();
Future<int> run({
@required TargetPlatform targetPlatform,
@required BuildMode buildMode,
@required String packagesPath,
@required String depfilePath,
Iterable<String> additionalArgs: const <String>[],
}) {
final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
final List<String> args = <String>[
'--assert_initializer',
'--await_is_keyword',
'--causal_async_stacks',
'--vm_snapshot_data=$vmSnapshotData',
'--isolate_snapshot_data=$isolateSnapshotData',
'--packages=$packagesPath',
'--dependencies=$depfilePath',
'--print_snapshot_sizes',
]..addAll(additionalArgs);
final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, targetPlatform, buildMode);
return runCommandAndStreamOutput(<String>[snapshotterPath]..addAll(args));
}
}
/// A collection of checksums for a set of input files.
///
@ -68,3 +104,109 @@ Future<Set<String>> readDepfile(String depfilePath) async {
.where((String path) => path.isNotEmpty)
.toSet();
}
/// Dart snapshot builder.
///
/// Builds Dart snapshots in one of three modes:
/// * Script snapshot: architecture-independent snapshot of a Dart script core
/// libraries.
/// * AOT snapshot: architecture-specific ahead-of-time compiled snapshot
/// suitable for loading with `mmap`.
/// * Assembly AOT snapshot: architecture-specific ahead-of-time compile to
/// assembly suitable for compilation as a static or dynamic library.
class Snapshotter {
/// Builds an architecture-independent snapshot of the specified script.
Future<int> buildScriptSnapshot({
@required String mainPath,
@required String snapshotPath,
@required String depfilePath,
@required String packagesPath
}) async {
final List<String> args = <String>[
'--snapshot_kind=script',
'--script_snapshot=$snapshotPath',
mainPath,
];
final String checksumsPath = '$depfilePath.checksums';
final int exitCode = await _build(
outputSnapshotPath: snapshotPath,
packagesPath: packagesPath,
snapshotArgs: args,
depfilePath: depfilePath,
mainPath: mainPath,
checksumsPath: checksumsPath,
);
if (exitCode != 0)
return exitCode;
await _writeChecksum(snapshotPath, depfilePath, mainPath, checksumsPath);
return exitCode;
}
/// Builds an architecture-specific ahead-of-time compiled snapshot of the specified script.
Future<Null> buildAotSnapshot() async {
throw new UnimplementedError('AOT snapshotting not yet implemented');
}
Future<int> _build({
@required List<String> snapshotArgs,
@required String outputSnapshotPath,
@required String packagesPath,
@required String depfilePath,
@required String mainPath,
@required String checksumsPath,
}) async {
if (!await _isBuildRequired(outputSnapshotPath, depfilePath, mainPath, checksumsPath)) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
}
// Build the snapshot.
final int exitCode = await genSnapshot.run(
targetPlatform: null,
buildMode: BuildMode.debug,
packagesPath: packagesPath,
depfilePath: depfilePath,
additionalArgs: snapshotArgs,
);
if (exitCode != 0)
return exitCode;
_writeChecksum(outputSnapshotPath, depfilePath, mainPath, checksumsPath);
return 0;
}
Future<bool> _isBuildRequired(String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
final File checksumFile = fs.file(checksumsPath);
final File outputSnapshotFile = fs.file(outputSnapshotPath);
final File depfile = fs.file(depfilePath);
if (!outputSnapshotFile.existsSync() || !depfile.existsSync() || !checksumFile.existsSync())
return true;
try {
if (checksumFile.existsSync()) {
final Checksum oldChecksum = new Checksum.fromJson(await checksumFile.readAsString());
final Set<String> checksumPaths = await readDepfile(depfilePath)
..addAll(<String>[outputSnapshotPath, mainPath]);
final Checksum newChecksum = new Checksum.fromFiles(checksumPaths);
return oldChecksum != newChecksum;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum output: $e\n$s');
}
return true;
}
Future<Null> _writeChecksum(String outputSnapshotPath, String depfilePath, String mainPath, String checksumsPath) async {
try {
final Set<String> checksumPaths = await readDepfile(depfilePath)
..addAll(<String>[outputSnapshotPath, mainPath]);
final Checksum checksum = new Checksum.fromFiles(checksumPaths);
await fs.file(checksumsPath).writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
print('Error during snapshot checksum output: $e\n$s');
}
}
}

77
packages/flutter_tools/lib/src/flx.dart
View File

@ -4,14 +4,10 @@
import 'dart:async';
import 'package:meta/meta.dart' show required;
import 'artifacts.dart';
import 'asset.dart';
import 'base/build.dart';
import 'base/common.dart';
import 'base/file_system.dart';
import 'base/process.dart';
import 'build_info.dart';
import 'dart/package_map.dart';
import 'devfs.dart';
@ -30,74 +26,6 @@ const String _kKernelKey = 'kernel_blob.bin';
const String _kSnapshotKey = 'snapshot_blob.bin';
const String _kDylibKey = 'libapp.so';
Future<int> _createSnapshot({
@required String mainPath,
@required String snapshotPath,
@required String depfilePath,
@required String packages
}) async {
assert(mainPath != null);
assert(snapshotPath != null);
assert(depfilePath != null);
assert(packages != null);
final String snapshotterPath = artifacts.getArtifactPath(Artifact.genSnapshot, null, BuildMode.debug);
final String vmSnapshotData = artifacts.getArtifactPath(Artifact.vmSnapshotData);
final String isolateSnapshotData = artifacts.getArtifactPath(Artifact.isolateSnapshotData);
final List<String> args = <String>[
snapshotterPath,
'--snapshot_kind=script',
'--vm_snapshot_data=$vmSnapshotData',
'--isolate_snapshot_data=$isolateSnapshotData',
'--packages=$packages',
'--script_snapshot=$snapshotPath',
'--dependencies=$depfilePath',
mainPath,
];
// Write the depfile path to disk.
fs.file(depfilePath).parent.childFile('gen_snapshot.d').writeAsString('$depfilePath: $snapshotterPath\n');
final File checksumFile = fs.file('$depfilePath.checksums');
final File snapshotFile = fs.file(snapshotPath);
final File depfile = fs.file(depfilePath);
if (snapshotFile.existsSync() && depfile.existsSync() && checksumFile.existsSync()) {
try {
final String json = await checksumFile.readAsString();
final Checksum oldChecksum = new Checksum.fromJson(json);
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum newChecksum = new Checksum.fromFiles(inputPaths);
if (oldChecksum == newChecksum) {
printTrace('Skipping snapshot build. Checksums match.');
return 0;
}
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum check: $e\n$s');
}
}
// Build the snapshot.
final int exitCode = await runCommandAndStreamOutput(args);
if (exitCode != 0)
return exitCode;
// Compute and record input file checksums.
try {
final Set<String> inputPaths = await readDepfile(depfilePath);
inputPaths.add(snapshotPath);
inputPaths.add(mainPath);
final Checksum checksum = new Checksum.fromFiles(inputPaths);
await checksumFile.writeAsString(checksum.toJson());
} catch (e, s) {
// Log exception and continue, this step is a performance improvement only.
printTrace('Error during snapshot checksum output: $e\n$s');
}
return 0;
}
Future<Null> build({
String mainPath: defaultMainPath,
String manifestPath: defaultManifestPath,
@ -123,11 +51,12 @@ Future<Null> build({
// In a precompiled snapshot, the instruction buffer contains script
// content equivalents
final int result = await _createSnapshot(
final Snapshotter snapshotter = new Snapshotter();
final int result = await snapshotter.buildScriptSnapshot(
mainPath: mainPath,
snapshotPath: snapshotPath,
depfilePath: depfilePath,
packages: packagesPath
packagesPath: packagesPath,
);
if (result != 0)
throwToolExit('Failed to run the Flutter compiler. Exit code: $result', exitCode: result);

183
packages/flutter_tools/test/base/build_test.dart
View File

@ -2,13 +2,54 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show JSON;
import 'package:file/memory.dart';
import 'package:flutter_tools/src/base/build.dart';
import 'package:flutter_tools/src/base/context.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/build_info.dart';
import 'package:test/test.dart';
import '../src/context.dart';
class _FakeGenSnapshot implements GenSnapshot {
_FakeGenSnapshot({
this.succeed: true,
this.snapshotPath: 'output.snapshot',
this.snapshotContent: '',
this.depfilePath: 'output.snapshot.d',
this.depfileContent: 'output.snapshot.d : main.dart',
});
final bool succeed;
final String snapshotPath;
final String snapshotContent;
final String depfilePath;
final String depfileContent;
int _callCount = 0;
int get callCount => _callCount;
@override
Future<int> run({
TargetPlatform targetPlatform,
BuildMode buildMode,
String packagesPath,
String depfilePath,
Iterable<String> additionalArgs,
}) async {
_callCount += 1;
if (!succeed)
return 1;
await fs.file(snapshotPath).writeAsString(snapshotContent);
await fs.file(depfilePath).writeAsString(depfileContent);
return 0;
}
}
void main() {
group('Checksum', () {
group('fromFiles', () {
@ -103,4 +144,146 @@ void main() {
]));
}, overrides: <Type, Generator>{ FileSystem: () => fs });
});
group('Snapshotter', () {
_FakeGenSnapshot genSnapshot;
MemoryFileSystem fs;
Snapshotter snapshotter;
setUp(() {
fs = new MemoryFileSystem();
genSnapshot = new _FakeGenSnapshot();
snapshotter = new Snapshotter();
});
testUsingContext('builds snapshot and checksums when no checksums are present', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot').create();
await fs.file('output.snapshot.d').writeAsString('snapshot : main.dart');
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json, hasLength(2));
expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
GenSnapshot: () => genSnapshot,
});
testUsingContext('builds snapshot and checksums when checksums differ', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot').create();
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'deadbeef01234567890abcdef0123456',
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json, hasLength(2));
expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
GenSnapshot: () => genSnapshot,
});
testUsingContext('builds snapshot and checksums when checksums match but previous snapshot not present', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json, hasLength(2));
expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
GenSnapshot: () => genSnapshot,
});
testUsingContext('builds snapshot and checksums when main entry point changes', () async {
final _FakeGenSnapshot genSnapshot = new _FakeGenSnapshot(
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
depfileContent: 'output.snapshot : other.dart',
);
context.setVariable(GenSnapshot, genSnapshot);
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('other.dart').writeAsString('void main() { print("Kanpai ima kimi wa jinsei no ookina ookina butai ni tachi"); }');
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'other.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 1);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json, hasLength(2));
expect(json['other.dart'], '3238d0ae341339b1731d3c2e195ad177');
expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
});
testUsingContext('skips snapshot when checksums match and previous snapshot is present', () async {
await fs.file('main.dart').writeAsString('void main() {}');
await fs.file('output.snapshot').create();
await fs.file('output.snapshot.d').writeAsString('output.snapshot : main.dart');
await fs.file('output.snapshot.d.checksums').writeAsString(JSON.encode(<String, dynamic>{
'main.dart': '27f5ebf0f8c559b2af9419d190299a5e',
'output.snapshot': 'd41d8cd98f00b204e9800998ecf8427e',
}));
await snapshotter.buildScriptSnapshot(
mainPath: 'main.dart',
snapshotPath: 'output.snapshot',
depfilePath: 'output.snapshot.d',
packagesPath: '.packages',
);
expect(genSnapshot.callCount, 0);
final Map<String, dynamic> json = JSON.decode(await fs.file('output.snapshot.d.checksums').readAsString());
expect(json, hasLength(2));
expect(json['main.dart'], '27f5ebf0f8c559b2af9419d190299a5e');
expect(json['output.snapshot'], 'd41d8cd98f00b204e9800998ecf8427e');
}, overrides: <Type, Generator>{
FileSystem: () => fs,
GenSnapshot: () => genSnapshot,
});
});
}