github/flutter
7
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2025-06-03 00:51:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Add SkiaPerfPoint and FlutterEngineMetricPoint (#70153)" (#70407)

Browse Source 
This reverts commit a0ec4d67e9.
This commit is contained in:
Jonah Williams 2020-11-12 14:16:25 -08:00 committed by GitHub
parent cca1215939
commit 2a5aa29442
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 2 additions and 553 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
dev/benchmarks/metrics_center/lib/src/flutter.dart
View File

@ -1,22 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:metrics_center/src/common.dart';
/// Convenient class to capture the benchmarks in the Flutter engine repo.
class FlutterEngineMetricPoint extends MetricPoint {
FlutterEngineMetricPoint(
String name,
double value,
String gitRevision, {
Map<String, String> moreTags = const <String, String>{},
}) : super(
value,
<String, String>{
kNameKey: name,
kGithubRepoKey: kFlutterEngineRepo,
kGitRevisionKey: gitRevision,
}..addAll(moreTags),
);
}

39
dev/benchmarks/metrics_center/lib/src/github_helper.dart
View File

@ -1,39 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This file uses Dart 2.12 semantics. This is needed as we can't upgrade
// the SDK constraint to `>=2.12.0-0` before the deps are ready.
// @dart=2.12
import 'package:github/github.dart';
/// Singleton class to query some Github info with an in-memory cache.
class GithubHelper {
/// Return the singleton helper.
factory GithubHelper() {
return _singleton;
}
GithubHelper._internal();
/// The result is cached in memory so querying the same thing again in the
/// same process is fast.
///
/// Our unit test requires that calling this method 1000 times for the same
/// `githubRepo` and `sha` should be done in 1 second.
Future<DateTime> getCommitDateTime(String githubRepo, String sha) async {
final String key = '$githubRepo/commit/$sha';
if (_commitDateTimeCache[key] == null) {
final RepositoryCommit commit = await _github.repositories
.getCommit(RepositorySlug.full(githubRepo), sha);
_commitDateTimeCache[key] = commit.commit.committer.date;
}
return _commitDateTimeCache[key]!;
}
static final GithubHelper _singleton = GithubHelper._internal();
final GitHub _github = GitHub();
final Map<String, DateTime> _commitDateTimeCache = <String, DateTime>{};
}

186
dev/benchmarks/metrics_center/lib/src/skiaperf.dart
View File

@ -1,186 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This file uses Dart 2.12 semantics. This is needed as we can't upgrade
// the SDK constraint to `>=2.12.0-0` before the deps are ready.
// @dart=2.12
import 'package:metrics_center/src/common.dart';
// Skia Perf Format is a JSON file that looks like:
// {
// "gitHash": "fe4a4029a080bc955e9588d05a6cd9eb490845d4",
// "key": {
// "arch": "x86",
// "gpu": "GTX660",
// "model": "ShuttleA",
// "os": "Ubuntu12"
// },
// "results": {
// "ChunkAlloc_PushPop_640_480": {
// "nonrendering": {
// "min_ms": 0.01485466666666667,
// "options": {
// "source_type": "bench"
// }
// }
// },
// "DeferredSurfaceCopy_discardable_640_480": {
// "565": {
// "min_ms": 2.215988,
// "options": {
// "source_type": "bench"
// }
// },
// ...
class SkiaPerfPoint extends MetricPoint {
SkiaPerfPoint._(this.githubRepo, this.gitHash, this.testName, this.subResult,
double value, this._options, this.jsonUrl)
: assert(_options[kGithubRepoKey] == null),
assert(_options[kGitRevisionKey] == null),
assert(_options[kNameKey] == null),
super(
value,
<String, String>{}
..addAll(_options)
..addAll(<String, String>{
kGithubRepoKey: githubRepo,
kGitRevisionKey: gitHash,
kNameKey: testName,
kSubResultKey: subResult,
}),
) {
assert(tags[kGithubRepoKey] != null);
assert(tags[kGitRevisionKey] != null);
assert(tags[kNameKey] != null);
}
/// Construct [SkiaPerfPoint] from a well-formed [MetricPoint].
///
/// The [MetricPoint] must have [kGithubRepoKey], [kGitRevisionKey],
/// [kNameKey] in its tags for this to be successful.
///
/// If the [MetricPoint] has a tag 'date', that tag will be removed so Skia
/// perf can plot multiple metrics with different date as a single trace.
/// Skia perf will use the git revision's date instead of this date tag in
/// the time axis.
factory SkiaPerfPoint.fromPoint(MetricPoint p) {
final String githubRepo = p.tags[kGithubRepoKey]!;
final String gitHash = p.tags[kGitRevisionKey]!;
final String name = p.tags[kNameKey]!;
final String subResult = p.tags[kSubResultKey] ?? kSkiaPerfValueKey;
final Map<String, String> options = <String, String>{}..addEntries(
p.tags.entries.where(
(MapEntry<String, dynamic> entry) =>
entry.key != kGithubRepoKey &&
entry.key != kGitRevisionKey &&
entry.key != kNameKey &&
entry.key != kSubResultKey &&
// https://github.com/google/benchmark automatically generates a
// 'date' field. If it's included in options, the Skia perf won't
// be able to connect different points in a single trace because
// the date is always different.
entry.key != 'date',
),
);
return SkiaPerfPoint._(
githubRepo, gitHash, name, subResult, p.value, options, null);
}
/// In the format of '<owner>/<name>' such as 'flutter/flutter' or
/// 'flutter/engine'.
final String githubRepo;
/// SHA such as 'ad20d368ffa09559754e4b2b5c12951341ca3b2d'
final String gitHash;
/// For Flutter devicelab, this is the task name (e.g.,
/// 'flutter_gallery__transition_perf'); for Google benchmark, this is the
/// benchmark name (e.g., 'BM_ShellShutdown').
///
/// In Skia perf web dashboard, this value can be queried and filtered by
/// "test".
final String testName;
/// The name of "subResult" comes from the special treatment of "sub_result"
/// in SkiaPerf. If not provided, its value will be set to kSkiaPerfValueKey.
///
/// When Google benchmarks are converted to SkiaPerfPoint, this subResult
/// could be "cpu_time" or "real_time".
///
/// When devicelab benchmarks are converted to SkiaPerfPoint, this subResult
/// is often the metric name such as "average_frame_build_time_millis" whereas
/// the [testName] is the benchmark or task name such as
/// "flutter_gallery__transition_perf".
final String subResult;
/// The url to the Skia perf json file in the Google Cloud Storage bucket.
///
/// This can be null if the point has been stored in the bucket yet.
final String? jsonUrl;
Map<String, dynamic> _toSubResultJson() {
return <String, dynamic>{
subResult: value,
kSkiaPerfOptionsKey: _options,
};
}
/// Convert a list of SkiaPoints with the same git repo and git revision into
/// a single json file in the Skia perf format.
///
/// The list must be non-empty.
static Map<String, dynamic> toSkiaPerfJson(List<SkiaPerfPoint> points) {
assert(points.isNotEmpty);
assert(() {
for (final SkiaPerfPoint p in points) {
if (p.githubRepo != points[0].githubRepo ||
p.gitHash != points[0].gitHash) {
return false;
}
}
return true;
}(), 'All points must have same githubRepo and gitHash');
final Map<String, dynamic> results = <String, dynamic>{};
for (final SkiaPerfPoint p in points) {
final Map<String, dynamic> subResultJson = p._toSubResultJson();
if (results[p.testName] == null) {
results[p.testName] = <String, dynamic>{
kSkiaPerfDefaultConfig: subResultJson,
};
} else {
// Flutter currently doesn't support having the same name but different
// options/configurations. If this actually happens in the future, we
// probably can use different values of config (currently there's only
// one kSkiaPerfDefaultConfig) to resolve the conflict.
assert(results[p.testName][kSkiaPerfDefaultConfig][kSkiaPerfOptionsKey]
.toString() ==
subResultJson[kSkiaPerfOptionsKey].toString());
assert(
results[p.testName][kSkiaPerfDefaultConfig][p.subResult] == null);
results[p.testName][kSkiaPerfDefaultConfig][p.subResult] = p.value;
}
}
return <String, dynamic>{
kSkiaPerfGitHashKey: points[0].gitHash,
kSkiaPerfResultsKey: results,
};
}
// Equivalent to tags without git repo, git hash, and name because those two
// are already stored somewhere else.
final Map<String, String> _options;
}
const String kSkiaPerfGitHashKey = 'gitHash';
const String kSkiaPerfResultsKey = 'results';
const String kSkiaPerfValueKey = 'value';
const String kSkiaPerfOptionsKey = 'options';
const String kSkiaPerfDefaultConfig = 'default';

41
dev/benchmarks/metrics_center/test/github_helper_test.dart
View File

@ -1,41 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@Timeout(Duration(seconds: 3600))
import 'package:metrics_center/src/github_helper.dart';
import 'common.dart';
void main() {
test('GithubHelper gets correct commit date time', () async {
final GithubHelper helper = GithubHelper();
expect(
await helper.getCommitDateTime(
'flutter/flutter',
'ad20d368ffa09559754e4b2b5c12951341ca3b2d',
),
equals(DateTime.parse('2019-12-06 03:33:01.000Z')),
);
});
test('GithubHelper is a singleton', () {
final GithubHelper helper1 = GithubHelper();
final GithubHelper helper2 = GithubHelper();
expect(helper1, equals(helper2));
});
test('GithubHelper can query the same commit 1000 times within 1 second',
() async {
final DateTime start = DateTime.now();
for (int i = 0; i < 1000; i += 1) {
await GithubHelper().getCommitDateTime(
'flutter/flutter',
'ad20d368ffa09559754e4b2b5c12951341ca3b2d',
);
}
final Duration duration = DateTime.now().difference(start);
expect(duration, lessThan(const Duration(seconds: 1)));
});
}

2
dev/benchmarks/metrics_center/test/google_benchmark_test.dart
View File

@ -2,6 +2,8 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// @dart = 2.8
import 'package:metrics_center/src/common.dart';
import 'package:metrics_center/google_benchmark.dart';

265
dev/benchmarks/metrics_center/test/skiaperf_test.dart
View File

@ -1,265 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@Timeout(Duration(seconds: 3600))
import 'dart:convert';
import 'package:metrics_center/src/common.dart';
import 'package:metrics_center/src/flutter.dart';
import 'package:metrics_center/src/skiaperf.dart';
import 'common.dart';
void main() {
const double kValue1 = 1.0;
const double kValue2 = 2.0;
const String kFrameworkRevision1 = '9011cece2595447eea5dd91adaa241c1c9ef9a33';
const String kTaskName = 'analyzer_benchmark';
const String kMetric1 = 'flutter_repo_batch_maximum';
const String kMetric2 = 'flutter_repo_watch_maximum';
final MetricPoint cocoonPointRev1Metric1 = MetricPoint(
kValue1,
const <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: kFrameworkRevision1,
kNameKey: kTaskName,
kSubResultKey: kMetric1,
kUnitKey: 's',
},
);
final MetricPoint cocoonPointRev1Metric2 = MetricPoint(
kValue2,
const <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: kFrameworkRevision1,
kNameKey: kTaskName,
kSubResultKey: kMetric2,
kUnitKey: 's',
},
);
final MetricPoint cocoonPointBetaRev1Metric1 = MetricPoint(
kValue1,
const <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: kFrameworkRevision1,
kNameKey: 'beta/$kTaskName',
kSubResultKey: kMetric1,
kUnitKey: 's',
'branch': 'beta',
},
);
final MetricPoint cocoonPointBetaRev1Metric1BadBranch = MetricPoint(
kValue1,
const <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: kFrameworkRevision1,
kNameKey: kTaskName,
kSubResultKey: kMetric1,
kUnitKey: 's',
// If we only add this 'branch' tag without changing the test or sub-result name, an exception
// would be thrown as Skia Perf currently only supports the same set of tags for a pair of
// kNameKey and kSubResultKey values. So to support branches, one also has to add the branch
// name to the test name.
'branch': 'beta',
},
);
const String engineMetricName = 'BM_PaintRecordInit';
const String engineRevision = 'ca799fa8b2254d09664b78ee80c43b434788d112';
const double engineValue1 = 101;
const double engineValue2 = 102;
final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint(
engineMetricName,
engineValue1,
engineRevision,
moreTags: const <String, String>{
kSubResultKey: 'cpu_time',
kUnitKey: 'ns',
'date': '2019-12-17 15:14:14',
'num_cpus': '56',
'mhz_per_cpu': '2594',
'cpu_scaling_enabled': 'true',
'library_build_type': 'release',
},
);
final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint(
engineMetricName,
engineValue2,
engineRevision,
moreTags: const <String, String>{
kSubResultKey: 'real_time',
kUnitKey: 'ns',
'date': '2019-12-17 15:14:14',
'num_cpus': '56',
'mhz_per_cpu': '2594',
'cpu_scaling_enabled': 'true',
'library_build_type': 'release',
},
);
test('Throw if invalid points are converted to SkiaPoint', () {
final MetricPoint noGithubRepoPoint = MetricPoint(
kValue1,
const <String, String>{
kGitRevisionKey: kFrameworkRevision1,
kNameKey: kTaskName,
},
);
final MetricPoint noGitRevisionPoint = MetricPoint(
kValue1,
const <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kNameKey: kTaskName,
},
);
final MetricPoint noTestNamePoint = MetricPoint(
kValue1,
const <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: kFrameworkRevision1,
},
);
expect(() => SkiaPerfPoint.fromPoint(noGithubRepoPoint), throwsA(anything));
expect(
() => SkiaPerfPoint.fromPoint(noGitRevisionPoint), throwsA(anything));
expect(() => SkiaPerfPoint.fromPoint(noTestNamePoint), throwsA(anything));
});
test('Correctly convert a metric point from cocoon to SkiaPoint', () {
final SkiaPerfPoint skiaPoint1 =
SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
expect(skiaPoint1, isNotNull);
expect(skiaPoint1.testName, equals(kTaskName));
expect(skiaPoint1.subResult, equals(kMetric1));
expect(skiaPoint1.value, equals(cocoonPointRev1Metric1.value));
expect(skiaPoint1.jsonUrl, isNull); // Not inserted yet
});
test('Cocoon points correctly encode into Skia perf json format', () {
final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
final SkiaPerfPoint p2 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric2);
final SkiaPerfPoint p3 =
SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1);
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
expect(
encoder
.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[p1, p2, p3])),
equals('''
{
"gitHash": "9011cece2595447eea5dd91adaa241c1c9ef9a33",
"results": {
"analyzer_benchmark": {
"default": {
"flutter_repo_batch_maximum": 1.0,
"options": {
"unit": "s"
},
"flutter_repo_watch_maximum": 2.0
}
},
"beta/analyzer_benchmark": {
"default": {
"flutter_repo_batch_maximum": 1.0,
"options": {
"branch": "beta",
"unit": "s"
}
}
}
}
}'''));
});
test('Engine metric points correctly encode into Skia perf json format', () {
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
expect(
encoder.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[
SkiaPerfPoint.fromPoint(enginePoint1),
SkiaPerfPoint.fromPoint(enginePoint2),
])),
equals(
'''
{
"gitHash": "ca799fa8b2254d09664b78ee80c43b434788d112",
"results": {
"BM_PaintRecordInit": {
"default": {
"cpu_time": 101.0,
"options": {
"cpu_scaling_enabled": "true",
"library_build_type": "release",
"mhz_per_cpu": "2594",
"num_cpus": "56",
"unit": "ns"
},
"real_time": 102.0
}
}
}
}''',
),
);
});
test(
'Throw if engine points with the same test name but different options are converted to '
'Skia perf points', () {
final FlutterEngineMetricPoint enginePoint1 = FlutterEngineMetricPoint(
'BM_PaintRecordInit',
101,
'ca799fa8b2254d09664b78ee80c43b434788d112',
moreTags: const <String, String>{
kSubResultKey: 'cpu_time',
kUnitKey: 'ns',
'cpu_scaling_enabled': 'true',
},
);
final FlutterEngineMetricPoint enginePoint2 = FlutterEngineMetricPoint(
'BM_PaintRecordInit',
102,
'ca799fa8b2254d09664b78ee80c43b434788d112',
moreTags: const <String, String>{
kSubResultKey: 'real_time',
kUnitKey: 'ns',
'cpu_scaling_enabled': 'false',
},
);
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
expect(
() => encoder.convert(SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[
SkiaPerfPoint.fromPoint(enginePoint1),
SkiaPerfPoint.fromPoint(enginePoint2),
])),
throwsA(anything),
);
});
test(
'Throw if two Cocoon metric points with the same name and subResult keys '
'but different options are converted to Skia perf points', () {
final SkiaPerfPoint p1 = SkiaPerfPoint.fromPoint(cocoonPointRev1Metric1);
final SkiaPerfPoint p2 =
SkiaPerfPoint.fromPoint(cocoonPointBetaRev1Metric1BadBranch);
expect(
() => SkiaPerfPoint.toSkiaPerfJson(<SkiaPerfPoint>[p1, p2]),
throwsA(anything),
);
});
}