diff --git a/packages/e2e/CHANGELOG.md b/packages/e2e/CHANGELOG.md index e48b95d099a7..2ccf9573a17d 100644 --- a/packages/e2e/CHANGELOG.md +++ b/packages/e2e/CHANGELOG.md @@ -1,3 +1,7 @@ +## 0.7.0 + +* Move utilities for tracking frame performance in an e2e test to `flutter_test`. + ## 0.6.3 * Add customizable `flutter_driver` adaptor. diff --git a/packages/e2e/lib/e2e_perf.dart b/packages/e2e/lib/e2e_perf.dart deleted file mode 100644 index 1b2ddd74c336..000000000000 --- a/packages/e2e/lib/e2e_perf.dart +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:ui'; - -import 'package:flutter/scheduler.dart'; -import 'package:flutter_test/flutter_test.dart'; -import 'package:flutter/widgets.dart'; - -import 'package:e2e/e2e.dart'; - -/// The maximum amount of time considered safe to spend for a frame's build -/// phase. Anything past that is in the danger of missing the frame as 60FPS. -/// -/// Changing this doesn't re-evaluate existing summary. -Duration kBuildBudget = const Duration(milliseconds: 16); -// TODO(CareF): Automatically calculate the refresh budget (#61958) - -bool _firstRun = true; - -/// The warning message to show when a benchmark is performed with assert on. -/// TODO(CareF) remove this and update pubspect when flutter/flutter#61509 is -/// in released version. -const String kDebugWarning = ''' -┏╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┓ -┇ ⚠ THIS BENCHMARK IS BEING RUN IN DEBUG MODE ⚠ ┇ -┡╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍╍┦ -│ │ -│ Numbers obtained from a benchmark while asserts are │ -│ enabled will not accurately reflect the performance │ -│ that will be experienced by end users using release ╎ -│ builds. Benchmarks should be run using this command ╎ -│ line: "flutter run --profile test.dart" or ┊ -│ or "flutter drive --profile -t test.dart". ┊ -│ ┊ -└─────────────────────────────────────────────────╌┄┈ 🐢 -'''; - -/// watches the [FrameTiming] of `action` and report it to the e2e binding. -Future watchPerformance( - E2EWidgetsFlutterBinding binding, - Future action(), { - String reportKey = 'performance', -}) async { - assert(() { - if (_firstRun) { - debugPrint(kDebugWarning); - _firstRun = false; - } - return true; - }()); - final List frameTimings = []; - final TimingsCallback watcher = frameTimings.addAll; - binding.addTimingsCallback(watcher); - await action(); - binding.removeTimingsCallback(watcher); - final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); - binding.reportData = {reportKey: frameTimes.summary}; -} - -/// This class and summarizes a list of [FrameTiming] for the performance -/// metrics. -class FrameTimingSummarizer { - /// Summarize `data` to frame build time and frame rasterizer time statistics. - /// - /// See [TimelineSummary.summaryJson] for detail. - factory FrameTimingSummarizer(List data) { - assert(data != null); - assert(data.isNotEmpty); - final List frameBuildTime = List.unmodifiable( - data.map((FrameTiming datum) => datum.buildDuration), - ); - final List frameBuildTimeSorted = - List.from(frameBuildTime)..sort(); - final List frameRasterizerTime = List.unmodifiable( - data.map((FrameTiming datum) => datum.rasterDuration), - ); - final List frameRasterizerTimeSorted = - List.from(frameRasterizerTime)..sort(); - final Duration Function(Duration, Duration) add = - (Duration a, Duration b) => a + b; - return FrameTimingSummarizer._( - frameBuildTime: frameBuildTime, - frameRasterizerTime: frameRasterizerTime, - // This avarage calculation is microsecond precision, which is fine - // because typical values of these times are milliseconds. - averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length, - p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90), - p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), - worstFrameBuildTime: frameBuildTimeSorted.last, - missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), - averageFrameRasterizerTime: - frameRasterizerTime.reduce(add) ~/ data.length, - p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), - p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.99), - worstFrameRasterizerTime: frameRasterizerTimeSorted.last, - missedFrameRasterizerBudget: - _countExceed(frameRasterizerTimeSorted, kBuildBudget), - ); - } - - const FrameTimingSummarizer._({ - @required this.frameBuildTime, - @required this.frameRasterizerTime, - @required this.averageFrameBuildTime, - @required this.p90FrameBuildTime, - @required this.p99FrameBuildTime, - @required this.worstFrameBuildTime, - @required this.missedFrameBuildBudget, - @required this.averageFrameRasterizerTime, - @required this.p90FrameRasterizerTime, - @required this.p99FrameRasterizerTime, - @required this.worstFrameRasterizerTime, - @required this.missedFrameRasterizerBudget, - }); - - /// List of frame build time in microseconds - final List frameBuildTime; - - /// List of frame rasterizer time in microseconds - final List frameRasterizerTime; - - /// The average value of [frameBuildTime] in milliseconds. - final Duration averageFrameBuildTime; - - /// The 90-th percentile value of [frameBuildTime] in milliseconds - final Duration p90FrameBuildTime; - - /// The 99-th percentile value of [frameBuildTime] in milliseconds - final Duration p99FrameBuildTime; - - /// The largest value of [frameBuildTime] in milliseconds - final Duration worstFrameBuildTime; - - /// Number of items in [frameBuildTime] that's greater than [kBuildBudget] - final int missedFrameBuildBudget; - - /// The average value of [frameRasterizerTime] in milliseconds. - final Duration averageFrameRasterizerTime; - - /// The 90-th percentile value of [frameRasterizerTime] in milliseconds. - final Duration p90FrameRasterizerTime; - - /// The 99-th percentile value of [frameRasterizerTime] in milliseconds. - final Duration p99FrameRasterizerTime; - - /// The largest value of [frameRasterizerTime] in milliseconds. - final Duration worstFrameRasterizerTime; - - /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget] - final int missedFrameRasterizerBudget; - - /// Convert the summary result to a json object. - /// - /// See [TimelineSummary.summaryJson] for detail. - Map get summary => { - 'average_frame_build_time_millis': - averageFrameBuildTime.inMicroseconds / 1E3, - '90th_percentile_frame_build_time_millis': - p90FrameBuildTime.inMicroseconds / 1E3, - '99th_percentile_frame_build_time_millis': - p99FrameBuildTime.inMicroseconds / 1E3, - 'worst_frame_build_time_millis': - worstFrameBuildTime.inMicroseconds / 1E3, - 'missed_frame_build_budget_count': missedFrameBuildBudget, - 'average_frame_rasterizer_time_millis': - averageFrameRasterizerTime.inMicroseconds / 1E3, - '90th_percentile_frame_rasterizer_time_millis': - p90FrameRasterizerTime.inMicroseconds / 1E3, - '99th_percentile_frame_rasterizer_time_millis': - p99FrameRasterizerTime.inMicroseconds / 1E3, - 'worst_frame_rasterizer_time_millis': - worstFrameRasterizerTime.inMicroseconds / 1E3, - 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, - 'frame_count': frameBuildTime.length, - 'frame_build_times': frameBuildTime - .map((Duration datum) => datum.inMicroseconds) - .toList(), - 'frame_rasterizer_times': frameRasterizerTime - .map((Duration datum) => datum.inMicroseconds) - .toList(), - }; -} - -// The following helper functions require data sorted - -// return the 100*p-th percentile of the data -T _findPercentile(List data, double p) { - assert(p >= 0 && p <= 1); - return data[((data.length - 1) * p).round()]; -} - -// return the number of items in data that > threshold -int _countExceed>(List data, T threshold) { - return data.length - - data.indexWhere((T datum) => datum.compareTo(threshold) > 0); -} diff --git a/packages/e2e/pubspec.yaml b/packages/e2e/pubspec.yaml index e4e476d146d3..221dd3a8b047 100644 --- a/packages/e2e/pubspec.yaml +++ b/packages/e2e/pubspec.yaml @@ -1,6 +1,6 @@ name: e2e description: Runs tests that use the flutter_test API as integration tests. -version: 0.6.3 +version: 0.7.0 homepage: https://github.com/flutter/plugins/tree/master/packages/e2e environment: diff --git a/packages/e2e/test/frame_timing_summarizer_test.dart b/packages/e2e/test/frame_timing_summarizer_test.dart deleted file mode 100644 index f3a12850122d..000000000000 --- a/packages/e2e/test/frame_timing_summarizer_test.dart +++ /dev/null @@ -1,35 +0,0 @@ -import 'dart:ui'; - -import 'package:flutter_test/flutter_test.dart'; - -import 'package:e2e/e2e_perf.dart'; - -void main() { - test('Test FrameTimingSummarizer', () { - List buildTimes = [ - for (int i = 1; i <= 100; i += 1) 1000 * i, - ]; - buildTimes = buildTimes.reversed.toList(); - List rasterTimes = [ - for (int i = 1; i <= 100; i += 1) 1000 * i + 1000, - ]; - rasterTimes = rasterTimes.reversed.toList(); - List inputData = [ - for (int i = 0; i < 100; i += 1) - FrameTiming([0, buildTimes[i], 500, rasterTimes[i]]), - ]; - FrameTimingSummarizer summary = FrameTimingSummarizer(inputData); - expect(summary.averageFrameBuildTime.inMicroseconds, 50500); - expect(summary.p90FrameBuildTime.inMicroseconds, 90000); - expect(summary.p99FrameBuildTime.inMicroseconds, 99000); - expect(summary.worstFrameBuildTime.inMicroseconds, 100000); - expect(summary.missedFrameBuildBudget, 84); - - expect(summary.averageFrameRasterizerTime.inMicroseconds, 51000); - expect(summary.p90FrameRasterizerTime.inMicroseconds, 90500); - expect(summary.p99FrameRasterizerTime.inMicroseconds, 99500); - expect(summary.worstFrameRasterizerTime.inMicroseconds, 100500); - expect(summary.missedFrameRasterizerBudget, 85); - expect(summary.frameBuildTime.length, 100); - }); -}