Skip to content

Commit 447b548

Browse files
stuartmorgan-gfotiDim
authored andcommitted
Ensure that integration tests are actually being run (flutter#3857)
Many of our integration tests weren't actually being run on CI because they were in the wrong place and/or missing the driver file, and it seems we've just never noticed. This makes a number of changes: - Ensures that all packages with integration tests also have a driver file in the right location. - Also standardizes the format of those files, as the boilerplate `main()` is available in `integration_test`. - Ensures that all integration_test directories are in the right place. - In a couple of places, removes a duplicate copy of the integration test file. - Makes it an error for a plugin that's not excluded to not have integration tests, so this can't easily happen again. - Adds logging of what's being run and skipped, so if something does go wrong in the future it's easy to determine what from the logs. - Excludes `*_platform_interface` since the logging was (potentially confusingly) reporting that they were skipped because they don't support the current platform. Skipping them is correct, just not for that reason. - Excludes the plugins that currently have no integration tests, with references to issues about adding them. Fixes flutter/flutter#81929
1 parent 7b155cd commit 447b548

File tree

59 files changed

+263
-492
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

59 files changed

+263
-492
lines changed

.cirrus.yml

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,7 @@ task:
112112
build_script:
113113
- flutter config --enable-linux-desktop
114114
- ./script/tool_runner.sh build-examples --linux
115-
test_script:
115+
drive_script:
116116
- xvfb-run ./script/tool_runner.sh drive-examples --linux
117117

118118
# Heavy-workload Linux tasks.
@@ -167,6 +167,8 @@ task:
167167
### Web tasks ###
168168
- name: build-web+drive-examples
169169
env:
170+
# Currently missing; see https://github.com/flutter/flutter/issues/81982
171+
PLUGINS_TO_EXCLUDE_INTEGRATION_TESTS: "image_picker_for_web,shared_preferences_web,video_player_web"
170172
matrix:
171173
CHANNEL: "master"
172174
CHANNEL: "stable"
@@ -178,10 +180,10 @@ task:
178180
- ./chromedriver/chromedriver --port=4444 &
179181
build_script:
180182
- ./script/tool_runner.sh build-examples --web
181-
test_script:
183+
drive_script:
182184
# TODO(stuartmorgan): Eliminate this check once 2.1 reaches stable.
183185
- if [[ "$CHANNEL" == "master" ]]; then
184-
- ./script/tool_runner.sh drive-examples --web
186+
- ./script/tool_runner.sh drive-examples --web --exclude $PLUGINS_TO_EXCLUDE_INTEGRATION_TESTS
185187
- else
186188
- echo "Requires null-safe integration_test; skipping."
187189
- fi
@@ -203,6 +205,9 @@ task:
203205
env:
204206
PATH: $PATH:/usr/local/bin
205207
PLUGINS_TO_SKIP_XCTESTS: "integration_test"
208+
# in_app_purchase_ios is currently missing tests; see https://github.com/flutter/flutter/issues/81695
209+
# sensor hangs on CI.
210+
PLUGINS_TO_EXCLUDE_INTEGRATION_TESTS: "in_app_purchase_ios,sensors"
206211
matrix:
207212
PLUGIN_SHARDING: "--shardIndex 0 --shardCount 4"
208213
PLUGIN_SHARDING: "--shardIndex 1 --shardCount 4"
@@ -217,12 +222,13 @@ task:
217222
- xcrun simctl create Flutter-iPhone com.apple.CoreSimulator.SimDeviceType.iPhone-11 com.apple.CoreSimulator.SimRuntime.iOS-14-3 | xargs xcrun simctl boot
218223
build_script:
219224
- ./script/tool_runner.sh build-examples --ipa
220-
test_script:
225+
xctest_script:
221226
- ./script/tool_runner.sh xctest --skip $PLUGINS_TO_SKIP_XCTESTS --ios-destination "platform=iOS Simulator,name=iPhone 11,OS=latest"
227+
drive_script:
222228
# `drive-examples` contains integration tests, which changes the UI of the application.
223229
# This UI change sometimes affects `xctest`.
224230
# So we run `drive-examples` after `xctest`, changing the order will result ci failure.
225-
- ./script/tool_runner.sh drive-examples --ios
231+
- ./script/tool_runner.sh drive-examples --ios --exclude $PLUGINS_TO_EXCLUDE_INTEGRATION_TESTS
226232
### macOS desktop tasks ###
227233
- name: build_all_plugins_macos
228234
env:
@@ -241,7 +247,7 @@ task:
241247
build_script:
242248
- flutter config --enable-macos-desktop
243249
- ./script/tool_runner.sh build-examples --macos --no-ipa
244-
test_script:
250+
drive_script:
245251
- ./script/tool_runner.sh drive-examples --macos
246252

247253
task:

packages/android_alarm_manager/example/test_driver/integration_test.dart

Lines changed: 2 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,6 @@
44

55
// @dart=2.9
66

7-
import 'dart:async';
8-
import 'dart:convert';
9-
import 'dart:io';
7+
import 'package:integration_test/integration_test_driver.dart';
108

11-
import 'package:flutter_driver/flutter_driver.dart';
12-
13-
Future<void> main() async {
14-
final FlutterDriver driver = await FlutterDriver.connect();
15-
final String data = await driver.requestData(
16-
null,
17-
timeout: const Duration(minutes: 1),
18-
);
19-
await driver.close();
20-
final Map<String, dynamic> result = jsonDecode(data);
21-
exit(result['result'] == 'true' ? 0 : 1);
22-
}
9+
Future<void> main() => integrationDriver();

packages/android_intent/example/test_driver/integration_test.dart

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,8 @@
22
// Use of this source code is governed by a BSD-style license that can be
33
// found in the LICENSE file.
44

5-
// @dart = 2.9
5+
// @dart=2.9
66

7-
import 'dart:async';
8-
import 'dart:convert';
9-
import 'dart:io';
7+
import 'package:integration_test/integration_test_driver.dart';
108

11-
import 'package:flutter_driver/flutter_driver.dart';
12-
13-
Future<void> main() async {
14-
final FlutterDriver driver = await FlutterDriver.connect();
15-
final String data = await driver.requestData(
16-
null,
17-
timeout: const Duration(minutes: 1),
18-
);
19-
await driver.close();
20-
final Map<String, dynamic> result = jsonDecode(data);
21-
exit(result['result'] == 'true' ? 0 : 1);
22-
}
9+
Future<void> main() => integrationDriver();

packages/battery/battery/integration_test/battery_test.dart renamed to packages/battery/battery/example/integration_test/battery_test.dart

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
// Use of this source code is governed by a BSD-style license that can be
33
// found in the LICENSE file.
44

5-
// @dart = 2.9
5+
// @dart=2.9
66

7+
import 'package:flutter/services.dart';
78
import 'package:flutter_test/flutter_test.dart';
89
import 'package:battery/battery.dart';
910
import 'package:integration_test/integration_test.dart';
@@ -13,7 +14,18 @@ void main() {
1314

1415
testWidgets('Can get battery level', (WidgetTester tester) async {
1516
final Battery battery = Battery();
16-
final int batteryLevel = await battery.batteryLevel;
17+
int batteryLevel;
18+
try {
19+
batteryLevel = await battery.batteryLevel;
20+
} on PlatformException catch (e) {
21+
// The "UNAVAIBLE" error just means that the system reported the battery
22+
// level as unknown (e.g., the test is running on simulator); it still
23+
// indicates that the plugin itself is working as expected, so consider it
24+
// as passing.
25+
if (e.code == 'UNAVAILABLE') {
26+
batteryLevel = 1;
27+
}
28+
}
1729
expect(batteryLevel, isNotNull);
1830
});
1931
}
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
// @dart=2.9
6+
7+
import 'package:integration_test/integration_test_driver.dart';
8+
9+
Future<void> main() => integrationDriver();

packages/connectivity/connectivity/example/test_driver/integration_test/connectivity_test.dart renamed to packages/connectivity/connectivity/example/integration_test/connectivity_test.dart

File renamed without changes.
Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
// @dart=2.9
6+
7+
import 'package:integration_test/integration_test_driver.dart';
8+
9+
Future<void> main() => integrationDriver();

packages/connectivity/connectivity/example/test_driver/test/integration_test.dart

Lines changed: 0 additions & 20 deletions
This file was deleted.

packages/connectivity/connectivity/integration_test/connectivity_test.dart

Lines changed: 0 additions & 27 deletions
This file was deleted.

packages/connectivity/connectivity_for_web/example/test_driver/integration_test.dart

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@
44

55
import 'package:integration_test/integration_test_driver.dart';
66

7-
Future<void> main() async => integrationDriver();
7+
Future<void> main() => integrationDriver();

0 commit comments

Comments
 (0)