[instrumentation_adapter] enable Firebase Test Lab Android testing

.ci/Dockerfile

@@ -5,7 +5,6 @@ RUN yes | sdkmanager \
     "platforms;android-27" \
     "build-tools;27.0.3" \
     "extras;google;m2repository" \
-    "extras;android;m2repository" \
-    "system-images;android-21;default;armeabi-v7a"
+    "extras;android;m2repository"
 
 RUN yes | sdkmanager --licenses

.cirrus.yml

@@ -30,11 +30,6 @@ task:
           PLUGIN_SHARDING: "--shardIndex 0 --shardCount 2"
           PLUGIN_SHARDING: "--shardIndex 1 --shardCount 2"
         MAPS_API_KEY: ENCRYPTED[596a9f6bca436694625ac50851dc5da6b4d34cba8025f7db5bc9465142e8cd44e15f69e3507787753accebfc4910d550]
-      create_device_script:
-        echo no | avdmanager -v create avd -n test -k "system-images;android-21;default;armeabi-v7a"
-      start_emulator_background_script:
-      - $ANDROID_HOME/emulator/emulator -avd test -no-audio -no-window
-      wait_for_emulator_script: adb wait-for-device
       script:
         # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they
         # might include non-ASCII characters which makes Gradle crash.
@@ -48,9 +43,6 @@ task:
         - export CIRRUS_COMMIT_MESSAGE=""
         - ./script/incremental_build.sh build-examples --apk
         - ./script/incremental_build.sh java-test  # must come after apk build
-        # TODO(jackson): Re-enable once Android emulators support Firebase
-        # https://github.com/flutter/flutter/issues/29571
-        # - ./script/incremental_build.sh drive-examples
         - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt`
         - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt`
 

CONTRIBUTING.md

@@ -50,13 +50,20 @@ To run the unit tests:
 flutter test test/<name_of_plugin>_test.dart
 ```
 
-To run the integration tests:
+To run the integration tests using Flutter driver:
 
 ```
 cd example
 flutter drive test/<name_of_plugin>.dart
 ```
 
+To run integration tests as instrumentation tests on a local Android device:
+
+```
+cd example
+(cd android && ./gradlew -Ptarget=$(pwd)/../test_live/<name_of_plugin>_test.dart connectedAndroidTest)
+```
+
 ## Contributing code
 
 We gladly accept contributions via GitHub pull requests.

packages/instrumentation_adapter/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+.dart_tool/
+
+.packages
+.pub/
+
+build/

packages/instrumentation_adapter/.metadata

@@ -0,0 +1,10 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: 3374ee380b499d99c50ed6dfdd45510aa8318741
+  channel: master
+
+project_type: plugin

packages/instrumentation_adapter/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+* Initial release

packages/instrumentation_adapter/LICENSE

@@ -0,0 +1,27 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/instrumentation_adapter/README.md

@@ -0,0 +1,45 @@
+# instrumentation_adapter
+
+Adapts flutter_test results as Android instrumentation tests, making them usable for Firebase Test Lab and other Android CI providers.
+
+iOS support is not available yet, but is planned in the future.
+
+## Usage
+
+Add a dependency on the `instrumentation_adapter` package in the `dev_dependencies` section of pubspec.yaml. For plugins, do this in the pubspec.yaml of the example app.
+
+Invoke `InstrumentationAdapterFlutterBinding.ensureInitialized()` at the start of a test file.
+
+```dart
+import 'package:instrumentation_adapter/instrumentation_adapter.dart';
+import '../test/package_info.dart' as test;
+
+void main() {
+  InstrumentationAdapterFlutterBinding.ensureInitialized();
+  testWidgets("failing test example", (WidgetTester tester) async {
+    expect(2 + 2, equals(5));
+  });
+}
+```
+
+Use gradle commands to build an instrumentation test for Android.
+
+```
+pushd android
+./gradlew assembleAndroidTest
+./gradlew assembleDebug -Ptarget=<path_to_test>.dart
+popd
+```
+
+Upload to Firebase Test Lab, making sure to replace <PATH_TO_KEY_FILE>, <PROJECT_NAME>, <RESULTS_BUCKET>, and <RESULTS_DIRECTORY> with your values.
+
+```
+gcloud auth activate-service-account --key-file=<PATH_TO_KEY_FILE>
+gcloud --quiet config set project <PROJECT_NAME>
+gcloud firebase test android run --type instrumentation \
+  --app build/app/outputs/apk/debug/app-debug.apk \
+  --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk\
+  --timeout 2m \
+  --results-bucket=<RESULTS_BUCKET> \
+  --results-dir=<RESULTS_DIRECTORY>
+```

packages/instrumentation_adapter/android/.gitignore

@@ -0,0 +1,8 @@
+*.iml
+.gradle
+/local.properties
+/.idea/workspace.xml
+/.idea/libraries
+.DS_Store
+/build
+/captures

packages/instrumentation_adapter/android/build.gradle

@@ -0,0 +1,41 @@
+group 'com.example.instrumentation_adapter'
+version '1.0-SNAPSHOT'
+
+buildscript {
+    repositories {
+        google()
+        jcenter()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.2.1'
+    }
+}
+
+rootProject.allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
+
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        minSdkVersion 16
+        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+    }
+    lintOptions {
+        disable 'InvalidPackage'
+    }
+    dependencies {
+        api 'junit:junit:4.12'
+        api 'androidx.test:core:1.0.0'
+        api 'androidx.test:runner:1.1.1'
+        api 'androidx.test:rules:1.1.1'
+        api 'androidx.test.espresso:espresso-core:3.1.1'
+    }
+}

packages/instrumentation_adapter/android/gradle.properties

@@ -0,0 +1,2 @@
+org.gradle.jvmargs=-Xmx1536M
+

packages/instrumentation_adapter/android/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'instrumentation_adapter'

packages/instrumentation_adapter/android/src/main/AndroidManifest.xml

@@ -0,0 +1,3 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+  package="dev.flutter.instrumentation_adapter">
+</manifest>

packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterRunner.java

@@ -0,0 +1,53 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dev.flutter.plugins.instrumentationadapter;
+
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import org.junit.runner.Description;
+import org.junit.runner.Runner;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunNotifier;
+
+public class FlutterRunner extends Runner {
+
+  final Class testClass;
+
+  public FlutterRunner(Class<FlutterTest> testClass) {
+    super();
+    this.testClass = testClass;
+    try {
+      testClass.newInstance().launchActivity();
+    } catch (InstantiationException | IllegalAccessException e) {
+      throw new IllegalThreadStateException("Unable to launch test");
+    }
+  }
+
+  @Override
+  public Description getDescription() {
+    return Description.createTestDescription(testClass, "Flutter Tests");
+  }
+
+  @Override
+  public void run(RunNotifier notifier) {
+    Map<String, String> results = null;
+    try {
+      results = InstrumentationAdapterPlugin.testResults.get();
+    } catch (ExecutionException | InterruptedException e) {
+      throw new IllegalThreadStateException("Unable to get test results");
+    }
+
+    for (String name : results.keySet()) {
+      Description d = Description.createTestDescription(testClass, name);
+      notifier.fireTestStarted(d);
+      String outcome = results.get(name);
+      if (outcome.equals("failed")) {
+        Exception dummyException = new Exception(outcome);
+        notifier.fireTestFailure(new Failure(d, dummyException));
+      }
+      notifier.fireTestFinished(d);
+    }
+  }
+}

packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/FlutterTest.java

@@ -0,0 +1,9 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dev.flutter.plugins.instrumentationadapter;
+
+public abstract class FlutterTest {
+  public abstract void launchActivity();
+}

packages/instrumentation_adapter/android/src/main/java/dev/flutter/instrumentationadapter/InstrumentationAdapterPlugin.java

@@ -0,0 +1,38 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package dev.flutter.plugins.instrumentationadapter;
+
+import io.flutter.plugin.common.MethodCall;
+import io.flutter.plugin.common.MethodChannel;
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
+import io.flutter.plugin.common.MethodChannel.Result;
+import io.flutter.plugin.common.PluginRegistry.Registrar;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+
+/** InstrumentationAdapterPlugin */
+public class InstrumentationAdapterPlugin implements MethodCallHandler {
+
+  public static CompletableFuture<Map<String, String>> testResults = new CompletableFuture<>();
+
+  private static final String CHANNEL = "dev.flutter/InstrumentationAdapterFlutterBinding";
+
+  /** Plugin registration. */
+  public static void registerWith(Registrar registrar) {
+    final MethodChannel channel = new MethodChannel(registrar.messenger(), CHANNEL);
+    channel.setMethodCallHandler(new InstrumentationAdapterPlugin());
+  }
+
+  @Override
+  public void onMethodCall(MethodCall call, Result result) {
+    if (call.method.equals("allTestsFinished")) {
+      Map<String, String> results = call.argument("results");
+      testResults.complete(results);
+      result.success(null);
+    } else {
+      result.notImplemented();
+    }
+  }
+}

packages/instrumentation_adapter/lib/instrumentation_adapter.dart

@@ -0,0 +1,48 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter/widgets.dart';
+
+/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
+/// on a channel to adapt them to native instrumentation test format.
+class InstrumentationAdapterFlutterBinding
+    extends LiveTestWidgetsFlutterBinding {
+  InstrumentationAdapterFlutterBinding() {
+    // TODO(jackson): Report test results as they arrive
+    tearDownAll(() async {
+      await _channel.invokeMethod<void>(
+          'allTestsFinished', <String, dynamic>{'results': _results});
+    });
+  }
+
+  static WidgetsBinding ensureInitialized() {
+    if (WidgetsBinding.instance == null) {
+      InstrumentationAdapterFlutterBinding();
+    }
+    assert(WidgetsBinding.instance is InstrumentationAdapterFlutterBinding);
+    return WidgetsBinding.instance;
+  }
+
+  static const MethodChannel _channel =
+      MethodChannel('dev.flutter/InstrumentationAdapterFlutterBinding');
+
+  static Map<String, String> _results = <String, String>{};
+
+  @override
+  Future<void> runTest(Future<void> testBody(), VoidCallback invariantTester,
+      {String description = '', Duration timeout}) async {
+    // TODO(jackson): Report the results individually instead of all at once
+    // See https://github.com/flutter/flutter/issues/38985
+    reportTestException =
+        (FlutterErrorDetails details, String testDescription) {
+      _results[description] = 'failed';
+    };
+    await super.runTest(testBody, invariantTester,
+        description: description, timeout: timeout);
+    _results[description] ??= 'success';
+  }
+}

packages/instrumentation_adapter/pubspec.yaml

@@ -0,0 +1,19 @@
+name: instrumentation_adapter
+description: Runs tests that use the flutter_test API as platform native instrumentation tests.
+version: 0.0.1
+author: Flutter Team <[email protected]>
+homepage: https://github.com/flutter/plugins/tree/master/packages/instrumentation_adapter
+
+environment:
+  sdk: ">=2.1.0 <3.0.0"
+
+dependencies:
+  flutter:
+    sdk: flutter
+  flutter_test:
+    sdk: flutter
+
+flutter:
+  plugin:
+    androidPackage: dev.flutter.plugins.instrumentationadapter
+    pluginClass: InstrumentationAdapterPlugin

script/build_all_plugins_app.sh

@@ -12,7 +12,7 @@ check_changed_packages > /dev/null
 
 cd $REPO_DIR/examples/all_plugins
 flutter clean > /dev/null
-(cd "$REPO_DIR" && pub global run flutter_plugin_tools gen-pubspec)
+(cd "$REPO_DIR" && pub global run flutter_plugin_tools gen-pubspec --exclude instrumentation_adapter)
 
 function error() {
   echo "$@" 1>&2