[instrumentation_adapter] enable Firebase Test Lab Android testing

.ci/Dockerfile

@@ -1,11 +1,24 @@
 
 FROM cirrusci/flutter:latest
 
+RUN sudo apt-get update
+RUN sudo apt-get install -y git wget curl unzip python lsb-release sudo apt-transport-https
+
+# Add repo for gcloud sdk and install it
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | \
+    sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+
+RUN curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | \
+    sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+
+RUN sudo apt-get update && sudo apt-get install -y google-cloud-sdk && \
+    gcloud config set core/disable_usage_reporting true && \
+    gcloud config set component_manager/disable_update_check true
+
 RUN yes | sdkmanager \
     "platforms;android-27" \
     "build-tools;27.0.3" \
     "extras;google;m2repository" \
-    "extras;android;m2repository" \
-    "system-images;android-21;default;armeabi-v7a"
+    "extras;android;m2repository"
 
 RUN yes | sdkmanager --licenses

.cirrus.yml

@@ -8,7 +8,8 @@ task:
     - flutter channel master
     - flutter upgrade
     - git fetch origin master
-  activate_script: pub global activate flutter_plugin_tools
+  activate_script:
+    - pub global activate flutter_plugin_tools
   matrix:
     - name: publishable
       script: ./script/check_publish.sh
@@ -30,12 +31,10 @@ task:
           PLUGIN_SHARDING: "--shardIndex 0 --shardCount 2"
           PLUGIN_SHARDING: "--shardIndex 1 --shardCount 2"
         MAPS_API_KEY: ENCRYPTED[596a9f6bca436694625ac50851dc5da6b4d34cba8025f7db5bc9465142e8cd44e15f69e3507787753accebfc4910d550]
-      create_device_script:
-        echo no | avdmanager -v create avd -n test -k "system-images;android-21;default;armeabi-v7a"
-      start_emulator_background_script:
-      - $ANDROID_HOME/emulator/emulator -avd test -no-audio -no-window
-      wait_for_emulator_script: adb wait-for-device
+        CLOUDSDK_CORE_DISABLE_PROMPTS: 1
+        GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[ae17cec7f4d708b1e5c2e841c3643006c2ef46f3745e28fd965bc26b224158f408a0c2da40a1beb0addb163fa4c11389]
       script:
+        - echo $GCLOUD_FIREBASE_TESTLAB_KEY > ${HOME}/gcloud-service-key.json
         # Unsetting CIRRUS_CHANGE_MESSAGE and CIRRUS_COMMIT_MESSAGE as they
         # might include non-ASCII characters which makes Gradle crash.
         # See: https://github.com/flutter/flutter/issues/24935
@@ -48,9 +47,7 @@ task:
         - export CIRRUS_COMMIT_MESSAGE=""
         - ./script/incremental_build.sh build-examples --apk
         - ./script/incremental_build.sh java-test  # must come after apk build
-        # TODO(jackson): Re-enable once Android emulators support Firebase
-        # https://github.com/flutter/flutter/issues/29571
-        # - ./script/incremental_build.sh drive-examples
+        - ./script/incremental_build.sh firebase-test-lab
         - export CIRRUS_CHANGE_MESSAGE=`cat /tmp/cirrus_change_message.txt`
         - export CIRRUS_COMMIT_MESSAGE=`cat /tmp/cirrus_commit_message.txt`
 

CONTRIBUTING.md

@@ -50,13 +50,20 @@ To run the unit tests:
 flutter test test/<name_of_plugin>_test.dart
 ```
 
-To run the integration tests:
+To run the integration tests using Flutter driver:
 
 ```
 cd example
 flutter drive test/<name_of_plugin>.dart
 ```
 
+To run integration tests as instrumentation tests on a local Android device:
+
+```
+cd example
+(cd android && ./gradlew -Ptarget=$(pwd)/../test_live/<name_of_plugin>_test.dart connectedAndroidTest)
+```
+
 ## Contributing code
 
 We gladly accept contributions via GitHub pull requests.

packages/instrumentation_test/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+.dart_tool/
+
+.packages
+.pub/
+
+build/

packages/instrumentation_test/.metadata

@@ -0,0 +1,10 @@
+# This file tracks properties of this Flutter project.
+# Used by Flutter tool to assess capabilities and perform upgrades etc.
+#
+# This file should be version controlled and should not be manually edited.
+
+version:
+  revision: 3374ee380b499d99c50ed6dfdd45510aa8318741
+  channel: master
+
+project_type: plugin

packages/instrumentation_test/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.0.1
+
+* Initial release

packages/instrumentation_test/LICENSE

@@ -0,0 +1,27 @@
+// Copyright 2019 The Chromium Authors. All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//    * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//    * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//    * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

packages/instrumentation_test/README.md

@@ -0,0 +1,43 @@
+# instrumentation_test
+
+Adapts flutter_test results as Android instrumentation tests, making them usable for Firebase Test Lab and other Android CI providers.
+
+iOS support is not available yet, but is planned in the future.
+
+## Usage
+
+Use `InstrumentationTestFlutterBinding()` at the start of a test file.
+
+```dart
+import 'package:instrumentation_test/instrumentation_test.dart';
+import '../test/package_info.dart' as test;
+
+void main() {
+  InstrumentationTestFlutterBinding();
+  testWidgets("failing test example", (WidgetTester tester) async {
+    expect(2 + 2, equals(5));
+  });
+}
+```
+
+Use gradle commands to build an instrumentation test for Android.
+
+```
+pushd android
+./gradlew assembleAndroidTest
+./gradlew assembleDebug -Ptarget=<path_to_test>.dart
+popd
+```
+
+Upload to Firebase Test Lab, making sure to replace <PATH_TO_KEY_FILE>, <PROJECT_NAME>, <RESULTS_BUCKET>, and <RESULTS_DIRECTORY> with your values.
+
+```
+gcloud auth activate-service-account --key-file=<PATH_TO_KEY_FILE>
+gcloud --quiet config set project <PROJECT_NAME>
+gcloud firebase test android run --type instrumentation \
+  --app build/app/outputs/apk/debug/app-debug.apk \
+  --test build/app/outputs/apk/androidTest/debug/app-debug-androidTest.apk\
+  --timeout 2m \
+  --results-bucket=<RESULTS_BUCKET> \
+  --results-dir=<RESULTS_DIRECTORY>
+```

packages/instrumentation_test/android/.gitignore

@@ -0,0 +1,8 @@
+*.iml
+.gradle
+/local.properties
+/.idea/workspace.xml
+/.idea/libraries
+.DS_Store
+/build
+/captures

packages/instrumentation_test/android/build.gradle

@@ -0,0 +1,41 @@
+group 'com.example.instrumentation_test'
+version '1.0-SNAPSHOT'
+
+buildscript {
+    repositories {
+        google()
+        jcenter()
+    }
+
+    dependencies {
+        classpath 'com.android.tools.build:gradle:3.2.1'
+    }
+}
+
+rootProject.allprojects {
+    repositories {
+        google()
+        jcenter()
+    }
+}
+
+apply plugin: 'com.android.library'
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        minSdkVersion 16
+        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+    }
+    lintOptions {
+        disable 'InvalidPackage'
+    }
+    dependencies {
+        api 'junit:junit:4.12'
+        api 'androidx.test:core:1.0.0'
+        api 'androidx.test:runner:1.1.1'
+        api 'androidx.test:rules:1.1.1'
+        api 'androidx.test.espresso:espresso-core:3.1.1'
+    }
+}

packages/instrumentation_test/android/gradle.properties

@@ -0,0 +1,2 @@
+org.gradle.jvmargs=-Xmx1536M
+

packages/instrumentation_test/android/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'instrumentation_test'

packages/instrumentation_test/android/src/main/AndroidManifest.xml

@@ -0,0 +1,3 @@
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+  package="com.example.instrumentation_test">
+</manifest>

packages/instrumentation_test/android/src/main/java/com/example/instrumentationtest/FlutterRunner.java

@@ -0,0 +1,74 @@
+package io.flutter.plugins.instrumentationtest;
+
+import androidx.test.rule.ActivityTestRule;
+import io.flutter.app.FlutterActivity;
+import io.flutter.plugin.common.MethodCall;
+import io.flutter.plugin.common.MethodChannel;
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
+import io.flutter.plugin.common.MethodChannel.Result;
+import io.flutter.view.FlutterView;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import org.junit.runner.Description;
+import org.junit.runner.Runner;
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunNotifier;
+
+public class FlutterRunner extends Runner {
+
+  private static final String CHANNEL = "dev.flutter/InstrumentationTestFlutterBinding";
+  CompletableFuture<Map<String, String>> testResults;
+
+  final Class activityClass;
+
+  public FlutterRunner(Class<?> klass) {
+    activityClass = klass;
+    ActivityTestRule<FlutterActivity> rule = new ActivityTestRule<>(activityClass);
+    FlutterActivity activity = rule.launchActivity(null);
+    FlutterView view = activity.getFlutterView();
+    MethodChannel methodChannel = new MethodChannel(view, CHANNEL);
+    testResults = new CompletableFuture<>();
+    methodChannel.setMethodCallHandler(
+        new MethodCallHandler() {
+          @Override
+          public void onMethodCall(MethodCall call, Result result) {
+            if (call.method.equals("testFinished")) {
+              Map<String, String> results = call.argument("results");
+              testResults.complete(results);
+              result.success(null);
+            } else {
+              result.notImplemented();
+            }
+          }
+        });
+  }
+
+  @Override
+  public Description getDescription() {
+    return Description.createTestDescription(activityClass, "Flutter Tests");
+  }
+
+  @Override
+  public void run(RunNotifier notifier) {
+    Map<String, String> results = null;
+    try {
+      results = testResults.get();
+    } catch (ExecutionException e) {
+      e.printStackTrace();
+    } catch (InterruptedException e) {
+      e.printStackTrace();
+    }
+
+    for (String name : results.keySet()) {
+      Description d = Description.createTestDescription(activityClass, name);
+      notifier.fireTestStarted(d);
+      String outcome = results.get(name);
+      if (outcome.equals("failed")) {
+        Exception dummyException = new Exception(outcome);
+        notifier.fireTestFailure(new Failure(d, dummyException));
+      }
+      notifier.fireTestFinished(d);
+    }
+  }
+}

packages/instrumentation_test/android/src/main/java/com/example/instrumentationtest/InstrumentationTestPlugin.java

@@ -0,0 +1,30 @@
+package com.example.instrumentationtest;
+
+import io.flutter.plugin.common.MethodCall;
+import io.flutter.plugin.common.MethodChannel;
+import io.flutter.plugin.common.MethodChannel.MethodCallHandler;
+import io.flutter.plugin.common.MethodChannel.Result;
+import io.flutter.plugin.common.PluginRegistry.Registrar;
+
+/** InstrumentationTestPlugin */
+public class InstrumentationTestPlugin implements MethodCallHandler {
+
+  private static final String CHANNEL = "dev.flutter/InstrumentationTestFlutterBinding";
+
+  /** Plugin registration. */
+  public static void registerWith(Registrar registrar) {
+    final MethodChannel channel = new MethodChannel(registrar.messenger(), CHANNEL);
+    channel.setMethodCallHandler(new InstrumentationTestPlugin());
+  }
+
+  @Override
+  public void onMethodCall(MethodCall call, Result result) {
+      if (call.method.equals("testFinished")) {
+          Map<String, String> results = call.argument("results");
+          FlutterRunner.completeTestResults(results);
+          result.success(null);
+      } else {
+          result.notImplemented();
+      }
+  }
+}

packages/instrumentation_test/lib/instrumentation_test.dart

@@ -0,0 +1,31 @@
+import 'package:flutter_test/flutter_test.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/services.dart';
+
+/// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
+/// on a channel to adapt them to native instrumentation test format.
+class InstrumentationTestFlutterBinding extends LiveTestWidgetsFlutterBinding {
+  InstrumentationTestFlutterBinding() {
+    tearDownAll(() {
+      _channel.invokeMethod<void>('testFinished', <String, dynamic>{'results': _results});
+    });
+  }
+  static const MethodChannel _channel =
+      MethodChannel('dev.flutter/InstrumentationTestFlutterBinding');
+
+  static Map<String, String> _results = <String, String>{};
+
+  @override
+  Future<void> runTest(Future<void> testBody(), VoidCallback invariantTester,
+      {String description = '', Duration timeout}) async {
+    // TODO(jackson): Report the results individually instead of all at once
+    // See https://github.com/flutter/flutter/issues/38985
+    reportTestException =
+        (FlutterErrorDetails details, String testDescription) {
+      _results[description] = 'failed';
+    };
+    await super.runTest(testBody, invariantTester,
+        description: description, timeout: timeout);
+    _results[description] ??= 'success';
+  }
+}

packages/instrumentation_test/pubspec.yaml

@@ -0,0 +1,19 @@
+name: instrumentation_test
+description: Runs tests that use the flutter_test API as platform native instrumentation tests.
+version: 0.0.1
+author:
+homepage:
+
+environment:
+  sdk: ">=2.1.0 <3.0.0"
+
+dependencies:
+  flutter:
+    sdk: flutter
+  flutter_test:
+    sdk: flutter
+
+flutter:
+  plugin:
+    androidPackage: io.flutter.plugins.instrumentationtest
+    pluginClass: InstrumentationTestPlugin