[vm/kernel/bytecode] Do not generate InstantiateType for instantiated generic function types

InstantiateType bytecode can only be used on a non-instantiated type.
Type literals for generic function types without free type variables
(e.g. '<X>(dynamic, X) => X') are now generated using PushConstant.

Change-Id: I5c26751de4b47a0b98b075d8df85060bf78157d8
Reviewed-on: https://dart-review.googlesource.com/67980
Reviewed-by: Régis Crelier <regis@google.com>
Commit-Queue: Alexander Markov <alexmarkov@google.com>

Author: alexmarkov

pkg/vm/lib/bytecode/gen_bytecode.dart

@@ -327,8 +327,8 @@ class BytecodeGenerator extends RecursiveVisitor<Null> {
     _genStaticCall(target, argDesc, totalArgCount);
   }
 
-  bool hasTypeParameters(List<DartType> typeArgs) {
-    final findTypeParams = new FindTypeParametersVisitor();
+  bool hasFreeTypeParameters(List<DartType> typeArgs) {
+    final findTypeParams = new FindFreeTypeParametersVisitor();
     return typeArgs.any((t) => t.accept(findTypeParams));
   }
 
@@ -342,7 +342,7 @@ class BytecodeGenerator extends RecursiveVisitor<Null> {
       }
     }
 
-    if (typeArgs.isEmpty || !hasTypeParameters(typeArgs)) {
+    if (typeArgs.isEmpty || !hasFreeTypeParameters(typeArgs)) {
       asm.emitPushConstant(typeArgsCPIndex());
     } else {
       if (_canReuseInstantiatorTypeArguments(typeArgs, instantiatingClass)) {
@@ -547,7 +547,7 @@ class BytecodeGenerator extends RecursiveVisitor<Null> {
 
     // TODO(alexmarkov): generate _simpleInstanceOf if possible
 
-    if (hasTypeParameters([type])) {
+    if (hasFreeTypeParameters([type])) {
       _genPushInstantiatorAndFunctionTypeArguments([type]);
     } else {
       _genPushNull(); // Instantiator type arguments.
@@ -1674,7 +1674,7 @@ class BytecodeGenerator extends RecursiveVisitor<Null> {
   visitTypeLiteral(TypeLiteral node) {
     final DartType type = node.type;
     final int typeCPIndex = cp.add(new ConstantType(type));
-    if (!hasTypeParameters([type])) {
+    if (!hasFreeTypeParameters([type])) {
       asm.emitPushConstant(typeCPIndex);
     } else {
       _genPushInstantiatorAndFunctionTypeArguments([type]);
@@ -2453,7 +2453,9 @@ class UnsupportedOperationError {
   String toString() => message;
 }
 
-class FindTypeParametersVisitor extends DartTypeVisitor<bool> {
+class FindFreeTypeParametersVisitor extends DartTypeVisitor<bool> {
+  Set<TypeParameter> _declaredTypeParameters;
+
   bool visit(DartType type) => type.accept(this);
 
   @override
@@ -2476,7 +2478,9 @@ class FindTypeParametersVisitor extends DartTypeVisitor<bool> {
   bool visitVectorType(VectorType node) => false;
 
   @override
-  bool visitTypeParameterType(TypeParameterType node) => true;
+  bool visitTypeParameterType(TypeParameterType node) =>
+      _declaredTypeParameters == null ||
+      !_declaredTypeParameters.contains(node.parameter);
 
   @override
   bool visitInterfaceType(InterfaceType node) =>
@@ -2487,10 +2491,21 @@ class FindTypeParametersVisitor extends DartTypeVisitor<bool> {
       node.typeArguments.any((t) => t.accept(this));
 
   @override
-  bool visitFunctionType(FunctionType node) =>
-      node.typeParameters.isNotEmpty ||
-      node.positionalParameters.any((t) => t.accept(this)) ||
-      node.namedParameters.any((p) => p.type.accept(this));
+  bool visitFunctionType(FunctionType node) {
+    if (node.typeParameters.isNotEmpty) {
+      _declaredTypeParameters ??= new Set<TypeParameter>();
+      _declaredTypeParameters.addAll(node.typeParameters);
+    }
+
+    final bool result = node.positionalParameters.any((t) => t.accept(this)) ||
+        node.namedParameters.any((p) => p.type.accept(this));
+
+    if (node.typeParameters.isNotEmpty) {
+      _declaredTypeParameters.removeAll(node.typeParameters);
+    }
+
+    return result;
+  }
 }
 
 // Drop kernel AST for members with bytecode.

pkg/vm/testcases/bytecode/literals.dart

@@ -82,4 +82,7 @@ class F<P, Q> extends E<Map<P, Q>> {
 
 testGenericConstInstance() => const F<int, String>();
 
+typedef GenericFunctionType = X Function<X>(X);
+testGenericFunctionTypeLiteral() => GenericFunctionType;
+
 main() {}

pkg/vm/testcases/bytecode/literals.dart.expect

@@ -2,6 +2,7 @@ library #lib;
 import self as self;
 import "dart:core" as core;
 
+typedef GenericFunctionType = <X extends core::Object = dynamic>(X) → X;
 class A extends core::Object {
   final field core::int index;
   final field core::String _name;
@@ -723,6 +724,21 @@ ConstantPool {
 ]static method testGenericConstInstance() → dynamic
   return const self::F::•<core::int, core::String>();
 [@vm.bytecode=
+Bytecode {
+  Entry                0
+  CheckStack
+  PushConstant         CP#0
+  ReturnTOS
+  PushConstant         CP#1
+  ReturnTOS
+}
+ConstantPool {
+  [0] = Type <X extends dart.core::Object = dynamic>(X) → X
+  [1] = Null
+}
+]static method testGenericFunctionTypeLiteral() → dynamic
+  return <X extends core::Object = dynamic>(X) → X;
+[@vm.bytecode=
 Bytecode {
   Entry                0
   CheckStack