Add template for a flattening conversion script (#258)

The generated method flattening semantics change between the the
original monolithic GAPIC generator and this GAPIC generator (aka the
Python Microgenerator). User code that calls methods from GAPICs
generated by the monolith will need to be converted to use the new
flattening semantics.

This change describes a template for an automated per-service
conversion script.

Cases the conversion script will NOT handle correctly include
* Dynamic method dispatch
* Method calls via an alias or as a free function
* Star arg exapnsion in a method call (*args or **kwargs)

Plain, normal, vanilla calls are the target for conversion.

Author: software-dov

packages/gapic-generator/gapic/templates/scripts/fixup_%service_keywords.py.j2

@@ -0,0 +1,125 @@
+{% extends '_base.py.j2' %}
+{% block content %}
+import argparse
+import os
+import libcst as cst
+from typing import (Any, Callable, Dict, List, Sequence, Tuple)
+
+
+def partition(
+    predicate: Callable[[Any], bool],
+    iterator: Sequence[Any]
+) -> Tuple[List[Any], List[Any]]:
+    """A stable, out-of-place partition."""
+    results = ([], [])
+
+    for i in iterator:
+        results[int(predicate(i))].append(i)
+
+    # Returns trueList, falseList
+    return results[1], results[0]
+
+
+class {{ service.client_name }}CallTransformer(cst.CSTTransformer):
+    CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata')
+
+    METHOD_TO_PARAMS: Dict[str, Tuple[str]] = {
+        {% for method in service.methods.values() -%}
+          '{{ method.name|snake_case }}': ({% for field in method.legacy_flattened_fields.values() %}'{{ field.name }}', {% endfor %}),
+        {% endfor -%}
+    }
+
+    def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode:
+        try:
+            key = original.func.attr.value
+            kword_params = self.METHOD_TO_PARAMS[key]
+        except (AttributeError, KeyError):
+            # Either not a method from the API or too convoluted to be sure.
+            return updated
+
+
+        # If the existing code is valid, keyword args come after positional args.
+        # Therefore, all positional args must map to the first parameters.
+        args, kwargs = partition(lambda a: not bool(a.keyword), updated.args)
+        if any(k.keyword.value == "request" for k in kwargs):
+            # We've already fixed this file, don't fix it again.
+            return updated
+
+        kwargs, ctrl_kwargs = partition(
+            lambda a: not a.keyword.value in self.CTRL_PARAMS,
+            kwargs
+        )
+
+        args, ctrl_args = args[:len(kword_params)], args[len(kword_params):]
+        ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl))
+                           for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS))
+
+        request_arg = cst.Arg(
+            value=cst.Dict([
+                cst.DictElement(
+                    cst.SimpleString("'{}'".format(name)),
+                    {# Inline comments and formatting are currently stripped out. #}
+                    {# My current attempts at preverving comments and formatting #}
+                    {# keep the comments, but the formatting is run through a log #}
+                    {# chipper, and an extra comma gets added, which causes a #}
+                    {# parse error. #}
+                    cst.Element(value=arg.value)
+                )
+                # Note: the args + kwargs looks silly, but keep in mind that
+                # the control parameters had to be stripped out, and that
+                # those could have been passed positionally or by keyword.
+                for name, arg in zip(kword_params, args + kwargs)]),
+            keyword=cst.Name("request")
+        )
+
+        return updated.with_changes(
+            args=[request_arg] + ctrl_kwargs
+        )
+
+
+def fix_files(
+    dirs: Sequence[str],
+    *,
+    transformer={{ service.client_name }}CallTransformer(),
+):
+    pyfile_gen = (os.path.join(root, f)
+                  for d in dirs
+                  for root, _, files in os.walk(d)
+                  for f in files if os.path.splitext(f)[1] == ".py")
+
+    for fpath in pyfile_gen:
+        with open(fpath, 'r+') as f:
+            src = f.read()
+            tree = cst.parse_module(src)
+            updated = tree.visit(transformer)
+            f.seek(0)
+            f.truncate()
+            f.write(updated.code)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description="""Fix up source that uses the {{ service.name }} client library.
+
+Note: This tool operates at a best-effort level at converting positional
+      parameters in client method calls to keyword based parameters.
+      Cases where it WILL FAIL include
+      A) * or ** expansion in a method call.
+      B) Calls via function or method alias (includes free function calls)
+      C) Indirect or dispatched calls (e.g. the method is looked up dynamically)
+
+      These all constitute false negatives. The tool will also detect false
+      positives when an API method shares a name with another method.
+
+      Be sure to back up your source files before running this tool and to compare the diffs.
+""")
+    parser.add_argument(
+        '-d',
+        metavar='dir',
+        dest='dirs',
+        action='append',
+        help='a directory to walk for python files to fix up'
+    )
+    args = parser.parse_args()
+    fix_files(args.dirs or ['.'])
+{% endblock %}

packages/gapic-generator/gapic/templates/setup.py.j2

@@ -21,6 +21,16 @@ setuptools.setup(
         'grpcio >= 1.10.0',
         'proto-plus >= 0.4.0',
     ),
+    setup_requires=[
+        'libcst >= 0.2.5',
+    ],
+    scripts=[
+        {% for proto in api.all_protos.values() -%}
+        {% for service in proto.services.values() -%}
+        'scripts/fixup_{{ service.module_name }}_keywords.py',
+        {% endfor -%}
+        {% endfor -%}
+    ],
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Intended Audience :: Developers',