diff --git a/utilities/new_client.py b/utilities/new_client.py new file mode 100644 index 000000000000..9b3ddcb1ae9a --- /dev/null +++ b/utilities/new_client.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import pathlib +import re +import sys +import subprocess + +import attr +from jinja2 import Environment, FileSystemLoader +from lxml import etree +from releasetool.commands.start import java as releasetool +from typing import List + +@attr.s(auto_attribs=True) +class Context: + service: str = None + api_version: str = None + artman_config: str = None + google_cloud_artifact: str = None + google_cloud_version: releasetool.Version = None + grpc_artifact: str = None + grpc_version: releasetool.Version = None + proto_artifact: str = None + proto_version: releasetool.Version = None + root_directory: pathlib.Path = \ + pathlib.Path(os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), ".."))) + description: str = "FIXME" + name: str = "FIXME" + versions: List[str] = None + jinja_env: Environment = None + + def __attrs_post_init__(self): + self.google_cloud_artifact = f"google-cloud-{self.service}" + self.grpc_artifact = f"grpc-google-cloud-{self.service}-{self.api_version}" + self.proto_artifact = f"proto-google-cloud-{self.service}-{self.api_version}" + self.jinja_env = Environment( + loader=FileSystemLoader(str(self.root_directory / "utilities/templates")) + ) + +def add_to_versions(ctx: Context) -> None: + """Add the new artifact's versions to the versions.txt manifest.""" + versions = [] + + # read from versions.txt + versions_path = ctx.root_directory / "versions.txt" + with open(versions_path) as f: + for line in f: + version_line = line.strip() + if not version_line or version_line.startswith("#"): + continue + + versions.append(releasetool.ArtifactVersions(version_line)) + + # Add new versions unless the artifacts already exist in the versions.txt manifest + ctx.google_cloud_version = next((v for v in versions if v.module == ctx.google_cloud_artifact), None) + if not ctx.google_cloud_version: + ctx.google_cloud_version = releasetool.ArtifactVersions(f"{ctx.google_cloud_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + versions.append(ctx.google_cloud_version) + + ctx.proto_version = next((v for v in versions if v.module == ctx.proto_artifact), None) + if not ctx.proto_version: + ctx.proto_version = releasetool.ArtifactVersions(f"{ctx.proto_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + versions.append(ctx.proto_version) + + ctx.grpc_version = next((v for v in versions if v.module == ctx.grpc_artifact), None) + if not ctx.grpc_version: + ctx.grpc_version = releasetool.ArtifactVersions(f"{ctx.grpc_artifact}:0.0.0-alpha:0.0.1-alpha-SNAPSHOT") + versions.append(ctx.grpc_version) + + # sort by name + versions.sort(key=lambda v: v.module) + + # update versions.txt + with open(versions_path, "w") as f: + f.write("# Format:\n") + f.write("# module:released-version:current-version\n\n") + for version in versions: + f.write("{}\n".format(version)) + + ctx.versions = versions + +def add_module_to_pom(pom: pathlib.Path, module_name: str) -> None: + """ + Insert a new name into a pom.xml file. + + This assumes that the modules are in alphabetical order and inserts it + in the correct place + """ + tree = etree.parse(str(pom)) + root = tree.getroot() + modules = root.find("{http://maven.apache.org/POM/4.0.0}modules") + + new_module = etree.Element("{http://maven.apache.org/POM/4.0.0}module") + new_module.text = module_name + + for i, module in enumerate(modules): + if module.text == module_name: + # print("already added to pom.xml, skipping") + return + + if module.text > module_name: + modules.insert(i, new_module) + break + + tree.write(str(pom), pretty_print=True, xml_declaration=True, encoding="utf-8") + +def add_dependency_management_to_pom(pom: pathlib.Path, group: str, artifact: str, version: str) -> None: + """Adds a ... section into a pom.xml file.""" + tree = etree.parse(str(pom)) + root = tree.getroot() + dependencies = root.find("{http://maven.apache.org/POM/4.0.0}dependencyManagement/{http://maven.apache.org/POM/4.0.0}dependencies") + + for dependency in dependencies: + existing = dependency.find("{http://maven.apache.org/POM/4.0.0}artifactId") + if existing is not None: + if existing.text == artifact: + print("already added dependency to pom.xml, skipping") + return + + new_dependency = etree.Element("{http://maven.apache.org/POM/4.0.0}dependency") + group_element = etree.Element("{http://maven.apache.org/POM/4.0.0}groupId") + group_element.text = group + new_dependency.append(group_element) + artifact_element = etree.Element("{http://maven.apache.org/POM/4.0.0}artifactId") + artifact_element.text = artifact + new_dependency.append(artifact_element) + version_element = etree.Element("{http://maven.apache.org/POM/4.0.0}version") + version_element.text = version + new_dependency.append(version_element) + comment = etree.Comment("{x-version-update:" + artifact + ":current}") + new_dependency.append(comment) + dependencies.append(new_dependency) + + tree.write(str(pom), pretty_print=True, xml_declaration=True, encoding="utf-8") + + +def write_synthfile(ctx: Context) -> None: + """Creates a synth.py file from a template.""" + template = ctx.jinja_env.get_template("synth.py") + synth = template.stream( + version=ctx.api_version, + service=ctx.service, + config_path=ctx.artman_config, + ) + path = ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "synth.py" + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory) + synth.dump(str(path)) + +def write_pom(template: str, path: str, ctx: Context, version: str) -> None: + """Creates a pom.xml file from a template.""" + template = ctx.jinja_env.get_template(template) + pom = template.stream( + api_version=ctx.api_version, + description=ctx.description, + name=ctx.name, + service=ctx.service, + version=version + ) + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory) + pom.dump(str(path)) + +def run_synthtool(ctx: Context) -> None: + """Runs synthtool for the initial client generation.""" + subprocess.run( + [sys.executable, "synth.py"], + check=True, + cwd=ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact + ) + +def update_stub_packages(ctx: Context) -> None: + """Updates the pom.xml documentation section to scrub certain packages.""" + # open google-cloud-clients/pom.xml and fix the Stub packages list + pom = ctx.root_directory / "google-cloud-clients/pom.xml" + tree = etree.parse(str(pom)) + + grpc_artifacts = [v.module for v in ctx.versions if v.module.startswith("grpc-")] + stub_classes = [] + for artifact in grpc_artifacts: + m = re.match("grpc-google-cloud-(.*)-(v.*)", artifact) + stub_classes.append(f"com.google.cloud.{m[1]}.{m[2]}.stub") + + for group in tree.findall(".//{http://maven.apache.org/POM/4.0.0}group"): + if group.find("{http://maven.apache.org/POM/4.0.0}title").text == "Stub packages": + group.find("{http://maven.apache.org/POM/4.0.0}packages").text = ":".join(stub_classes) + + tree.write(str(pom), pretty_print=True, xml_declaration=True, encoding="utf-8") + +def write_readme(ctx: Context) -> None: + """Creates a README.md from a template.""" + template = ctx.jinja_env.get_template("README.md") + pom = template.stream( + api_version=ctx.api_version, + description=ctx.description, + name=ctx.name, + service=ctx.service, + version=ctx.google_cloud_version + ) + path = ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "README.md" + directory = os.path.dirname(path) + if not os.path.isdir(directory): + os.makedirs(directory) + pom.dump(str(path)) + +def main(): + parser = argparse.ArgumentParser(description="Create a new client") + parser.add_argument("-v", required=True, help="API version (i.e. v1)") + parser.add_argument("-c", required=True, help="Path to config in googleapis/googleapis") + parser.add_argument("-s", required=True, help="Service name") + args = parser.parse_args() + + ctx = Context( + api_version=args.v, + service=args.s, + artman_config=args.c + ) + + add_to_versions(ctx) + write_synthfile(ctx) + run_synthtool(ctx) + write_pom( + ctx=ctx, + template="cloud_pom.xml", + path=ctx.root_directory / "google-cloud-clients" / ctx.google_cloud_artifact / "pom.xml", + version=ctx.google_cloud_version.current + ) + add_module_to_pom( + pom=ctx.root_directory / "google-cloud-clients/pom.xml", + module_name="google-cloud-iamcredentials" + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / "google-api-grpc/pom.xml", + group="com.google.api.grpc", + artifact=ctx.proto_artifact, + version=str(ctx.proto_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / "google-api-grpc/pom.xml", + group="com.google.api.grpc", + artifact=ctx.grpc_artifact, + version=str(ctx.grpc_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / "google-cloud-bom/pom.xml", + group="com.google.api.grpc", + artifact=ctx.proto_artifact, + version=str(ctx.proto_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / "google-cloud-bom/pom.xml", + group="com.google.api.grpc", + artifact=ctx.grpc_artifact, + version=str(ctx.grpc_version.current) + ) + add_dependency_management_to_pom( + pom=ctx.root_directory / "google-cloud-bom/pom.xml", + group="com.google.cloud", + artifact=ctx.google_cloud_artifact, + version=str(ctx.google_cloud_version.current) + ) + write_pom( + ctx=ctx, + template="proto_pom.xml", + path=ctx.root_directory / "google-api-grpc" / ctx.proto_artifact / "pom.xml", + version=ctx.proto_version.current + ) + write_pom( + ctx=ctx, + template="grpc_pom.xml", + path=ctx.root_directory / "google-api-grpc" / ctx.grpc_artifact / "pom.xml", + version=ctx.grpc_version.current + ) + add_module_to_pom( + pom=ctx.root_directory / "google-api-grpc/pom.xml", + module_name=ctx.grpc_artifact + ) + add_module_to_pom( + pom=ctx.root_directory / "google-api-grpc/pom.xml", + module_name=ctx.proto_artifact + ) + update_stub_packages(ctx) + write_readme(ctx) + +if __name__ == "__main__": + main() diff --git a/utilities/requirements.txt b/utilities/requirements.txt new file mode 100644 index 000000000000..6ce1600b1b6e --- /dev/null +++ b/utilities/requirements.txt @@ -0,0 +1,5 @@ +attr +gcp-releasetool +jinja2 +lxml~=4.2 +typing~=3.6 diff --git a/utilities/templates/README.md b/utilities/templates/README.md new file mode 100644 index 000000000000..be22edfd07be --- /dev/null +++ b/utilities/templates/README.md @@ -0,0 +1,100 @@ +Google Cloud Java Client for {{name}} +=================================================== + +Java idiomatic client for [{{name}}][product-overview]. + +[![Kokoro CI](http://storage.googleapis.com/cloud-devrel-public/java/badges/google-cloud-java/master.svg)](http://storage.googleapis.com/cloud-devrel-public/java/badges/google-cloud-java/master.html) +[![Maven](https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-{{service}}.svg)](https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-{{service}}.svg) +[![Codacy Badge](https://api.codacy.com/project/badge/grade/9da006ad7c3a4fe1abd142e77c003917)](https://www.codacy.com/app/mziccard/google-cloud-java) + +- [Product Documentation][product-docs] +- [Client Library Documentation][lib-docs] + +> Note: This client is a work-in-progress, and may occasionally +> make backwards-incompatible changes. + +Quickstart +---------- + +[//]: # ({x-version-update-start:google-cloud-{{service}}:released}) +If you are using Maven, add this to your pom.xml file +```xml + + com.google.cloud + google-cloud-{{service}} + 0.71.0-beta + +``` +If you are using Gradle, add this to your dependencies +```Groovy +compile 'com.google.cloud:google-cloud-{{service}}:0.71.0-beta' +``` +If you are using SBT, add this to your dependencies +```Scala +libraryDependencies += "com.google.cloud" % "google-cloud-{{service}}" % "0.71.0-beta" +``` +[//]: # ({x-version-update-end}) + +Authentication +-------------- + +See the [Authentication](https://github.com/GoogleCloudPlatform/google-cloud-java#authentication) section in the base directory's README. + +About {{name}} +---------------------------- + +[{{name}}][product-overview] FIXME + +See the [{{name}} client library docs][lib-docs] to learn how to use this {{name}} Client Library. + +Getting Started +--------------- +#### Prerequisites +You will need a [Google Developers Console](https://console.developers.google.com/) project with the {{name}} API enabled. [Follow these instructions](https://cloud.google.com/resource-manager/docs/creating-managing-projects) to get your project set up. You will also need to set up the local development environment by [installing the Google Cloud SDK](https://cloud.google.com/sdk/) and running the following commands in command line: `gcloud auth login` and `gcloud config set project [YOUR PROJECT ID]`. + +#### Installation and setup +You'll need to obtain the `google-cloud-{{service}}` library. See the [Quickstart](#quickstart) section to add `google-cloud-{{service}}` as a dependency in your code. + +Troubleshooting +--------------- + +To get help, follow the instructions in the [shared Troubleshooting document](https://github.com/googleapis/google-cloud-common/blob/master/troubleshooting/readme.md#troubleshooting). + +Transport +--------- +{{name}} uses gRPC for the transport layer. + +Java Versions +------------- + +Java 7 or above is required for using this client. + +Versioning +---------- + +This library follows [Semantic Versioning](http://semver.org/). + +It is currently in major version zero (``0.y.z``), which means that anything may change at any time and the public API should not be considered stable. + +Contributing +------------ + +Contributions to this library are always welcome and highly encouraged. + +See `google-cloud`'s [CONTRIBUTING] documentation and the [shared documentation](https://github.com/googleapis/google-cloud-common/blob/master/contributing/readme.md#how-to-contribute-to-gcloud) for more information on how to get started. + +Please note that this project is released with a Contributor Code of Conduct. By participating in this project you agree to abide by its terms. See [Code of Conduct][code-of-conduct] for more information. + +License +------- + +Apache 2.0 - See [LICENSE] for more information. + + +[CONTRIBUTING]:https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/CONTRIBUTING.md +[code-of-conduct]:https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/CODE_OF_CONDUCT.md#contributor-code-of-conduct +[LICENSE]: https://github.com/GoogleCloudPlatform/google-cloud-java/blob/master/LICENSE +[cloud-platform]: https://cloud.google.com/ +[product-overview]: FIXME +[product-docs]: FIXME +[lib-docs]: https://googleapis.github.io/google-cloud-java/google-cloud-clients/apidocs/index.html?com/google/cloud/{{service}}/{{api_version}}/package-summary.html diff --git a/utilities/templates/cloud_pom.xml b/utilities/templates/cloud_pom.xml new file mode 100644 index 000000000000..f13b78fffb64 --- /dev/null +++ b/utilities/templates/cloud_pom.xml @@ -0,0 +1,87 @@ + + + 4.0.0 + google-cloud-{{service}} + {{version}} + jar + {{name}} + https://github.com/googleapis/google-cloud-java/tree/master/google-cloud-clients/google-cloud-{{service}} + + {{description}} + + + com.google.cloud + google-cloud-clients + {{version}} + + + google-cloud-{{service}} + + + + ${project.groupId} + google-cloud-core + + + ${project.groupId} + google-cloud-core-grpc + + + com.google.api.grpc + proto-google-cloud-{{service}}-{{api_version}} + + + com.google.api.grpc + grpc-google-cloud-{{service}}-{{api_version}} + + + io.grpc + grpc-netty-shaded + + + io.grpc + grpc-stub + + + io.grpc + grpc-auth + + + ${project.groupId} + google-cloud-core + test-jar + test + + + junit + junit + test + + + org.easymock + easymock + test + + + org.objenesis + objenesis + test + + + com.google.truth + truth + test + + + com.google.api.grpc + grpc-google-iam-v1 + test + + + com.google.api + gax-grpc + testlib + test + + + diff --git a/utilities/templates/grpc_pom.xml b/utilities/templates/grpc_pom.xml new file mode 100644 index 000000000000..5fd6917880b5 --- /dev/null +++ b/utilities/templates/grpc_pom.xml @@ -0,0 +1,31 @@ + + 4.0.0 + grpc-google-cloud-{{service}}-{{api_version}} + {{version}} + grpc-google-cloud-{{service}}-{{api_version}} + GRPC library for grpc-google-cloud-{{service}}-{{api_version}} + + com.google.api.grpc + google-api-grpc + {{version}} + + + + io.grpc + grpc-stub + compile + + + io.grpc + grpc-protobuf + compile + + + com.google.api.grpc + proto-google-cloud-{{service}}-{{api_version}} + compile + + + diff --git a/utilities/templates/proto_pom.xml b/utilities/templates/proto_pom.xml new file mode 100644 index 000000000000..7567238d0488 --- /dev/null +++ b/utilities/templates/proto_pom.xml @@ -0,0 +1,36 @@ + + 4.0.0 + proto-google-cloud-{{service}}-{{api_version}} + {{version}} + proto-google-cloud-{{service}}-{{api_version}} + PROTO library for proto-google-cloud-{{service}}-{{api_version}} + + com.google.api.grpc + google-api-grpc + {{version}} + + + + com.google.protobuf + protobuf-java + compile + + + com.google.api + api-common + compile + + + com.google.api.grpc + proto-google-common-protos + compile + + + com.google.api.grpc + proto-google-iam-v1 + compile + + + diff --git a/utilities/templates/synth.py b/utilities/templates/synth.py new file mode 100644 index 000000000000..1c97d7f6fb34 --- /dev/null +++ b/utilities/templates/synth.py @@ -0,0 +1,40 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""This script is used to synthesize generated parts of this library.""" + +import synthtool as s +import synthtool.gcp as gcp +import synthtool.languages.java as java + +gapic = gcp.GAPICGenerator() +common_templates = gcp.CommonTemplates() + +versions = ['{{version}}'] +service = '{{service}}' + +for version in versions: + library = gapic.java_library( + service=service, + version=version, + config_path=f'{{config_path}}', + artman_output_name='') + + s.copy(library / f'gapic-google-cloud-{service}-{version}/src', 'src') + s.copy(library / f'grpc-google-cloud-{service}-{version}/src', f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src') + s.copy(library / f'proto-google-cloud-{service}-{version}/src', f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src') + + java.format_code('./src') + java.format_code(f'../../google-api-grpc/grpc-google-cloud-{service}-{version}/src') + java.format_code(f'../../google-api-grpc/proto-google-cloud-{service}-{version}/src')