diff --git a/.gitignore b/.gitignore index cd502da93be4..19f0ce05fe3e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ src/test/resources/gcd-v1beta2-rev1-2.1.1.zip # API key file containing value of GOOGLE_API_KEY for integration tests api_key + +# Python utilities +*.pyc +artman-genfiles diff --git a/utilities/batch_generate_apis.py b/utilities/batch_generate_apis.py new file mode 100644 index 000000000000..a7ca2c031089 --- /dev/null +++ b/utilities/batch_generate_apis.py @@ -0,0 +1,85 @@ +# Copyright 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Instructions: +# +# Check out the googleapis repo somewhere locally (e.g. from +# https://github.com/googleapis/googleapis): +# +# $ git checkout https://github.com/googleapis/googleapis.git +# +# Run this script: +# +# $ python utilities/batch_generate_apis.py PATH_TO_GOOGLEAPIS + +import argparse +import os + +import generate_api + + +def run(googleapis): + def generate(artman_yaml): + generate_api.run_generate_api(os.path.join(googleapis, artman_yaml)) + + # TODO Needs to have java_proto called instead of java_grpc + #generate('google/datastore/artman_datastore.yaml') + + generate('google/cloud/bigquery/datatransfer/artman_bigquerydatatransfer.yaml') + generate('google/bigtable/artman_bigtable.yaml') + generate('google/bigtable/admin/artman_bigtableadmin.yaml') + generate('google/container/artman_container.yaml') + generate('google/cloud/dataproc/artman_dataproc_v1.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2.yaml') + generate('google/cloud/dialogflow/artman_dialogflow_v2beta1_java.yaml') + generate('google/privacy/dlp/artman_dlp_v2.yaml') + generate('google/devtools/clouderrorreporting/artman_errorreporting.yaml') + generate('google/firestore/artman_firestore.yaml') + generate('google/cloud/language/artman_language_v1.yaml') + generate('google/cloud/language/artman_language_v1beta2.yaml') + generate('google/logging/artman_logging.yaml') + generate('google/monitoring/artman_monitoring.yaml') + generate('google/pubsub/artman_pubsub.yaml') + generate('google/cloud/oslogin/artman_oslogin_v1.yaml') + generate('google/cloud/redis/artman_redis_v1beta1.yaml') + generate('google/spanner/artman_spanner.yaml') + generate('google/spanner/admin/database/artman_spanner_admin_database.yaml') + generate('google/spanner/admin/instance/artman_spanner_admin_instance.yaml') + generate('google/cloud/speech/artman_speech_v1.yaml') + generate('google/cloud/speech/artman_speech_v1beta1.yaml') + generate('google/cloud/speech/artman_speech_v1p1beta1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1.yaml') + generate('google/cloud/texttospeech/artman_texttospeech_v1beta1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v1.yaml') + generate('google/devtools/cloudtrace/artman_cloudtrace_v2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1beta2.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1p1beta1.yaml') + generate('google/cloud/videointelligence/artman_videointelligence_v1.yaml') + generate('google/cloud/vision/artman_vision_v1.yaml') + generate('google/cloud/vision/artman_vision_v1p1beta1.yaml') + generate('google/cloud/vision/artman_vision_v1p2beta1.yaml') + generate('google/cloud/websecurityscanner/artman_websecurityscanner_v1alpha.yaml') + + +def main(): + # TODO Make the docker image the default, add --local option + parser = argparse.ArgumentParser(description='Batch generate all APIs.') + parser.add_argument('googleapis', help='The path to the googleapis repo') + args = parser.parse_args() + + run(args.googleapis) + +if __name__ == '__main__': + main() diff --git a/utilities/generate_api.py b/utilities/generate_api.py index 4e040afbcafc..ff0f810c058b 100644 --- a/utilities/generate_api.py +++ b/utilities/generate_api.py @@ -12,6 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Instructions: +# +# Find the artman config file the describes the API you want to generate a client for. +# +# $ python utilities/generate_api.py PATH_TO_ARTMAN_CONFIG_FILE + import argparse import io import os @@ -21,14 +27,27 @@ from distutils import dir_util from ruamel import yaml -def run(config_path): - api_dir_index = config_path.find('/google/') - if api_dir_index == -1: - raise ValueError('Didn\'t find /google/ in config file path; need absolute path to the artman config file.') - root_dir = config_path[0:api_dir_index] - api_dir = config_path[api_dir_index+1:] - subprocess.check_call(['artman', '--config', api_dir, '--local', '-v', '--root-dir', root_dir, 'generate', 'java_gapic']) +dir_overrides = { + 'bigtable-admin': 'google-cloud-bigtable', + 'error-reporting': 'google-cloud-errorreporting', + 'spanner-admin-instance': 'google-cloud-spanner', + 'spanner-admin-database': 'google-cloud-spanner' +} + + +def run_generate_api(config_path, noisy=False): + googleapis_index = config_path.rfind('/google/') + if googleapis_index == -1: + raise ValueError('Didn\'t find /googleapis/ in config file path; need absolute path to the artman config file.') + root_dir = config_path[0:googleapis_index] + api_dir = config_path[googleapis_index+1:] + + extra_options = [] + if noisy: + extra_options = ['-v'] + + subprocess.check_call(['artman', '--config', api_dir, '--local', '--root-dir', root_dir] + extra_options + ['generate', 'java_gapic']) with io.open(config_path, encoding='UTF-8') as config_file: artman_config_data = yaml.load(config_file, Loader=yaml.Loader) @@ -44,9 +63,6 @@ def run(config_path): proto_dir = os.path.join('artman-genfiles', 'java', proto_dirname) grpc_dir = os.path.join('artman-genfiles', 'java', grpc_dirname) gapic_dir = os.path.join('artman-genfiles', 'java', gapic_dirname) - print(proto_dir) - print(grpc_dir) - print(gapic_dir) if not os.path.exists(proto_dir): raise ValueError('generated proto dir doesn\'t exist: {}'.format(proto_dir)) if not os.path.exists(grpc_dir): @@ -78,24 +94,30 @@ def run(config_path): subprocess.call(['git', 'checkout', os.path.join(target_grpc_dir, 'pom.xml')]) api_unversioned_name = '{}-{}'.format(org_name, api_name) + if api_name in dir_overrides: + api_unversioned_name = dir_overrides[api_name] + target_gapic_dir = os.path.join('google-cloud-clients', api_unversioned_name) dir_util.copy_tree(os.path.join(gapic_dir, 'src'), os.path.join(target_gapic_dir, 'src')) - print('**** REMAINING MANUAL WORK: *****') - print('This script doesn\'t set up new clients. If this is a new client, you need to:') - print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml') - print('3. Copy an existing client pom.xml to the new client directory, update its text') - print('4. Copy an existing client README.md to the new client directory, update its text') - print('5. Add the new proto, grpc, and client modules into versions.txt') - print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml') - print('7. Run `utilities/replace_versions.py` to update the module versions') + if noisy: + print('**** REMAINING MANUAL WORK: *****') + print('This script doesn\'t set up new clients. If this is a new client, you need to:') + print('1. Add the new proto and grpc modules into google-api-grpc/pom.xml') + print('3. Copy an existing client pom.xml to the new client directory, update its text') + print('4. Copy an existing client README.md to the new client directory, update its text') + print('5. Add the new proto, grpc, and client modules into versions.txt') + print('6. Add the new proto, grpc, and client modules into google-cloud-bom/pom.xml') + print('7. Run `utilities/replace_versions.py` to update the module versions') def main(): parser = argparse.ArgumentParser(description='Regenerate a single API.') parser.add_argument('config_file', help='The artman config file for the API') + parser.add_argument('--quiet', action="store_true", default=False, + help='Don\'t print informational instructions') args = parser.parse_args() - run(args.config_file) + run_generate_api(args.config_file, not args.quiet) if __name__ == '__main__': main()