From 19fdef378858e30aa3a6908dd69c1947f58514dd Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Tue, 2 Jun 2020 21:12:54 +0000 Subject: [PATCH 1/6] chore(deps): update dependency google-cloud-texttospeech to v2 --- translate/cloud-client/hybrid_glossaries/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/translate/cloud-client/hybrid_glossaries/requirements.txt b/translate/cloud-client/hybrid_glossaries/requirements.txt index d13c5f7d808..ed9bbd1eae8 100644 --- a/translate/cloud-client/hybrid_glossaries/requirements.txt +++ b/translate/cloud-client/hybrid_glossaries/requirements.txt @@ -1,3 +1,3 @@ google-cloud-translate==2.0.1 google-cloud-vision==1.0.0 -google-cloud-texttospeech==1.0.1 +google-cloud-texttospeech==2.0.0 From d282383de596d691b1e34135466bb1abd9612804 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 2 Jun 2020 22:38:00 +0000 Subject: [PATCH 2/6] fix: enable tests and fix --- .../hybrid_glossaries/hybrid_tutorial.py | 19 +- ...orial_tests.py => hybrid_tutorial_test.py} | 11 +- .../cloud-client/hybrid_glossaries/noxfile.py | 225 ++++++++++++++++++ .../resources/non_standard_format.txt | 2 +- 4 files changed, 244 insertions(+), 13 deletions(-) rename translate/cloud-client/hybrid_glossaries/{hybrid_tutorial_tests.py => hybrid_tutorial_test.py} (94%) create mode 100644 translate/cloud-client/hybrid_glossaries/noxfile.py diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py index 0da84b72207..1b320e5b2a4 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial.py @@ -193,21 +193,28 @@ def text_to_speech(text, outfile): client = texttospeech.TextToSpeechClient() # Sets the text input to be synthesized - synthesis_input = texttospeech.types.SynthesisInput(ssml=ssml) + synthesis_input = texttospeech.SynthesisInput(ssml=ssml) # Builds the voice request, selects the language code ("en-US") and # the SSML voice gender ("MALE") - voice = texttospeech.types.VoiceSelectionParams( + voice = texttospeech.VoiceSelectionParams( language_code='en-US', - ssml_gender=texttospeech.enums.SsmlVoiceGender.MALE) + ssml_gender=texttospeech.SsmlVoiceGender.MALE) # Selects the type of audio file to return - audio_config = texttospeech.types.AudioConfig( - audio_encoding=texttospeech.enums.AudioEncoding.MP3) + audio_config = texttospeech.AudioConfig( + audio_encoding=texttospeech.AudioEncoding.MP3) # Performs the text-to-speech request on the text input with the selected # voice parameters and audio file type - response = client.synthesize_speech(synthesis_input, voice, audio_config) + + request = texttospeech.SynthesizeSpeechRequest( + input=synthesis_input, + voice=voice, + audio_config=audio_config + ) + + response = client.synthesize_speech(request=request) # Writes the synthetic audio to the output file. with open(outfile, 'wb') as out: diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_tests.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py similarity index 94% rename from translate/cloud-client/hybrid_glossaries/hybrid_tutorial_tests.py rename to translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py index 5224ca524a9..cfc1c5915a3 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_tests.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py @@ -13,6 +13,7 @@ # limitations under the License. import os +import re import sys from hybrid_tutorial import create_glossary @@ -28,21 +29,19 @@ def test_vision_standard_format(): - - expected_text = 'This is\na test!\n' - alt_expected_text = 'This\nis\na test!\n' - # Generate text using Vision API text = pic_to_text('resources/standard_format.jpeg') - assert (text == expected_text) or (text == alt_expected_text) + assert re.match("This\s?is\s?a\s?test!\s?", text) + def test_vision_non_standard_format(): # Generate text text = pic_to_text('resources/non_standard_format.png') - + with open("bad-output", "w") as f: + f.write(text) # Read expected text with open('resources/non_standard_format.txt') as f: expected_text = f.read() diff --git a/translate/cloud-client/hybrid_glossaries/noxfile.py b/translate/cloud-client/hybrid_glossaries/noxfile.py new file mode 100644 index 00000000000..b23055f14a6 --- /dev/null +++ b/translate/cloud-client/hybrid_glossaries/noxfile.py @@ -0,0 +1,225 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import print_function + +import os +from pathlib import Path +import sys + +import nox + + +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING +# DO NOT EDIT THIS FILE EVER! +# WARNING - WARNING - WARNING - WARNING - WARNING +# WARNING - WARNING - WARNING - WARNING - WARNING + +# Copy `noxfile_config.py` to your directory and modify it instead. + + +# `TEST_CONFIG` dict is a configuration hook that allows users to +# modify the test configurations. The values here should be in sync +# with `noxfile_config.py`. Users will copy `noxfile_config.py` into +# their directory and modify it. + +TEST_CONFIG = { + # You can opt out from the test for specific Python versions. + 'ignored_versions': ["2.7"], + + # An envvar key for determining the project id to use. Change it + # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a + # build specific Cloud project. You can also use your own string + # to use your own Cloud project. + 'gcloud_project_env': 'GCLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + + # A dictionary you want to inject into your test. Don't put any + # secrets here. These values will override predefined values. + 'envs': {}, +} + + +try: + # Ensure we can import noxfile_config in the project's directory. + sys.path.append('.') + from noxfile_config import TEST_CONFIG_OVERRIDE +except ImportError as e: + print("No user noxfile_config found: detail: {}".format(e)) + TEST_CONFIG_OVERRIDE = {} + +# Update the TEST_CONFIG with the user supplied values. +TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) + + +def get_pytest_env_vars(): + """Returns a dict for pytest invocation.""" + ret = {} + + # Override the GCLOUD_PROJECT and the alias. + env_key = TEST_CONFIG['gcloud_project_env'] + # This should error out if not set. + ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] + ret['GCLOUD_PROJECT'] = os.environ[env_key] + + # Apply user supplied envs. + ret.update(TEST_CONFIG['envs']) + return ret + + +# DO NOT EDIT - automatically generated. +# All versions used to tested samples. +ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] + +# Any default versions that should be ignored. +IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] + +TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) + +INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) +# +# Style Checks +# + + +def _determine_local_import_names(start_dir): + """Determines all import names that should be considered "local". + + This is used when running the linter to insure that import order is + properly checked. + """ + file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] + return [ + basename + for basename, extension in file_ext_pairs + if extension == ".py" + or os.path.isdir(os.path.join(start_dir, basename)) + and basename not in ("__pycache__") + ] + + +# Linting with flake8. +# +# We ignore the following rules: +# E203: whitespace before ‘:’ +# E266: too many leading ‘#’ for block comment +# E501: line too long +# I202: Additional newline in a section of imports +# +# We also need to specify the rules which are ignored by default: +# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] +FLAKE8_COMMON_ARGS = [ + "--show-source", + "--builtin=gettext", + "--max-complexity=20", + "--import-order-style=google", + "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", + "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", + "--max-line-length=88", +] + + +@nox.session +def lint(session): + session.install("flake8", "flake8-import-order") + + local_names = _determine_local_import_names(".") + args = FLAKE8_COMMON_ARGS + [ + "--application-import-names", + ",".join(local_names), + "." + ] + session.run("flake8", *args) + + +# +# Sample Tests +# + + +PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] + + +def _session_tests(session, post_install=None): + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars() + ) + + +@nox.session(python=ALL_VERSIONS) +def py(session): + """Runs py.test for a sample using the specified version of Python.""" + if session.python in TESTED_VERSIONS: + _session_tests(session) + else: + session.skip("SKIPPED: {} tests are disabled for this sample.".format( + session.python + )) + + +# +# Readmegen +# + + +def _get_repo_root(): + """ Returns the root folder of the project. """ + # Get root of this repository. Assume we don't have directories nested deeper than 10 items. + p = Path(os.getcwd()) + for i in range(10): + if p is None: + break + if Path(p / ".git").exists(): + return str(p) + p = p.parent + raise Exception("Unable to detect repository root.") + + +GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) + + +@nox.session +@nox.parametrize("path", GENERATED_READMES) +def readmegen(session, path): + """(Re-)generates the readme for a sample.""" + session.install("jinja2", "pyyaml") + dir_ = os.path.dirname(path) + + if os.path.exists(os.path.join(dir_, "requirements.txt")): + session.install("-r", os.path.join(dir_, "requirements.txt")) + + in_file = os.path.join(dir_, "README.rst.in") + session.run( + "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file + ) diff --git a/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt b/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt index 8a6e3c113d1..529799ee94b 100644 --- a/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt +++ b/translate/cloud-client/hybrid_glossaries/resources/non_standard_format.txt @@ -19,11 +19,11 @@ $10 Firebase Fruit Salad watermelon, honeydew melon, and pineapple +$6 Compute Engine Burger quarter-pound burger with cheddar cheese $10 -$6 BigQuery BLT bacon, lettuce, and tomato sandwich From c8515260b1d39ea6f55a3ed73df71341f4fc5213 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 2 Jun 2020 22:38:59 +0000 Subject: [PATCH 3/6] chore: revert unintended changes --- .../hybrid_glossaries/hybrid_tutorial_test.py | 3 +- .../cloud-client/hybrid_glossaries/noxfile.py | 225 ------------------ 2 files changed, 1 insertion(+), 227 deletions(-) delete mode 100644 translate/cloud-client/hybrid_glossaries/noxfile.py diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py index cfc1c5915a3..272a7ec0cdd 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py @@ -40,8 +40,7 @@ def test_vision_non_standard_format(): # Generate text text = pic_to_text('resources/non_standard_format.png') - with open("bad-output", "w") as f: - f.write(text) + # Read expected text with open('resources/non_standard_format.txt') as f: expected_text = f.read() diff --git a/translate/cloud-client/hybrid_glossaries/noxfile.py b/translate/cloud-client/hybrid_glossaries/noxfile.py deleted file mode 100644 index b23055f14a6..00000000000 --- a/translate/cloud-client/hybrid_glossaries/noxfile.py +++ /dev/null @@ -1,225 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import os -from pathlib import Path -import sys - -import nox - - -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING -# DO NOT EDIT THIS FILE EVER! -# WARNING - WARNING - WARNING - WARNING - WARNING -# WARNING - WARNING - WARNING - WARNING - WARNING - -# Copy `noxfile_config.py` to your directory and modify it instead. - - -# `TEST_CONFIG` dict is a configuration hook that allows users to -# modify the test configurations. The values here should be in sync -# with `noxfile_config.py`. Users will copy `noxfile_config.py` into -# their directory and modify it. - -TEST_CONFIG = { - # You can opt out from the test for specific Python versions. - 'ignored_versions': ["2.7"], - - # An envvar key for determining the project id to use. Change it - # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a - # build specific Cloud project. You can also use your own string - # to use your own Cloud project. - 'gcloud_project_env': 'GCLOUD_PROJECT', - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', - - # A dictionary you want to inject into your test. Don't put any - # secrets here. These values will override predefined values. - 'envs': {}, -} - - -try: - # Ensure we can import noxfile_config in the project's directory. - sys.path.append('.') - from noxfile_config import TEST_CONFIG_OVERRIDE -except ImportError as e: - print("No user noxfile_config found: detail: {}".format(e)) - TEST_CONFIG_OVERRIDE = {} - -# Update the TEST_CONFIG with the user supplied values. -TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) - - -def get_pytest_env_vars(): - """Returns a dict for pytest invocation.""" - ret = {} - - # Override the GCLOUD_PROJECT and the alias. - env_key = TEST_CONFIG['gcloud_project_env'] - # This should error out if not set. - ret['GOOGLE_CLOUD_PROJECT'] = os.environ[env_key] - ret['GCLOUD_PROJECT'] = os.environ[env_key] - - # Apply user supplied envs. - ret.update(TEST_CONFIG['envs']) - return ret - - -# DO NOT EDIT - automatically generated. -# All versions used to tested samples. -ALL_VERSIONS = ["2.7", "3.6", "3.7", "3.8"] - -# Any default versions that should be ignored. -IGNORED_VERSIONS = TEST_CONFIG['ignored_versions'] - -TESTED_VERSIONS = sorted([v for v in ALL_VERSIONS if v not in IGNORED_VERSIONS]) - -INSTALL_LIBRARY_FROM_SOURCE = bool(os.environ.get("INSTALL_LIBRARY_FROM_SOURCE", False)) -# -# Style Checks -# - - -def _determine_local_import_names(start_dir): - """Determines all import names that should be considered "local". - - This is used when running the linter to insure that import order is - properly checked. - """ - file_ext_pairs = [os.path.splitext(path) for path in os.listdir(start_dir)] - return [ - basename - for basename, extension in file_ext_pairs - if extension == ".py" - or os.path.isdir(os.path.join(start_dir, basename)) - and basename not in ("__pycache__") - ] - - -# Linting with flake8. -# -# We ignore the following rules: -# E203: whitespace before ‘:’ -# E266: too many leading ‘#’ for block comment -# E501: line too long -# I202: Additional newline in a section of imports -# -# We also need to specify the rules which are ignored by default: -# ['E226', 'W504', 'E126', 'E123', 'W503', 'E24', 'E704', 'E121'] -FLAKE8_COMMON_ARGS = [ - "--show-source", - "--builtin=gettext", - "--max-complexity=20", - "--import-order-style=google", - "--exclude=.nox,.cache,env,lib,generated_pb2,*_pb2.py,*_pb2_grpc.py", - "--ignore=E121,E123,E126,E203,E226,E24,E266,E501,E704,W503,W504,I202", - "--max-line-length=88", -] - - -@nox.session -def lint(session): - session.install("flake8", "flake8-import-order") - - local_names = _determine_local_import_names(".") - args = FLAKE8_COMMON_ARGS + [ - "--application-import-names", - ",".join(local_names), - "." - ] - session.run("flake8", *args) - - -# -# Sample Tests -# - - -PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] - - -def _session_tests(session, post_install=None): - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars() - ) - - -@nox.session(python=ALL_VERSIONS) -def py(session): - """Runs py.test for a sample using the specified version of Python.""" - if session.python in TESTED_VERSIONS: - _session_tests(session) - else: - session.skip("SKIPPED: {} tests are disabled for this sample.".format( - session.python - )) - - -# -# Readmegen -# - - -def _get_repo_root(): - """ Returns the root folder of the project. """ - # Get root of this repository. Assume we don't have directories nested deeper than 10 items. - p = Path(os.getcwd()) - for i in range(10): - if p is None: - break - if Path(p / ".git").exists(): - return str(p) - p = p.parent - raise Exception("Unable to detect repository root.") - - -GENERATED_READMES = sorted([x for x in Path(".").rglob("*.rst.in")]) - - -@nox.session -@nox.parametrize("path", GENERATED_READMES) -def readmegen(session, path): - """(Re-)generates the readme for a sample.""" - session.install("jinja2", "pyyaml") - dir_ = os.path.dirname(path) - - if os.path.exists(os.path.join(dir_, "requirements.txt")): - session.install("-r", os.path.join(dir_, "requirements.txt")) - - in_file = os.path.join(dir_, "README.rst.in") - session.run( - "python", _get_repo_root() + "/scripts/readme-gen/readme_gen.py", in_file - ) From f732ab7395f77a247c19d105f6c294c49bd5ceb3 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 2 Jun 2020 22:46:24 +0000 Subject: [PATCH 4/6] chore: fix lint --- .../cloud-client/hybrid_glossaries/hybrid_tutorial_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py index 272a7ec0cdd..9d82b05d12e 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py @@ -32,8 +32,7 @@ def test_vision_standard_format(): # Generate text using Vision API text = pic_to_text('resources/standard_format.jpeg') - assert re.match("This\s?is\s?a\s?test!\s?", text) - + assert re.match(r"This\s?is\s?a\s?test!\s?", text) def test_vision_non_standard_format(): From f61d5759d57bb76394ba4e9c894534c514f7cc5f Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 2 Jun 2020 23:15:35 +0000 Subject: [PATCH 5/6] chore: add pytest.ini to stop recursion --- translate/cloud-client/pytest.ini | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 translate/cloud-client/pytest.ini diff --git a/translate/cloud-client/pytest.ini b/translate/cloud-client/pytest.ini new file mode 100644 index 00000000000..f139880676b --- /dev/null +++ b/translate/cloud-client/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +# don't recurse into hybrid_glossaries directory +norecursedirs = * \ No newline at end of file From 2e867bec99afa80611c8d986dcdde982b9fe7b5f Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 3 Jun 2020 00:57:55 +0000 Subject: [PATCH 6/6] test: use uuid for test glossary --- .../cloud-client/hybrid_glossaries/hybrid_tutorial_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py index 9d82b05d12e..f99be330b92 100644 --- a/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py +++ b/translate/cloud-client/hybrid_glossaries/hybrid_tutorial_test.py @@ -15,6 +15,7 @@ import os import re import sys +import uuid from hybrid_tutorial import create_glossary from hybrid_tutorial import pic_to_text @@ -55,7 +56,7 @@ def test_create_and_delete_glossary(): from beta_snippets import delete_glossary languages = ['fr', 'en'] - glossary_name = 'test-glossary' + glossary_name = f'test-glossary-{uuid.uuid4()}' glossary_uri = 'gs://cloud-samples-data/translation/bistro_glossary.csv' # create_glossary will raise an exception if creation fails