diff --git a/.cloud/.gitignore b/.cloud/.gitignore new file mode 100644 index 000000000000..a4dfcdbb1d06 --- /dev/null +++ b/.cloud/.gitignore @@ -0,0 +1,7 @@ +generated.tfplan +generated.tfplan.json +generated.auto.tfvars +generated-env.sh +generated-main.tf +generated-outputs.tf +generated-variables.tf diff --git a/.cloud/.terraform.lock.hcl b/.cloud/.terraform.lock.hcl new file mode 100644 index 000000000000..a6d14c7346a4 --- /dev/null +++ b/.cloud/.terraform.lock.hcl @@ -0,0 +1,40 @@ +# This file is maintained automatically by "terraform init". +# Manual edits may be lost in future updates. + +provider "registry.terraform.io/hashicorp/google" { + version = "4.42.1" + hashes = [ + "h1:U7SsFbO2wrxPiHd4V2CR90sYXgreQj4fv+rF+I+8aeQ=", + "zh:0a4a244b2b65c4e403ae3825d610949cf98c2593a6f6894c36ccd21f5dea7b1e", + "zh:0fc5dfaa24d1c790fd17eafbe768d3f8a15b450d80cf3d7ab18f3f81d521adb6", + "zh:37fe844eac11ee61da141fdc8502ecec9f722a828754dff7795499de5981de22", + "zh:3cacce3c045b19e9ebe985d3de939f0a281cdd8b20cce0a212b48593c172a16a", + "zh:429048016c8227e63bb325e6fd78b8f9abc72cf4257b7cd7cba6d950a8601f69", + "zh:5a183bace4306b2c3e678dcaa37fa65dd2b5cbcbccf4a859611afac827fd2524", + "zh:5c2b99af20eb931321b2b95eff9dbc56ba87744d1047120beb2a62a722baa98f", + "zh:87d39b32f6448adb4e17758794c25ecebd19f1889a1634d0b57391a24908a4ef", + "zh:8c787a08ff05c67e4182b248943143bcaee8568e4d88562ff7982f711a6664cf", + "zh:9dcdda7993d7bf21e660cb521f61a72f3ec5bf5c52f7f25502e4c7648b1f885e", + "zh:b122b5a9a521ee96636e517c41c4ead0bf0446df59657b5a136c8a9081449a96", + "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", + ] +} + +provider "registry.terraform.io/hashicorp/time" { + version = "0.9.1" + hashes = [ + "h1:VxyoYYOCaJGDmLz4TruZQTSfQhvwEcMxvcKclWdnpbs=", + "zh:00a1476ecf18c735cc08e27bfa835c33f8ac8fa6fa746b01cd3bcbad8ca84f7f", + "zh:3007f8fc4a4f8614c43e8ef1d4b0c773a5de1dcac50e701d8abc9fdc8fcb6bf5", + "zh:5f79d0730fdec8cb148b277de3f00485eff3e9cf1ff47fb715b1c969e5bbd9d4", + "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3", + "zh:8c8094689a2bed4bb597d24a418bbbf846e15507f08be447d0a5acea67c2265a", + "zh:a6d9206e95d5681229429b406bc7a9ba4b2d9b67470bda7df88fa161508ace57", + "zh:aa299ec058f23ebe68976c7581017de50da6204883950de228ed9246f309e7f1", + "zh:b129f00f45fba1991db0aa954a6ba48d90f64a738629119bfb8e9a844b66e80b", + "zh:ef6cecf5f50cda971c1b215847938ced4cb4a30a18095509c068643b14030b00", + "zh:f1f46a4f6c65886d2dd27b66d92632232adc64f92145bf8403fe64d5ffa5caea", + "zh:f79d6155cda7d559c60d74883a24879a01c4d5f6fd7e8d1e3250f3cd215fb904", + "zh:fd59fa73074805c3575f08cd627eef7acda14ab6dac2c135a66e7a38d262201c", + ] +} diff --git a/.cloud/README.md b/.cloud/README.md new file mode 100644 index 000000000000..f4bd211f5297 --- /dev/null +++ b/.cloud/README.md @@ -0,0 +1,149 @@ +# Introduction + +When you experience integration test failures, they may be due to concurrent tests accessing the +same GCP project resources, required resources not existing on the GCP project, quota limits being +exceeded due to overreliance on a single project, incorrect permissions, disabled APIs, and/or +service account impersonation requirements. + +[Terraform](https://www.terraform.io/) is an "Infrastructure as Code" (IaC) tool that allows cloud +and on-prem resources to be defined in configuration files to be versioned, reused, and shared. By +using Terraform to provision our GCP projects prior to running integration tests we: + +* Ensure infrastructure provisioning consistency, +* Prevent conflicts between concurrent integration test invocations using the same GCP project by + randomizing resource names, +* Provide a managed lifecycle including the ability to destroy short-lived testing environments, +* Version the infrastructure requirements with the test source. + +# Running `google-cloud-java` Integration Tests + +When you need to troubleshoot integration tests in the google-cloud-java repository, follow one of +the supported workflows below to provision a GCP project with the required configuration. + +## Installation + +1. Download and install the latest version of Terraform: https://www.terraform.io/downloads +2. *(Optional)* Update your default shell environment variables. (For Mac users, + modify `~/.bash_profile`. + For Linux users, modify `~/.bashrc`.) + * See + also: [About Cloud Billing accounts](https://cloud.google.com/billing/docs/how-to/manage-billing-account) + * See + also: [Creating and managing Folders](https://cloud.google.com/resource-manager/docs/creating-managing-folders) + +```shell +export GOOGLE_CLOUD_FOLDER_ID="000000000000" +export GOOGLE_CLOUD_BILLING_ACCOUNT="000000-000000-000000" +export GOOGLE_CLOUD_PROJECT_PREFIX="my-project" +``` + +## Supported Workflows + +During these workflows, you may be asked to authenticate one or two times with gcloud as it +optionally creates a GCP project, and then sets up the +[application-default credentials](https://cloud.google.com/docs/authentication/application-default-credentials) +to be used by Terraform and the client libraries. + +A service account will be created, assigned `roles/owner`[^1], and used for all client library +integration tests. Your user credentials must have permission in the designated GCP project to +self-assign `roles/iam.serviceAccountTokenCreator` to impersonate the service account. + +[^1]: Basic roles like `roles/owner` are +[not recommended in production environments](https://cloud.google.com/iam/docs/understanding-roles#basic) +. + +### Workflow: Set up a GCP Project for **One or More** Client Libraries, then Test + +1. Invoke `setup.sh` with the directory names of the selected client libraries: + ```shell + $ ./.cloud/setup.sh [library][,library ...] + ``` + + Examples: + * `./.cloud/setup.sh java-accessapproval` + * `./.cloud/setup.sh java-asset,java-compute,java-container` + +2. When ready to begin testing, invoke `verify.sh`: + ```shell + $ ./.cloud/verify.sh [library][,library ...] + ``` + + Examples: + * `./.cloud/verify.sh java-accessapproval` + * `./.cloud/verify.sh java-asset,java-compute,java-container` + +Note: Invoking `verify.sh` without arguments will begin performing integration testing on all client +libraries. In this workflow, the GCP project will only have been configured the project for a subset +of client libraries, this command is unlikely to succeed unless the GCP project has been previously +resourced and configured manually. If you wish to perform integration testing on all client +libraries, use the below workflow to first provision the project for all client libraries. + +### Workflow: Set up a GCP Project for **All** Client Libraries, then Test + +1. Invoke `setup.sh` without arguments: + ```shell + $ ./.cloud/setup.sh + ``` + Every client library with a `.cloud` subdirectory will be included in the generated Terraform + root module, and the GCP project will be prepared to test all client libraries. +2. When ready to begin testing, invoke `verify.sh` with or without arguments: + ```shell + $ ./.cloud/verify.sh + ``` + The above will perform integration tests on all client libraries. + ```shell + $ ./.cloud/verify.sh [library][,library ...] + ``` + The above will only perform integration testing on the specified client libraries. + + Examples: + * `./.cloud/verify.sh` + * `./.cloud/verify.sh java-accessapproval` + * `./.cloud/verify.sh java-asset,java-compute,java-container` + +### Workflow: Set up and Test with One Command + +Use `test.sh` to perform a combination of both `setup.sh` and `verify.sh`. + +```shell +$ ./.cloud/test.sh [library][,library ...] +``` + +Examples: + +* `./.cloud/test.sh` +* `./.cloud/test.sh java-accessapproval` +* `./.cloud/test.sh java-asset,java-compute,java-container` + +### Workflow: Clean up GCP Project of Test Resources + +To clean-up (or “destroy”) all provisioned resources under Terraform’s management, +invoke `cleanup.sh`: + +```shell +$ ./.cloud/cleanup.sh +``` + +### Workflow: Add / Remove Client Libraries in an Existing Test Configuration + +Terraform will automatically manage the resources for a GCP project when regenerating the root +module to include different client libraries. No additional commands are necessary to support this +use-case. + +Example: + +```shell +$ ./.cloud/setup.sh java-bigqueryconnection +# Terraform provisions the project for java-bigqueryconnection's requirements. + +$ ./.cloud/setup.sh java-accessapproval +# Terraform simultaneously destroys java-bigqueryconnection's resources that +# are no longer being used, and provisions the project for java-accessapproval's +# requirements. +# Warning: This 'implicit removal' is prone to errors. If an error occurs, an +# explicit invocation of 'cleanup.sh' may be required. + +$ ./.cloud/setup.sh java-accessapproval,java-texttospeech +# Terraform maintains the java-accessapproval resources while also provisioning +# the project for java-texttospeech's requirements. +``` diff --git a/.cloud/cleanup.sh b/.cloud/cleanup.sh new file mode 100755 index 000000000000..5cecb1b5c88a --- /dev/null +++ b/.cloud/cleanup.sh @@ -0,0 +1,31 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$scriptDir" >/dev/null + +# Ensure GCP project environment variables are initialized. +if [[ $(terraform state list) == "" ]]; then + echo "Nothing to destroy." + exit +fi + +source ./helpers/gcloud-sync-env.sh +source ./helpers/destroy.sh +source ./helpers/gcloud-delete-project.sh + +popd >/dev/null diff --git a/.cloud/helpers/apply.sh b/.cloud/helpers/apply.sh new file mode 100755 index 000000000000..2a1afe965cd3 --- /dev/null +++ b/.cloud/helpers/apply.sh @@ -0,0 +1,21 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null || exit +terraform apply "generated.tfplan" || exit +popd >/dev/null || exit diff --git a/.cloud/helpers/common.sh b/.cloud/helpers/common.sh new file mode 100644 index 000000000000..b1f88ef1b983 --- /dev/null +++ b/.cloud/helpers/common.sh @@ -0,0 +1,71 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Find all directories starting with 'java-', sort them, then join +# with ',' as the delimiter. +function listAllModules() { + # Ensure current directory is repo root. + helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + pushd "$helperDir/../.." >/dev/null + + ls -1 -d java-* | sort | paste -s -d, - + + popd >/dev/null +} + +# Replaces '-' with '_' to get a Terraform output-friendly label +function getFriendlyOutputName() { + echo "$1" | tr '-' _ +} + +# Get the output object in JSON format for the given module. +function getOutput() { + friendlyName=$(getFriendlyOutputName "$1") + terraform output -json "$friendlyName" +} + +# Parse stdin and get the value associated with the given key. +function parseJson() { + python3 -c "import sys, json; print(json.load(sys.stdin)['$1'])" +} + +# Example use: getModuleOutput java-redis redis_network +function getModuleOutput() { + getOutput "$1" | parseJson "$2" +} + +# @returns exit code 0 if list $1 contains entry $2. +function contains() { + echo "$1" | grep -w -q "$2" +} + +# @returns a "new line"-delimited list of active terraform modules +function getActiveTerraformModules() { + terraform state list | awk -F'[/.]' '{print $2}' | uniq +} + +function getTerraformServiceAccountName() { + echo "terraform-service-account" +} + +function getTerraformServiceAccountEmail() { + if [ -z "${GOOGLE_CLOUD_PROJECT}" ]; then + echo "GOOGLE_CLOUD_PROJECT must be defined." + exit 1 + fi + echo "$(getTerraformServiceAccountName)@$GOOGLE_CLOUD_PROJECT.iam.gserviceaccount.com" +} diff --git a/.cloud/helpers/destroy.sh b/.cloud/helpers/destroy.sh new file mode 100755 index 000000000000..0e5dd06b846a --- /dev/null +++ b/.cloud/helpers/destroy.sh @@ -0,0 +1,39 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +# Execute 'predestroy.sh' scripts for any active modules +source ./helpers/common.sh +allModules=$(listAllModules) +activeModules=$(getActiveTerraformModules) +IFS=',' +for module in $allModules; do + friendlyName=$(getFriendlyOutputName "$module") + if ! contains "$activeModules" "$friendlyName"; then + continue # Skip unless active. + fi + + if [[ -f "../$module/.cloud/predestroy.sh" ]]; then + # shellcheck disable=SC1090 + source "../$module/.cloud/predestroy.sh" + fi +done + +terraform destroy -auto-approve +popd >/dev/null diff --git a/.cloud/helpers/gcloud-create-project.sh b/.cloud/helpers/gcloud-create-project.sh new file mode 100755 index 000000000000..513d65977bf7 --- /dev/null +++ b/.cloud/helpers/gcloud-create-project.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +if [ -n "${GOOGLE_CLOUD_PROJECT}" ]; then + echo "Using current GOOGLE_CLOUD_PROJECT: $GOOGLE_CLOUD_PROJECT" + return +fi + +currentProject=$(gcloud config get project) +if [ -n "${currentProject}" ]; then + echo -n "Do you want to use the current gcloud project ($currentProject)? (Y|n): " + read -r shouldUseCurrent + if [[ "$shouldUseCurrent" != n* ]] && [[ "$shouldUseCurrent" != N* ]]; then + GOOGLE_CLOUD_PROJECT=$currentProject + export GOOGLE_CLOUD_PROJECT + return + fi +fi + +echo -n "GOOGLE_CLOUD_PROJECT not set. Do you want to create a project? (Y|n): " +read -r shouldCreate +if [[ "$shouldCreate" == n* ]] || [[ "$shouldCreate" == N* ]]; then + echo "Project required. Exiting." + exit 1 +fi + +# Ensure required environment variables are set. +if [ -z "${GOOGLE_CLOUD_FOLDER_ID}" ]; then + echo -n "GOOGLE_CLOUD_FOLDER_ID not set. GCP Folder ID: " + read -r folder_id + export GOOGLE_CLOUD_FOLDER_ID="${folder_id}" +fi +if [ -z "${GOOGLE_CLOUD_BILLING_ACCOUNT}" ]; then + echo -n "GOOGLE_CLOUD_BILLING_ACCOUNT not set. GCP Billing Account ID: " + read -r billing_acct + export GOOGLE_CLOUD_BILLING_ACCOUNT="${billing_acct}" +fi +if [ -z "${GOOGLE_CLOUD_PROJECT_PREFIX}" ]; then + echo -n "GOOGLE_CLOUD_PROJECT_PREFIX not set. Prefix for New Project: " + read -r prefix + export GOOGLE_CLOUD_PROJECT_PREFIX="${prefix}" +fi + +# Provision GCP Project +projectId="${GOOGLE_CLOUD_PROJECT_PREFIX}"-"$RANDOM" +gcloud projects create --folder="$GOOGLE_CLOUD_FOLDER_ID" "$projectId" || exit +gcloud config set project "$projectId" +gcloud services enable cloudresourcemanager.googleapis.com +gcloud beta billing projects link "$projectId" --billing-account="$GOOGLE_CLOUD_BILLING_ACCOUNT" +GOOGLE_CLOUD_PROJECT=$projectId diff --git a/.cloud/helpers/gcloud-create-service-account.sh b/.cloud/helpers/gcloud-create-service-account.sh new file mode 100644 index 000000000000..12a340f2ac9a --- /dev/null +++ b/.cloud/helpers/gcloud-create-service-account.sh @@ -0,0 +1,65 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Use the project ID in gcloud set-quota-project. Clear the existing quota project directly from +# the configuration, and re-set. +gcloud config set project "$GOOGLE_CLOUD_PROJECT" +sed -i.bak '/quota_project_id/d' ~/.config/gcloud/application_default_credentials.json +gcloud auth application-default set-quota-project "$GOOGLE_CLOUD_PROJECT" + +# Assign permission for current gcloud account to impersonate a service account. +gcloud_account=$(gcloud config get account) +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="user:$gcloud_account" \ + --role="roles/iam.serviceAccountTokenCreator" >/dev/null + +# Set up service account for impersonation +source ./helpers/common.sh +service_account_name=$(getTerraformServiceAccountName) +service_account_email=$(getTerraformServiceAccountEmail) +# If it doesn't already exist, create the service account. +set +e +gcloud iam service-accounts describe "$service_account_email" &>/dev/null +if [[ $? -ne 0 ]]; then + gcloud iam service-accounts create "$service_account_name" + createdServiceAccount=true +else + createdServiceAccount=false +fi +set -e + +# Assign permissions to the service account. +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:$service_account_email" \ + --role="roles/owner" >/dev/null +gcloud projects add-iam-policy-binding "$GOOGLE_CLOUD_PROJECT" \ + --member="serviceAccount:$service_account_email" \ + --role="roles/resourcemanager.projectIamAdmin" >/dev/null + +# See https://cloud.google.com/blog/topics/developers-practitioners/using-google-cloud-service-account-impersonation-your-terraform-code +export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$service_account_email + +if $createdServiceAccount; then + echo "Waiting 2m for service account permissions to take effect... [0s elapsed]" + sleep 30 + echo "Waiting 2m for service account permissions to take effect... [30s elapsed]" + sleep 30 + echo "Waiting 2m for service account permissions to take effect... [1m0s elapsed]" + sleep 30 + echo "Waiting 2m for service account permissions to take effect... [1m30s elapsed]" + sleep 30 +fi diff --git a/.cloud/helpers/gcloud-delete-project.sh b/.cloud/helpers/gcloud-delete-project.sh new file mode 100644 index 000000000000..87f45200a857 --- /dev/null +++ b/.cloud/helpers/gcloud-delete-project.sh @@ -0,0 +1,35 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +if [ -n "${GOOGLE_CLOUD_PROJECT}" ]; then + helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + pushd "$helperDir/.." >/dev/null + + # Always verify whether or not to destroy the project. + echo -n "Delete project ($GOOGLE_CLOUD_PROJECT)? (y/N): " + read -r shouldDestroy + if [[ "$shouldDestroy" == y* ]] || [[ "$shouldDestroy" == Y* ]]; then + # Do not use service account when attempting to delete the project + unset GOOGLE_IMPERSONATE_SERVICE_ACCOUNT + gcloud projects delete "$GOOGLE_CLOUD_PROJECT" + gcloud config unset project + unset GOOGLE_CLOUD_PROJECT + rm ./generated.auto.tfvars + fi + + popd >/dev/null +fi diff --git a/.cloud/helpers/gcloud-login.sh b/.cloud/helpers/gcloud-login.sh new file mode 100755 index 000000000000..e790dbd6dc3a --- /dev/null +++ b/.cloud/helpers/gcloud-login.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Perform gcloud auth login if no current credentials are available. +if gcloud auth print-access-token &>/dev/null; then + true +else + if ! gcloud auth login; then + exit + fi +fi +if gcloud auth application-default print-access-token &>/dev/null; then + true +else + if ! gcloud auth application-default login; then + exit + fi +fi diff --git a/.cloud/helpers/gcloud-sync-env.sh b/.cloud/helpers/gcloud-sync-env.sh new file mode 100644 index 000000000000..728a32bd6396 --- /dev/null +++ b/.cloud/helpers/gcloud-sync-env.sh @@ -0,0 +1,46 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +if ! terraform output -json project_id &>/dev/null; then + if ! terraform refresh &>/dev/null; then + echo "No terraform state found." + exit 1 + fi + if ! terraform output -json project_id &>/dev/null; then + echo "No project found in terraform state." + exit 1 + fi +fi + +# Ensure the gcloud project matches Terraform's current state. +terraform_project_id=$(terraform output -raw project_id) +gcloud_project_id=$(gcloud config get project) || "" +if [[ "$terraform_project_id" != "$gcloud_project_id" ]]; then + echo -n "Do you want to make $terraform_project_id your current gcloud project? (Y/n): " + read -r shouldSwitch + if [[ "$shouldSwitch" == n* ]] || [[ "$shouldSwitch" == N* ]]; then + exit + fi + gcloud config set project "$terraform_project_id" +fi + +GOOGLE_CLOUD_PROJECT="$terraform_project_id" +export GOOGLE_CLOUD_PROJECT + +source ./helpers/common.sh +GOOGLE_IMPERSONATE_SERVICE_ACCOUNT=$(getTerraformServiceAccountEmail) +export GOOGLE_IMPERSONATE_SERVICE_ACCOUNT diff --git a/.cloud/helpers/generate-config.sh b/.cloud/helpers/generate-config.sh new file mode 100755 index 000000000000..5ca0665199bf --- /dev/null +++ b/.cloud/helpers/generate-config.sh @@ -0,0 +1,68 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +function initializeGeneratedFiles() { + cp ./helpers/generated-main.template.tf generated-main.tf + cp ./helpers/generated-outputs.template.tf generated-outputs.tf + cp ./helpers/generated-variables.template.tf generated-variables.tf +} + +function appendModule() { + friendlyName=$(getFriendlyOutputName "$1") + + # Append the given module to the generated-main.tf configuration to be + # included in the project's resources during 'terraform apply'. + echo "module \"$friendlyName\" { + source = \"./../$1/.cloud\" + inputs = local.data + depends_on = [time_sleep.for_1m_allowBaseCloudApisToFullyEnable] + }" >>generated-main.tf + + # Append the given module to the generated-output.tf file to provide + # all of this module's outputs as an object. + # See https://www.terraform.io/cli/commands/output + echo "output \"$friendlyName\" { + value = module.$friendlyName + sensitive = true + }" >>generated-outputs.tf +} + +function appendAllModules() { + # Either use given module list, or get a list of all modules in the parent directory. + if [ -n "$1" ]; then + modules=$1 + else + modules=$(listAllModules) + fi + IFS=',' + for module in $modules; do + # Only include modules with a .cloud subdirectory in the generated config. + if [ -d "../$module/.cloud" ]; then + appendModule "${module%/}" # Remove possible trailing '/' + fi + done +} + +# Ensure current directory is /.cloud +generateDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$generateDir/.." >/dev/null + +source ./helpers/common.sh +initializeGeneratedFiles +appendAllModules "$1" + +popd >/dev/null diff --git a/.cloud/helpers/generated-main.template.tf b/.cloud/helpers/generated-main.template.tf new file mode 100644 index 000000000000..a16ed1481577 --- /dev/null +++ b/.cloud/helpers/generated-main.template.tf @@ -0,0 +1,64 @@ +# Auto-generated by generate-config.sh +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +provider "google" { + project = var.project_id + region = var.region + zone = var.zone +} +locals { + # Objects with additional key-value entries may be passed as variables + # to modules needing an object with only a subset of those entries. + # So this 'data' object is a superset of key-value entries that may be + # needed, and we pass it to every module. + data = { + project_id = var.project_id + region = var.region + zone = var.zone + should_create_container_network = var.should_create_container_network + should_create_redis_network = var.should_create_redis_network + should_enable_apis_on_apply = var.should_enable_apis_on_apply + should_disable_apis_on_destroy = var.should_disable_apis_on_destroy + } +} +resource "google_project_service" "cloudresourcemanager" { + service = "cloudresourcemanager.googleapis.com" + project = local.data.project_id + count = local.data.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = local.data.should_disable_apis_on_destroy +} +resource "google_project_service" "iam" { + service = "iam.googleapis.com" + project = local.data.project_id + count = local.data.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = local.data.should_disable_apis_on_destroy +} +resource "google_project_service" "iamcredentials" { + service = "iamcredentials.googleapis.com" + project = local.data.project_id + count = local.data.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = local.data.should_disable_apis_on_destroy +} +resource "google_project_service" "serviceusage" { + service = "serviceusage.googleapis.com" + project = local.data.project_id + count = local.data.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = local.data.should_disable_apis_on_destroy +} +resource "time_sleep" "for_1m_allowBaseCloudApisToFullyEnable" { + create_duration = "1m" + depends_on = [ + google_project_service.cloudresourcemanager, + google_project_service.iam, + google_project_service.iamcredentials, + google_project_service.serviceusage + ] + triggers = { + when_project_created = local.data.project_id + } +} diff --git a/.cloud/helpers/generated-outputs.template.tf b/.cloud/helpers/generated-outputs.template.tf new file mode 100644 index 000000000000..d0e7ae9a9f97 --- /dev/null +++ b/.cloud/helpers/generated-outputs.template.tf @@ -0,0 +1,5 @@ +# Auto-generated by generate-config.sh +# See https://www.terraform.io/cli/commands/output +output "project_id" { + value = local.data.project_id +} diff --git a/.cloud/helpers/generated-variables.template.tf b/.cloud/helpers/generated-variables.template.tf new file mode 100644 index 000000000000..647c4b891b3b --- /dev/null +++ b/.cloud/helpers/generated-variables.template.tf @@ -0,0 +1,35 @@ +# Auto-generated by generate-config.sh +variable "project_id" { + type = string + description = "GCP Project ID of the project being used" +} +variable "region" { + type = string + description = "GCP region used to deploy resources" + default = "us-central1" # NOTE: Some integration tests have hardcoded this region. +} +variable "should_create_redis_network" { + type = bool + default = false + description = "If true, a Redis network will be created if needed during java-redis testing." +} +variable "should_create_container_network" { + type = bool + default = false + description = "If true, a container network will be created if needed during java-container testing." +} +variable "should_enable_apis_on_apply" { + type = bool + default = true + description = "If true, required APIs for active client libraries will be automatically enabled on apply." +} +variable "should_disable_apis_on_destroy" { + type = bool + default = false + description = "If true, any APIs enabled by Terraform during apply will be disabled on destroy." +} +variable "zone" { + type = string + description = "GCP zone used to deploy resources. Must be a zone in the chosen region." + default = "us-central1-c" +} diff --git a/.cloud/helpers/init.sh b/.cloud/helpers/init.sh new file mode 100755 index 000000000000..84050309e8f4 --- /dev/null +++ b/.cloud/helpers/init.sh @@ -0,0 +1,27 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Ensure current directory is same as script. +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +export TF_IN_AUTOMATION=true +source ./helpers/generate-config.sh "$1" +terraform fmt >/dev/null +terraform init + +popd >/dev/null diff --git a/.cloud/helpers/plan.sh b/.cloud/helpers/plan.sh new file mode 100755 index 000000000000..e6df16364edc --- /dev/null +++ b/.cloud/helpers/plan.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +# Ensure current directory is same as script. +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +# If Terraform was previously configured to work with a different project, +# then remove the previous Terraform state. +prev_project_id=$(terraform output -raw project_id &>/dev/null) +if [[ $? -eq 0 ]]; then + if [[ "$prev_project_id" != "$GOOGLE_CLOUD_PROJECT" ]]; then + if [[ -f terraform.tfstate ]]; then + rm terraform.tfstate + fi + fi +fi + +# Create generated.auto.tfvars which will be used as input values to generated-variables.tf +touch generated.auto.tfvars +echo "# Auto-generated by ./.cloud/plan.sh +project_id = \"$GOOGLE_CLOUD_PROJECT\" +" >generated.auto.tfvars + +# Either use given module list, or get a list of all modules in the parent directory. +if [ -n "$1" ]; then + modules=$1 +else + modules=$(listAllModules) +fi +echo "Planning around modules $modules" +IFS=',' +for module in $modules; do + # Only include modules with a .cloud subdirectory in the generated config. + if [ -f "../$module/.cloud/preplan.sh" ]; then + # shellcheck disable=SC1090 + source "../$module/.cloud/preplan.sh" generated.auto.tfvars + fi +done + +terraform fmt -list=false generated.auto.tfvars +terraform plan -out generated.tfplan +terraform show -json generated.tfplan >generated.tfplan.json + +popd >/dev/null diff --git a/.cloud/helpers/populate-env.sh b/.cloud/helpers/populate-env.sh new file mode 100644 index 000000000000..d29ac52563cd --- /dev/null +++ b/.cloud/helpers/populate-env.sh @@ -0,0 +1,43 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +function modifyEnvironment() { + # Set module-specific environment variables for upcoming integration test(s) + if [[ -f "../$1/.cloud/env.sh" ]]; then + # shellcheck disable=SC1090 + source "../$1/.cloud/env.sh" + fi +} + +helperDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$helperDir/.." >/dev/null + +# Invoking 'modifyEnvironment' for any active Terraform mdoule +source ./helpers/common.sh +allModules=$(listAllModules) +activeModules=$(getActiveTerraformModules) +IFS=',' +for module in $allModules; do + friendlyName=$(getFriendlyOutputName "$module") + if ! contains "$activeModules" "$friendlyName"; then + continue # Skip unless active. + fi + + modifyEnvironment "$module" +done + +popd >/dev/null diff --git a/.cloud/setup.sh b/.cloud/setup.sh new file mode 100755 index 000000000000..ea14d2785d11 --- /dev/null +++ b/.cloud/setup.sh @@ -0,0 +1,33 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +pushd "$scriptDir" >/dev/null +source ./helpers/init.sh "$1" +source ./helpers/gcloud-login.sh +source ./helpers/gcloud-create-project.sh +source ./helpers/gcloud-create-service-account.sh +source ./helpers/plan.sh "$1" +source ./helpers/apply.sh +popd >/dev/null + +# Ensure all SNAPSHOTs are available in the local mvn repository +pushd "$scriptDir/.." >/dev/null +source ./.kokoro/common.sh +install_modules +popd >/dev/null diff --git a/.cloud/test.sh b/.cloud/test.sh new file mode 100755 index 000000000000..33576e044b6a --- /dev/null +++ b/.cloud/test.sh @@ -0,0 +1,21 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail + +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +pushd "$scriptDir" >/dev/null +time (source ./setup.sh "$1" && source ./verify.sh "$1") +popd >/dev/null diff --git a/.cloud/verify.sh b/.cloud/verify.sh new file mode 100755 index 000000000000..43c489b8d859 --- /dev/null +++ b/.cloud/verify.sh @@ -0,0 +1,35 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -eo pipefail +scriptDir="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" + +# Prepare the environment +pushd "$scriptDir" >/dev/null +source ./helpers/gcloud-sync-env.sh +source ./helpers/populate-env.sh +popd >/dev/null + +# Perform integration testing using Kokoro's mvn syntax. +pushd "$scriptDir/.." >/dev/null +source ./.kokoro/common.sh +if [ -n "$1" ]; then + # If given a specific module list, only perform integration tests on those. + run_integration_tests "$1" +else + # If not given an argument list, integration test the parent project. + run_integration_tests -:google-cloud-os-login,-:google-cloud-recommender,-:google-cloud-talent +fi +popd >/dev/null diff --git a/.gitignore b/.gitignore index 49f05b1e9d8d..6b8637e01fc0 100644 --- a/.gitignore +++ b/.gitignore @@ -57,4 +57,15 @@ api_key generation/new_client/workspace # Monorepo repository generation -monorepo \ No newline at end of file +monorepo + +# Terraform +**/.cloud/.terraform/ +**/.cloud/plugins/ +**/.cloud/providers/ +**/.cloud/plugin_path +*.lock. +*.tfstate +*.tfstate.backup +*.tfstate.*.backup +*.tfstate.lock.info diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 8668c80304dc..cf6d82df7d8e 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -36,20 +36,38 @@ if [ -f "${KOKORO_GFILE_DIR}/secret_manager/java-bigqueryconnection-samples-secr source "${KOKORO_GFILE_DIR}/secret_manager/java-bigqueryconnection-samples-secrets" fi +function setup_cloud() { + gcloud config set project "$GOOGLE_CLOUD_PROJECT" + time ( + terraform -version && + source ./.cloud/helpers/init.sh "$1" && + source ./.cloud/helpers/plan.sh "$1" && + source ./.cloud/helpers/apply.sh && + source ./.cloud/helpers/populate-env.sh + ) + + destroy() { + arguments=$? + time source ./.cloud/helpers/destroy.sh + exit $arguments + } + trap destroy EXIT +} + RETURN_CODE=0 case ${JOB_TYPE} in test) retry_with_backoff 3 10 \ mvn -B -ntp \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dcheckstyle.skip=true \ - -Dflatten.skip=true \ - -Danimal.sniffer.skip=true \ - -Dmaven.wagon.http.retryHandler.count=5 \ - -T 1C \ - test + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true \ + -Dmaven.wagon.http.retryHandler.count=5 \ + -T 1C \ + test RETURN_CODE=$? echo "Finished running unit tests" ;; @@ -61,26 +79,21 @@ case ${JOB_TYPE} in echo "${modified_module_list[*]}" ) install_modules - printf "Running Integration Tests for:\n%s\n" "${module_list}" - mvn -B ${INTEGRATION_TEST_ARGS} \ - -pl "${module_list}" \ - -amd \ - -ntp \ - -Penable-integration-tests \ - -DtrimStackTrace=false \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dcheckstyle.skip=true \ - -Dflatten.skip=true \ - -Danimal.sniffer.skip=true \ - -Djacoco.skip=true \ - -DskipUnitTests=true \ - -Dmaven.wagon.http.retryHandler.count=5 \ - -fae \ - -T 1C \ - verify - RETURN_CODE=$? - printf "Finished Integration Tests for:\n%s\n" "${module_list}" + run_integration_tests "$module_list" + else + echo "No Integration Tests to run" + fi + ;; + terraform-integration) + generate_modified_modules_list + if [[ ${#modified_module_list[@]} -gt 0 ]]; then + module_list=$( + IFS=, + echo "${modified_module_list[*]}" + ) + setup_cloud "$module_list" + install_modules + run_integration_tests "$module_list" else echo "No Integration Tests to run" fi diff --git a/.kokoro/common.sh b/.kokoro/common.sh index 1b3f38dcc6b7..f175beaf981e 100644 --- a/.kokoro/common.sh +++ b/.kokoro/common.sh @@ -27,6 +27,7 @@ function retry_with_backoff { # allow a failures to continue set +e + unset IFS ${command} exit_code=$? @@ -94,8 +95,33 @@ function generate_modified_modules_list() { fi } +function run_integration_tests() { + printf "Running Integration Tests for:\n%s\n" "$1" + mvn -B ${INTEGRATION_TEST_ARGS} \ + -pl "$1" \ + -amd \ + -ntp \ + -Penable-integration-tests \ + -DtrimStackTrace=false \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dcheckstyle.skip=true \ + -Dflatten.skip=true \ + -Danimal.sniffer.skip=true \ + -Djacoco.skip=true \ + -DskipUnitTests=true \ + -Dmaven.wagon.http.retryHandler.count=5 \ + -fae \ + -T 1C \ + verify + + RETURN_CODE=$? + printf "Finished Integration Tests for:\n%s\n" "$1" +} + function run_graalvm_tests() { printf "Running GraalVM ITs on:\n%s\n" "${module_list[*]}" + mvn -B ${INTEGRATION_TEST_ARGS} \ -pl "${module_list}" \ -amd \ @@ -109,6 +135,7 @@ function run_graalvm_tests() { -Pnative \ -fae \ test + RETURN_CODE=$? printf "Finished Unit and Integration Tests for GraalVM:\n%s\n" "${module_list}" } diff --git a/.kokoro/nightly/graalvm-native-17.cfg b/.kokoro/nightly/graalvm-native-17.cfg index b6a45882dc6e..5a2428033573 100644 --- a/.kokoro/nightly/graalvm-native-17.cfg +++ b/.kokoro/nightly/graalvm-native-17.cfg @@ -14,20 +14,20 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } \ No newline at end of file diff --git a/.kokoro/nightly/graalvm-native.cfg b/.kokoro/nightly/graalvm-native.cfg index 40cfa131b7b1..916c79c0836b 100644 --- a/.kokoro/nightly/graalvm-native.cfg +++ b/.kokoro/nightly/graalvm-native.cfg @@ -14,20 +14,20 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } \ No newline at end of file diff --git a/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg b/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg index ed7c401700ed..b0a0134f838b 100644 --- a/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg +++ b/.kokoro/nightly/graalvm-sub-jobs/native-17/common.cfg @@ -35,22 +35,22 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } env_vars: { diff --git a/.kokoro/nightly/graalvm-sub-jobs/native/common.cfg b/.kokoro/nightly/graalvm-sub-jobs/native/common.cfg index 91beba22d79d..34d489376b35 100644 --- a/.kokoro/nightly/graalvm-sub-jobs/native/common.cfg +++ b/.kokoro/nightly/graalvm-sub-jobs/native/common.cfg @@ -35,22 +35,22 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-17-aiplatform.cfg b/.kokoro/presubmit/graalvm-native-17-aiplatform.cfg index af6363393343..1100ffb1cb7d 100644 --- a/.kokoro/presubmit/graalvm-native-17-aiplatform.cfg +++ b/.kokoro/presubmit/graalvm-native-17-aiplatform.cfg @@ -14,22 +14,22 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-17-compute.cfg b/.kokoro/presubmit/graalvm-native-17-compute.cfg index 5748c3b3955d..e45bad597fbb 100644 --- a/.kokoro/presubmit/graalvm-native-17-compute.cfg +++ b/.kokoro/presubmit/graalvm-native-17-compute.cfg @@ -14,22 +14,22 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-aiplatform.cfg b/.kokoro/presubmit/graalvm-native-aiplatform.cfg index 09d38af036d0..ed73ce9ac31c 100644 --- a/.kokoro/presubmit/graalvm-native-aiplatform.cfg +++ b/.kokoro/presubmit/graalvm-native-aiplatform.cfg @@ -14,22 +14,22 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native-compute.cfg b/.kokoro/presubmit/graalvm-native-compute.cfg index 5748c3b3955d..e45bad597fbb 100644 --- a/.kokoro/presubmit/graalvm-native-compute.cfg +++ b/.kokoro/presubmit/graalvm-native-compute.cfg @@ -14,22 +14,22 @@ env_vars: { # TODO: remove this after we've migrated all tests and scripts env_vars: { key: "GCLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_CLOUD_PROJECT" - value: "gcloud-devel" + value: "cloud-java-ci-test" } env_vars: { key: "GOOGLE_APPLICATION_CREDENTIALS" - value: "secret_manager/java-it-service-account" + value: "secret_manager/cloud-java-ci-it-service-account" } env_vars: { key: "SECRET_MANAGER_KEYS" - value: "java-it-service-account, java-bigqueryconnection-samples-secrets" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" } env_vars: { diff --git a/.kokoro/presubmit/integration.cfg b/.kokoro/presubmit/integration.cfg index f2ec5b907420..5db8d5da2e90 100644 --- a/.kokoro/presubmit/integration.cfg +++ b/.kokoro/presubmit/integration.cfg @@ -30,4 +30,4 @@ env_vars: { env_vars: { key: "SECRET_MANAGER_KEYS" value: "java-it-service-account, java-bigqueryconnection-samples-secrets" -} \ No newline at end of file +} diff --git a/.kokoro/presubmit/terraform-integration.cfg b/.kokoro/presubmit/terraform-integration.cfg new file mode 100644 index 000000000000..9acae084f394 --- /dev/null +++ b/.kokoro/presubmit/terraform-integration.cfg @@ -0,0 +1,33 @@ +# Format: //devtools/kokoro/config/proto/build.proto + +# Configure the docker image for kokoro-trampoline. +env_vars: { + key: "TRAMPOLINE_IMAGE" + value: "gcr.io/cloud-devrel-kokoro-resources/java11" +} + +env_vars: { + key: "JOB_TYPE" + value: "integration" +} + +# TODO: remove this after we've migrated all tests and scripts +env_vars: { + key: "GCLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_CLOUD_PROJECT" + value: "cloud-java-ci-test" +} + +env_vars: { + key: "GOOGLE_APPLICATION_CREDENTIALS" + value: "secret_manager/cloud-java-ci-it-service-account" +} + +env_vars: { + key: "SECRET_MANAGER_KEYS" + value: "cloud-java-ci-it-service-account, java-bigqueryconnection-samples-secrets" +} diff --git a/java-accessapproval/.cloud/main.tf b/java-accessapproval/.cloud/main.tf new file mode 100644 index 000000000000..27c2557e9d75 --- /dev/null +++ b/java-accessapproval/.cloud/main.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + google = { source = "hashicorp/google" } + } +} +resource "google_project_service" "accessapproval" { + service = "accessapproval.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-accessapproval/.cloud/variables.tf b/java-accessapproval/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-accessapproval/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-asset/.cloud/main.tf b/java-asset/.cloud/main.tf new file mode 100644 index 000000000000..ce02a4d57c83 --- /dev/null +++ b/java-asset/.cloud/main.tf @@ -0,0 +1,27 @@ +terraform { + required_providers { + google = { source = "hashicorp/google" } + } +} +resource "google_project_service" "cloudasset" { + service = "cloudasset.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "cloudresourcemanager" { + service = "cloudresourcemanager.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "pubsub" { + service = "pubsub.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "time_sleep" "for_90s_allowPubsubToFullyEnable" { + depends_on = [google_project_service.pubsub] + create_duration = "90s" +} diff --git a/java-asset/.cloud/variables.tf b/java-asset/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-asset/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-bigqueryconnection/.cloud/env.sh b/java-bigqueryconnection/.cloud/env.sh new file mode 100644 index 000000000000..cee32b92a214 --- /dev/null +++ b/java-bigqueryconnection/.cloud/env.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +MY_SQL_DATABASE=$(getModuleOutput java-bigqueryconnection db_name) +echo "Setting environment variable MY_SQL_DATABASE=$MY_SQL_DATABASE" +export MY_SQL_DATABASE + +MY_SQL_INSTANCE=$(getModuleOutput java-bigqueryconnection db_instance) +echo "Setting environment variable MY_SQL_INSTANCE=$MY_SQL_INSTANCE" +export MY_SQL_INSTANCE + +echo "Setting environment variable DB_PWD=" +DB_PWD=$(getModuleOutput java-bigqueryconnection db_password) +export DB_PWD + +echo "Setting environment variable DB_USER=default" +export DB_USER="default" diff --git a/java-bigqueryconnection/.cloud/main.tf b/java-bigqueryconnection/.cloud/main.tf new file mode 100644 index 000000000000..e9d7e2e1805c --- /dev/null +++ b/java-bigqueryconnection/.cloud/main.tf @@ -0,0 +1,48 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "bigquery" { + service = "bigquery.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "bigqueryconnection" { + service = "bigqueryconnection.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "sqladmin" { + service = "sqladmin.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} + +resource "random_id" "id" { + byte_length = 3 +} +locals { + db_name = lower("mysql-db-${random_id.id.hex}") +} + +module "mysql-db" { + source = "GoogleCloudPlatform/sql-db/google//modules/mysql" + version = "12.0.0" + name = local.db_name + database_version = "MYSQL_8_0" + deletion_protection = false + project_id = var.inputs.project_id + zone = var.inputs.zone + region = var.inputs.region + depends_on = [ + google_project_service.bigquery, + google_project_service.bigqueryconnection, + google_project_service.sqladmin + ] +} diff --git a/java-bigqueryconnection/.cloud/outputs.tf b/java-bigqueryconnection/.cloud/outputs.tf new file mode 100644 index 000000000000..df13828f2ad4 --- /dev/null +++ b/java-bigqueryconnection/.cloud/outputs.tf @@ -0,0 +1,12 @@ +output "db_name" { + value = local.db_name +} + +output "db_instance" { + value = module.mysql-db.instance_name +} + +output "db_password" { + value = module.mysql-db.generated_user_password + sensitive = true +} diff --git a/java-bigqueryconnection/.cloud/variables.tf b/java-bigqueryconnection/.cloud/variables.tf new file mode 100644 index 000000000000..1127e1061d1e --- /dev/null +++ b/java-bigqueryconnection/.cloud/variables.tf @@ -0,0 +1,12 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + + # GCP zone and region in which to deploy the SQL database. + # Zone must be in the chosen region. + zone = string + region = string + }) +} \ No newline at end of file diff --git a/java-compute/.cloud/main.tf b/java-compute/.cloud/main.tf new file mode 100644 index 000000000000..220dde6c1ead --- /dev/null +++ b/java-compute/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "compute" { + service = "compute.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-compute/.cloud/variables.tf b/java-compute/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-compute/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-container/.cloud/env.sh b/java-container/.cloud/env.sh new file mode 100644 index 000000000000..5ac8afddf115 --- /dev/null +++ b/java-container/.cloud/env.sh @@ -0,0 +1,22 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +CONTAINER_NETWORK_NAME=$(getModuleOutput java-container container_network_name) +echo "Setting environment variable CONTAINER_NETWORK_NAME=$CONTAINER_NETWORK_NAME" +export CONTAINER_NETWORK_NAME diff --git a/java-container/.cloud/main.tf b/java-container/.cloud/main.tf new file mode 100644 index 000000000000..f8ae051c6edc --- /dev/null +++ b/java-container/.cloud/main.tf @@ -0,0 +1,35 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "compute" { + service = "compute.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "container" { + service = "container.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +locals { + container_network_name = "java-container-network" +} +resource "google_compute_network" "java_container_network" { + count = var.inputs.should_create_container_network ? 1 : 0 + name = local.container_network_name + auto_create_subnetworks = true + depends_on = [ + google_project_service.compute, + google_project_service.container + ] +} +data "google_compute_network" "existing_network" { + count = var.inputs.should_create_container_network ? 0 : 1 + name = local.container_network_name +} diff --git a/java-container/.cloud/outputs.tf b/java-container/.cloud/outputs.tf new file mode 100644 index 000000000000..cb1d61996177 --- /dev/null +++ b/java-container/.cloud/outputs.tf @@ -0,0 +1,3 @@ +output "container_network_name" { + value = local.container_network_name +} diff --git a/java-container/.cloud/predestroy.sh b/java-container/.cloud/predestroy.sh new file mode 100644 index 000000000000..9ebc4c1b1bfb --- /dev/null +++ b/java-container/.cloud/predestroy.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The resource path is from the perspective of /.cloud/generated-main.tf. +resourceToForget='module.java_container.google_compute_network.java_container_network' +if terraform state list | grep -q $resourceToForget +then + terraform state rm $resourceToForget +fi diff --git a/java-container/.cloud/preplan.sh b/java-container/.cloud/preplan.sh new file mode 100755 index 000000000000..c4b18f6dd188 --- /dev/null +++ b/java-container/.cloud/preplan.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Only create the network if it doesn't already exist. +# Initial check on compute API is required to prevent interactive prompt. +if ! gcloud services list | grep -q 'compute.googleapis.com' || + ! gcloud compute networks list | grep -q 'java-container-network' +then + echo "should_create_container_network = true" >>"$1" +fi diff --git a/java-container/.cloud/variables.tf b/java-container/.cloud/variables.tf new file mode 100644 index 000000000000..79ae8eee31f2 --- /dev/null +++ b/java-container/.cloud/variables.tf @@ -0,0 +1,9 @@ +variable "inputs" { + type = object({ + project_id = string + # If true, creates the compute network to be used for integration testing. + should_create_container_network = bool + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} diff --git a/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java b/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java index 86edf8211d18..ad3e5dc1c0c8 100644 --- a/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java +++ b/java-container/google-cloud-container/src/test/java/com/google/cloud/container/v1/it/ITSystemTest.java @@ -57,7 +57,9 @@ public class ITSystemTest { + "/zones/us-central1-a/clusters/" + CLUSTER_NAME; private static final String NODE_POOL_SEL_LINK = SELF_LINK + "/nodePools/" + NODE_POOL_NAME; - private static final String NETWORK = "java-container-network"; + private static final String CONTAINER_NETWORK_ENV_NAME = "CONTAINER_NETWORK_NAME"; + private static final String DEFAULT_NETWORK = "java-container-network"; + private static final String NETWORK = getContainerNetworkName(); private static final int INITIAL_NODE_COUNT = 1; @BeforeClass @@ -180,4 +182,12 @@ public void listNodePoolsTest() { } } } + + private static String getContainerNetworkName() { + String name = System.getenv(CONTAINER_NETWORK_ENV_NAME); + if (name == null) { + return DEFAULT_NETWORK; + } + return name; + } } diff --git a/java-datacatalog/.cloud/main.tf b/java-datacatalog/.cloud/main.tf new file mode 100644 index 000000000000..29f9deeb0667 --- /dev/null +++ b/java-datacatalog/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "datacatalog" { + service = "datacatalog.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-datacatalog/.cloud/variables.tf b/java-datacatalog/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-datacatalog/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dataflow/.cloud/main.tf b/java-dataflow/.cloud/main.tf new file mode 100644 index 000000000000..bfab8605fe19 --- /dev/null +++ b/java-dataflow/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "dialogflow" { + service = "dialogflow.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-dataflow/.cloud/variables.tf b/java-dataflow/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dataflow/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-datalabeling/.cloud/main.tf b/java-datalabeling/.cloud/main.tf new file mode 100644 index 000000000000..302655296f40 --- /dev/null +++ b/java-datalabeling/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "datalabeling" { + service = "datalabeling.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-datalabeling/.cloud/variables.tf b/java-datalabeling/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-datalabeling/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dataproc/.cloud/main.tf b/java-dataproc/.cloud/main.tf new file mode 100644 index 000000000000..746eccd88efa --- /dev/null +++ b/java-dataproc/.cloud/main.tf @@ -0,0 +1,27 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "compute" { + service = "compute.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "dataproc" { + service = "dataproc.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +data "google_compute_default_service_account" "default" { + depends_on = [google_project_service.compute, google_project_service.dataproc] +} +resource "google_project_iam_member" "dataproc_iam" { + project = var.inputs.project_id + role = "roles/dataproc.worker" + member = "serviceAccount:${data.google_compute_default_service_account.default.email}" +} diff --git a/java-dataproc/.cloud/predestroy.sh b/java-dataproc/.cloud/predestroy.sh new file mode 100644 index 000000000000..f4eaf09d416a --- /dev/null +++ b/java-dataproc/.cloud/predestroy.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The resource path is from the perspective of /.cloud/generated-main.tf. +resourceToForget='module.java_dataproc.google_project_iam_member.dataproc_iam' +if terraform state list | grep -q $resourceToForget +then + terraform state rm $resourceToForget +fi diff --git a/java-dataproc/.cloud/variables.tf b/java-dataproc/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dataproc/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dialogflow-cx/.cloud/main.tf b/java-dialogflow-cx/.cloud/main.tf new file mode 100644 index 000000000000..bfab8605fe19 --- /dev/null +++ b/java-dialogflow-cx/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "dialogflow" { + service = "dialogflow.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-dialogflow-cx/.cloud/variables.tf b/java-dialogflow-cx/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dialogflow-cx/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dialogflow/.cloud/env.sh b/java-dialogflow/.cloud/env.sh new file mode 100644 index 000000000000..c5bc5bf1903d --- /dev/null +++ b/java-dialogflow/.cloud/env.sh @@ -0,0 +1,22 @@ +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +DIALOGFLOW_AGENT=$(getModuleOutput java-dialogflow dialogflow_agent) +echo "Setting environment variable DIALOGFLOW_AGENT=$DIALOGFLOW_AGENT" +export DIALOGFLOW_AGENT diff --git a/java-dialogflow/.cloud/main.tf b/java-dialogflow/.cloud/main.tf new file mode 100644 index 000000000000..7a827eab55e3 --- /dev/null +++ b/java-dialogflow/.cloud/main.tf @@ -0,0 +1,23 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "dialogflow" { + service = "dialogflow.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +locals { + agent_display_name = "google-cloud-java-tests" +} +resource "google_dialogflow_agent" "design_time_agent" { + display_name = local.agent_display_name + default_language_code = "en" + time_zone = "America/Los_Angeles" + match_mode = "MATCH_MODE_HYBRID" + depends_on = [google_project_service.dialogflow] +} diff --git a/java-dialogflow/.cloud/outputs.tf b/java-dialogflow/.cloud/outputs.tf new file mode 100644 index 000000000000..55a93effe9f4 --- /dev/null +++ b/java-dialogflow/.cloud/outputs.tf @@ -0,0 +1,3 @@ +output "dialogflow_agent" { + value = local.agent_display_name +} diff --git a/java-dialogflow/.cloud/predestroy.sh b/java-dialogflow/.cloud/predestroy.sh new file mode 100644 index 000000000000..8d0df60dd68a --- /dev/null +++ b/java-dialogflow/.cloud/predestroy.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The resource path is from the perspective of /.cloud/generated-main.tf. +resourceToForget='module.java_dialogflow.google_dialogflow_agent.design_time_agent' +if terraform state list | grep -q $resourceToForget +then + terraform state rm $resourceToForget +fi diff --git a/java-dialogflow/.cloud/variables.tf b/java-dialogflow/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dialogflow/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java b/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java index eb0d20ca4f53..d50ef6443fcd 100644 --- a/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java +++ b/java-dialogflow/google-cloud-dialogflow/src/test/java/com/google/cloud/dialogflow/v2/it/ITSystemTest.java @@ -77,8 +77,9 @@ public class ITSystemTest { private static final ProjectName PROJECT_NAME = ProjectName.of(PROJECT_ID); private static final ProjectAgentName PROJECT_AGENT_NAME = ProjectAgentName.of(PROJECT_ID); private static final String DEFAULT_DISPLAY_NAME = "google-cloud-java-tests"; - private static final String DISPLAY_NAME = - System.getProperty("dialogflow.agent", DEFAULT_DISPLAY_NAME); + private static final String LEGACY_AGENT_ENV_NAME = "dialogflow.agent"; + private static final String AGENT_ENV_NAME = "DIALOGFLOW_AGENT"; + private static final String DISPLAY_NAME = getDialogFlowAgentDisplayName(); private static final String TIME_ZONE = "America/Los_Angeles"; private static final String DEFAULT_LANGUAGE_CODE = "en"; private static final String ENTITY_NAME = "test-entity-" + ID; @@ -300,4 +301,14 @@ public void getContextTest() { Context actualContext = contextsClient.getContext(request); assertEquals(context.getName(), actualContext.getName()); } + + private static String getDialogFlowAgentDisplayName() { + String name = System.getProperty(AGENT_ENV_NAME, System.getenv(AGENT_ENV_NAME)); + if (name != null) return name; + + name = System.getProperty(LEGACY_AGENT_ENV_NAME, System.getenv(LEGACY_AGENT_ENV_NAME)); + if (name != null) return name; + + return DEFAULT_DISPLAY_NAME; + } } diff --git a/java-dns/.cloud/main.tf b/java-dns/.cloud/main.tf new file mode 100644 index 000000000000..94a0ed08e6e0 --- /dev/null +++ b/java-dns/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "dns" { + service = "dns.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-dns/.cloud/variables.tf b/java-dns/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-dns/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-errorreporting/.cloud/main.tf b/java-errorreporting/.cloud/main.tf new file mode 100644 index 000000000000..c54a88a61e11 --- /dev/null +++ b/java-errorreporting/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "clouderrorreporting" { + service = "clouderrorreporting.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-errorreporting/.cloud/variables.tf b/java-errorreporting/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-errorreporting/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-iam-admin/.cloud/main.tf b/java-iam-admin/.cloud/main.tf new file mode 100644 index 000000000000..eaf6804585e9 --- /dev/null +++ b/java-iam-admin/.cloud/main.tf @@ -0,0 +1,24 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "iam" { + service = "iam.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "random_id" "id" { + byte_length = 3 +} +locals { + service_account_id = lower("service-account-id-${random_id.id.hex}") +} +resource "google_service_account" "service_account" { + account_id = local.service_account_id + display_name = "Service Account" + depends_on = [google_project_service.iam] +} diff --git a/java-iam-admin/.cloud/variables.tf b/java-iam-admin/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-iam-admin/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-iot/.cloud/main.tf b/java-iot/.cloud/main.tf new file mode 100644 index 000000000000..5d5daabbe34f --- /dev/null +++ b/java-iot/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "cloudiot" { + service = "cloudiot.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-iot/.cloud/variables.tf b/java-iot/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-iot/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-kms/.cloud/main.tf b/java-kms/.cloud/main.tf new file mode 100644 index 000000000000..7d8a6af9a034 --- /dev/null +++ b/java-kms/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "cloudkms" { + service = "cloudkms.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-kms/.cloud/variables.tf b/java-kms/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-kms/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-kms/google-cloud-kms/src/test/java/com/google/cloud/kms/it/ITKmsTest.java b/java-kms/google-cloud-kms/src/test/java/com/google/cloud/kms/it/ITKmsTest.java index 2a50455c39a8..96befff120b5 100644 --- a/java-kms/google-cloud-kms/src/test/java/com/google/cloud/kms/it/ITKmsTest.java +++ b/java-kms/google-cloud-kms/src/test/java/com/google/cloud/kms/it/ITKmsTest.java @@ -88,9 +88,8 @@ private static KeyRing getKeyRing( MetadataUtils.attachHeaders(kmsStub, requestParamsHeader); return stubForCreateKeyRing.createKeyRing(createKeyRingRequest); } else { - Assert.fail("Error creating or looking up key"); + throw ex; } } - return null; } } diff --git a/java-notebooks/.cloud/main.tf b/java-notebooks/.cloud/main.tf new file mode 100644 index 000000000000..95c466861cbd --- /dev/null +++ b/java-notebooks/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "notebooks" { + service = "notebooks.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-notebooks/.cloud/variables.tf b/java-notebooks/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-notebooks/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-notification/.cloud/env.sh b/java-notification/.cloud/env.sh new file mode 100644 index 000000000000..8704dc0f680d --- /dev/null +++ b/java-notification/.cloud/env.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +GOOGLE_STORAGE_SERVICE_AGENT=$(getModuleOutput java-notification storage_service_agent) +echo "Setting environment variable GOOGLE_STORAGE_SERVICE_AGENT=$GOOGLE_STORAGE_SERVICE_AGENT" +export GOOGLE_STORAGE_SERVICE_AGENT diff --git a/java-notification/.cloud/main.tf b/java-notification/.cloud/main.tf new file mode 100644 index 000000000000..fe4971ae69fa --- /dev/null +++ b/java-notification/.cloud/main.tf @@ -0,0 +1,19 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "pubsub" { + service = "pubsub.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "time_sleep" "for_90s_allowPubsubToFullyEnable" { + depends_on = [google_project_service.pubsub] + create_duration = "90s" +} +data "google_storage_project_service_account" "gcs_account" { +} diff --git a/java-notification/.cloud/outputs.tf b/java-notification/.cloud/outputs.tf new file mode 100644 index 000000000000..c6e296e76399 --- /dev/null +++ b/java-notification/.cloud/outputs.tf @@ -0,0 +1,3 @@ +output "storage_service_agent" { + value = data.google_storage_project_service_account.gcs_account.email_address +} diff --git a/java-notification/.cloud/variables.tf b/java-notification/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-notification/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-notification/README.md b/java-notification/README.md index 7a60734ab503..be35adbef6f9 100644 --- a/java-notification/README.md +++ b/java-notification/README.md @@ -22,20 +22,20 @@ If you are using Maven, add this to your pom.xml file: com.google.cloud google-cloud-notification - 0.122.9-beta + 0.123.21-beta ``` -If you are using Gradle without BOM, add this to your dependencies +If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-notification:0.122.9-beta' +implementation 'com.google.cloud:google-cloud-notification:0.123.21-beta' ``` -If you are using SBT, add this to your dependencies +If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-notification" % "0.122.9-beta" +libraryDependencies += "com.google.cloud" % "google-cloud-notification" % "0.123.21-beta" ``` ## Authentication @@ -80,7 +80,7 @@ To get help, follow the instructions in the [shared Troubleshooting document][tr ## Supported Java Versions -Java 7 or above is required for using this client. +Java 8 or above is required for using this client. Google's Java client libraries, [Google Cloud Client Libraries][cloudlibs] @@ -152,7 +152,6 @@ Apache 2.0 - See [LICENSE][license] for more information. Java Version | Status ------------ | ------ -Java 7 | [![Kokoro CI][kokoro-badge-image-1]][kokoro-badge-link-1] Java 8 | [![Kokoro CI][kokoro-badge-image-2]][kokoro-badge-link-2] Java 8 OSX | [![Kokoro CI][kokoro-badge-image-3]][kokoro-badge-link-3] Java 8 Windows | [![Kokoro CI][kokoro-badge-image-4]][kokoro-badge-link-4] @@ -161,7 +160,7 @@ Java 11 | [![Kokoro CI][kokoro-badge-image-5]][kokoro-badge-link-5] Java is a registered trademark of Oracle and/or its affiliates. [product-docs]: -[javadocs]: https://googleapis.dev/java/google-cloud-notification/latest/index.html +[javadocs]: https://cloud.google.com/java/docs/reference/google-cloud-notification/latest/history [kokoro-badge-image-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-notification/java7.svg [kokoro-badge-link-1]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-notification/java7.html [kokoro-badge-image-2]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-notification/java8.svg @@ -172,7 +171,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-4]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-notification/java8-win.html [kokoro-badge-image-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-notification/java11.svg [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-notification/java11.html -[stability-image]: https://img.shields.io/badge/stability-beta-yellow +[stability-image]: https://img.shields.io/badge/stability-preview-yellow [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-notification.svg [maven-version-link]: https://search.maven.org/search?q=g:com.google.cloud%20AND%20a:google-cloud-notification&core=gav [authentication]: https://github.com/googleapis/google-cloud-java#authentication diff --git a/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java b/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java index 44e166fef9a8..c9f0715a0311 100644 --- a/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java +++ b/java-notification/src/test/java/com/google/cloud/notification/it/ITSystemTest.java @@ -38,12 +38,14 @@ import java.security.spec.InvalidKeySpecException; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.logging.Level; import java.util.logging.Logger; import org.junit.AfterClass; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.Timeout; @@ -56,6 +58,10 @@ public class ITSystemTest { private static Storage storageService; private static final Logger log = Logger.getLogger(ITSystemTest.class.getName()); + private static final String STORAGE_SERVICE_AGENT = + Optional.ofNullable(System.getenv("GOOGLE_STORAGE_SERVICE_AGENT")) + .map(agent -> "serviceAccount:" + agent) + .orElse("allAuthenticatedUsers"); private static final String BUCKET = RemoteStorageHelper.generateBucketName(); private static final String NAME_SUFFIX = UUID.randomUUID().toString(); private static String projectId; @@ -68,8 +74,7 @@ public static void beforeClass() remoteStorageHelper = RemoteStorageHelper.create(); topicAdminClient = TopicAdminClient.create(); storageService = remoteStorageHelper.getOptions().getService(); - notificationService = - new DefaultNotificationFactory().create(remoteStorageHelper.getOptions().getService()); + notificationService = new DefaultNotificationFactory().create(storageService); storageService.create(BucketInfo.of(BUCKET)); projectId = ServiceOptions.getDefaultProjectId(); } @@ -90,6 +95,7 @@ private String formatForTest(String resourceName) { return resourceName + "-" + NAME_SUFFIX; } + @Ignore @Test public void testNotifications() { // Use Pubsub to create a Topic. @@ -99,11 +105,15 @@ public void testNotifications() { Policy policy = topicAdminClient.getIamPolicy(topic.toString()); Binding binding = - Binding.newBuilder().setRole("roles/owner").addMembers("allAuthenticatedUsers").build(); + Binding.newBuilder() + .setRole("roles/pubsub.publisher") + .addMembers(STORAGE_SERVICE_AGENT) + .build(); Policy newPolicy = topicAdminClient.setIamPolicy( topic.toString(), policy.toBuilder().addBindings(binding).build()); assertTrue(newPolicy.getBindingsList().contains(binding)); + String permissionName = "pubsub.topics.get"; List permissions = topicAdminClient diff --git a/java-os-login/.cloud/main.tf b/java-os-login/.cloud/main.tf new file mode 100644 index 000000000000..726fb8b05bb0 --- /dev/null +++ b/java-os-login/.cloud/main.tf @@ -0,0 +1,11 @@ +terraform { + required_providers { + google = { source = "hashicorp/google" } + } +} +resource "google_project_service" "oslogin" { + service = "oslogin.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-os-login/.cloud/variables.tf b/java-os-login/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-os-login/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-recommender/.cloud/main.tf b/java-recommender/.cloud/main.tf new file mode 100644 index 000000000000..d4205c07086c --- /dev/null +++ b/java-recommender/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "recommender" { + service = "recommender.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-recommender/.cloud/variables.tf b/java-recommender/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-recommender/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java b/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java index 3cabcf4c3b50..e327ee5ee64d 100644 --- a/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java +++ b/java-recommender/google-cloud-recommender/src/test/java/com/google/cloud/recommender/v1beta1/it/ITSystemTest.java @@ -32,6 +32,7 @@ import java.util.List; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; public class ITSystemTest { @@ -58,14 +59,16 @@ public void tearDown() { client.close(); } + @Ignore("Recommendations are not available for brand-new GCP projects.") @Test public void listRecommendationsTest() { ListRecommendationsRequest request = ListRecommendationsRequest.newBuilder().setParent(FORMATTED_PARENT).setFilter("").build(); List recommendations = Lists.newArrayList(client.listRecommendations(request).iterateAll()); - assertThat(recommendations.size() > 0).isTrue(); - assertThat(recommendations.contains(null)).isFalse(); + + assertThat(recommendations).isNotEmpty(); + assertThat(recommendations).doesNotContain(null); } @Test(expected = InvalidArgumentException.class) diff --git a/java-redis/.cloud/env.sh b/java-redis/.cloud/env.sh new file mode 100644 index 000000000000..43ded9b22dba --- /dev/null +++ b/java-redis/.cloud/env.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Current working directory will be /.terraform/ +source ./helpers/common.sh + +REDIS_NETWORK=$(getModuleOutput java-redis redis_network) +echo "Setting environment variable REDIS_NETWORK=$REDIS_NETWORK" +export REDIS_NETWORK diff --git a/java-redis/.cloud/main.tf b/java-redis/.cloud/main.tf new file mode 100644 index 000000000000..ce3e8e5c9611 --- /dev/null +++ b/java-redis/.cloud/main.tf @@ -0,0 +1,39 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "compute" { + service = "compute.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "redis" { + service = "redis.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +locals { + redis_vpc_id = "redis-vpc" +} +resource "google_compute_network" "redis_vpc" { + count = var.inputs.should_create_redis_network ? 1 : 0 + name = local.redis_vpc_id + depends_on = [ + google_project_service.compute, + google_project_service.redis + ] +} +resource "time_sleep" "for_2m_allowRedisVpcToFullyEnable" { + count = var.inputs.should_create_redis_network ? 1 : 0 + depends_on = [google_compute_network.redis_vpc] + create_duration = "2m" +} +data "google_compute_network" "existing_network" { + count = var.inputs.should_create_redis_network ? 0 : 1 + name = local.redis_vpc_id +} diff --git a/java-redis/.cloud/outputs.tf b/java-redis/.cloud/outputs.tf new file mode 100644 index 000000000000..abec222f7acf --- /dev/null +++ b/java-redis/.cloud/outputs.tf @@ -0,0 +1,3 @@ +output "redis_network" { + value = local.redis_vpc_id +} diff --git a/java-redis/.cloud/predestroy.sh b/java-redis/.cloud/predestroy.sh new file mode 100644 index 000000000000..ea960da0e9ba --- /dev/null +++ b/java-redis/.cloud/predestroy.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# The resource path is from the perspective of /.cloud/generated-main.tf. +resourceToForget='module.java_redis.google_compute_network.redis_vpc' +if terraform state list | grep -q $resourceToForget +then + terraform state rm $resourceToForget +fi diff --git a/java-redis/.cloud/preplan.sh b/java-redis/.cloud/preplan.sh new file mode 100755 index 000000000000..6902d0744dc6 --- /dev/null +++ b/java-redis/.cloud/preplan.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Only create the network if it doesn't already exist. +# Initial check on compute API is required to prevent interactive prompt. +if ! gcloud services list | grep -q 'compute.googleapis.com' || + ! gcloud compute networks list | grep -q 'redis-vpc' +then + echo "should_create_redis_network = true" >>"$1" +fi diff --git a/java-redis/.cloud/variables.tf b/java-redis/.cloud/variables.tf new file mode 100644 index 000000000000..e4fb8568cf83 --- /dev/null +++ b/java-redis/.cloud/variables.tf @@ -0,0 +1,9 @@ +variable "inputs" { + type = object({ + project_id = string + # If true, creates the compute network to be used for integration testing. + should_create_redis_network = bool + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} diff --git a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java index 3e67d7db1f6c..e5e42c9b5e7c 100644 --- a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java +++ b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1/it/ITSystemTest.java @@ -32,6 +32,7 @@ import java.util.Arrays; import java.util.Calendar; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -46,7 +47,8 @@ public class ITSystemTest { private static final Logger LOG = Logger.getLogger(ITSystemTest.class.getName()); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - private static final String NETWORK = System.getProperty("redis.network", "redis-vpc"); + private static final String NETWORK = + Optional.ofNullable(System.getenv("REDIS_NETWORK")).orElse("redis-vpc"); private static final String INSTANCE = "test-instance-" + UUID.randomUUID().toString().substring(0, 8); private static final String LOCATION = "us-central1"; diff --git a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java index 4da1c16662d4..e9a43b71a516 100644 --- a/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java +++ b/java-redis/google-cloud-redis/src/test/java/com/google/cloud/redis/v1beta1/it/ITSystemTest.java @@ -29,6 +29,7 @@ import com.google.protobuf.FieldMask; import java.util.Arrays; import java.util.List; +import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.logging.Logger; @@ -43,7 +44,8 @@ public class ITSystemTest { private static final Logger LOG = Logger.getLogger(ITSystemTest.class.getName()); private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId(); - private static final String NETWORK = System.getProperty("redis.network", "redis-vpc"); + private static final String NETWORK = + Optional.ofNullable(System.getenv("REDIS_NETWORK")).orElse("redis-vpc"); private static final String INSTANCE = "test-instance-" + UUID.randomUUID().toString().substring(0, 8); private static final String LOCATION = "us-central1"; diff --git a/java-resourcemanager/.cloud/main.tf b/java-resourcemanager/.cloud/main.tf new file mode 100644 index 000000000000..cf38e3e12778 --- /dev/null +++ b/java-resourcemanager/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "cloudresourcemanager" { + service = "cloudresourcemanager.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-resourcemanager/.cloud/variables.tf b/java-resourcemanager/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-resourcemanager/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-scheduler/.cloud/main.tf b/java-scheduler/.cloud/main.tf new file mode 100644 index 000000000000..87ffc36ec28a --- /dev/null +++ b/java-scheduler/.cloud/main.tf @@ -0,0 +1,36 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + time = { + source = "hashicorp/time" + } + } +} +resource "google_project_service" "pubsub" { + service = "pubsub.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "cloudscheduler" { + service = "cloudscheduler.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "google_project_service" "cloudtrace" { + service = "cloudtrace.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "time_sleep" "for_1m_allowServicesTimeToFullyEnable" { + create_duration = "1m" + depends_on = [ + google_project_service.pubsub, + google_project_service.cloudscheduler, + google_project_service.cloudtrace + ] +} diff --git a/java-scheduler/.cloud/variables.tf b/java-scheduler/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-scheduler/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java index 16552ee3a5a7..bb2321ebb57b 100644 --- a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java +++ b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1/it/ITSystemTest.java @@ -19,6 +19,7 @@ import static org.junit.Assert.assertEquals; import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.ResourceExhaustedException; import com.google.cloud.ServiceOptions; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.scheduler.v1.CloudSchedulerClient; @@ -33,6 +34,8 @@ import com.google.pubsub.v1.ProjectTopicName; import java.util.List; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -145,12 +148,36 @@ public void updateJobTest() { @Test public void runJobTest() throws Exception { RunJobRequest jobRequest = RunJobRequest.newBuilder().setName(JOB_NAME).build(); - ApiFuture job = client.runJobCallable().futureCall(jobRequest); - while (true) { - if (job.isDone()) { - assertJobDetails(job.get()); - break; + + // In a new project, the scheduler queue can take a couple of minutes to initialize + // after the first job is created before it is ready to accept tasks. + retryTwiceIfResourceExhausted( + () -> { + ApiFuture job = client.runJobCallable().futureCall(jobRequest); + while (true) { + if (job.isDone()) { + assertJobDetails(job.get()); + break; + } + } + return null; + }); + } + + private static void retryTwiceIfResourceExhausted(Callable callable) throws Exception { + retryIfResourceExhausted(callable, 2); + } + + private static void retryIfResourceExhausted(Callable callable, int retries) + throws Exception { + try { + callable.call(); + } catch (ResourceExhaustedException ex) { + if (retries == 0) { + throw ex; } + TimeUnit.SECONDS.sleep(60); + retryIfResourceExhausted(callable, retries - 1); } } } diff --git a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java index 00276324172f..a97e8132150c 100644 --- a/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java +++ b/java-scheduler/google-cloud-scheduler/src/test/java/com/google/cloud/scheduler/v1beta1/it/ITSystemTest.java @@ -18,6 +18,7 @@ import static org.junit.Assert.assertEquals; import com.google.api.core.ApiFuture; +import com.google.api.gax.rpc.ResourceExhaustedException; import com.google.cloud.ServiceOptions; import com.google.cloud.pubsub.v1.TopicAdminClient; import com.google.cloud.scheduler.v1beta1.CloudSchedulerClient; @@ -32,6 +33,8 @@ import com.google.pubsub.v1.ProjectTopicName; import java.util.List; import java.util.UUID; +import java.util.concurrent.Callable; +import java.util.concurrent.TimeUnit; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -144,12 +147,36 @@ public void updateJobTest() { @Test public void runJobTest() throws Exception { RunJobRequest jobRequest = RunJobRequest.newBuilder().setName(JOB_NAME).build(); - ApiFuture job = client.runJobCallable().futureCall(jobRequest); - while (true) { - if (job.isDone()) { - assertJobDetails(job.get()); - break; + + // In a new project, the scheduler queue can take a couple of minutes to initialize + // after the first job is created before it is ready to accept tasks. + retryTwiceIfResourceExhausted( + () -> { + ApiFuture job = client.runJobCallable().futureCall(jobRequest); + while (true) { + if (job.isDone()) { + assertJobDetails(job.get()); + break; + } + } + return null; + }); + } + + private static void retryTwiceIfResourceExhausted(Callable callable) throws Exception { + retryIfResourceExhausted(callable, 2); + } + + private static void retryIfResourceExhausted(Callable callable, int retries) + throws Exception { + try { + callable.call(); + } catch (ResourceExhaustedException ex) { + if (retries == 0) { + throw ex; } + TimeUnit.SECONDS.sleep(60); + retryIfResourceExhausted(callable, retries - 1); } } } diff --git a/java-secretmanager/.cloud/main.tf b/java-secretmanager/.cloud/main.tf new file mode 100644 index 000000000000..4a617ddd0ab5 --- /dev/null +++ b/java-secretmanager/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "secretmanager" { + service = "secretmanager.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-secretmanager/.cloud/variables.tf b/java-secretmanager/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-secretmanager/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-speech/.cloud/main.tf b/java-speech/.cloud/main.tf new file mode 100644 index 000000000000..26037d6ba9b5 --- /dev/null +++ b/java-speech/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "speech" { + service = "speech.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-speech/.cloud/variables.tf b/java-speech/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-speech/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-talent/.cloud/main.tf b/java-talent/.cloud/main.tf new file mode 100644 index 000000000000..29a63df2a5ba --- /dev/null +++ b/java-talent/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "jobs" { + service = "jobs.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-talent/.cloud/variables.tf b/java-talent/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-talent/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-tasks/.cloud/main.tf b/java-tasks/.cloud/main.tf new file mode 100644 index 000000000000..ea191c9860d4 --- /dev/null +++ b/java-tasks/.cloud/main.tf @@ -0,0 +1,17 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "cloudtasks" { + service = "cloudtasks.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "time_sleep" "for_90s_allowTasksToFullyEnable" { + depends_on = [google_project_service.cloudtasks] + create_duration = "90s" +} \ No newline at end of file diff --git a/java-tasks/.cloud/variables.tf b/java-tasks/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-tasks/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-texttospeech/.cloud/main.tf b/java-texttospeech/.cloud/main.tf new file mode 100644 index 000000000000..565546e3e931 --- /dev/null +++ b/java-texttospeech/.cloud/main.tf @@ -0,0 +1,18 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "texttospeech" { + service = "texttospeech.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} +resource "time_sleep" "for_90s_allowTextToSpeechToFullyEnable" { + depends_on = [google_project_service.texttospeech] + create_duration = "90s" +} + diff --git a/java-texttospeech/.cloud/variables.tf b/java-texttospeech/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-texttospeech/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-trace/.cloud/main.tf b/java-trace/.cloud/main.tf new file mode 100644 index 000000000000..1379de8446d8 --- /dev/null +++ b/java-trace/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "cloudtrace" { + service = "cloudtrace.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-trace/.cloud/variables.tf b/java-trace/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-trace/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-translate/.cloud/main.tf b/java-translate/.cloud/main.tf new file mode 100644 index 000000000000..8842b077ea7e --- /dev/null +++ b/java-translate/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "translate" { + service = "translate.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-translate/.cloud/variables.tf b/java-translate/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-translate/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-video-intelligence/.cloud/main.tf b/java-video-intelligence/.cloud/main.tf new file mode 100644 index 000000000000..96414e3a9539 --- /dev/null +++ b/java-video-intelligence/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "videointelligence" { + service = "videointelligence.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-video-intelligence/.cloud/variables.tf b/java-video-intelligence/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-video-intelligence/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-vision/.cloud/main.tf b/java-vision/.cloud/main.tf new file mode 100644 index 000000000000..6bf457a678fc --- /dev/null +++ b/java-vision/.cloud/main.tf @@ -0,0 +1,13 @@ +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + } +} +resource "google_project_service" "vision" { + service = "vision.googleapis.com" + project = var.inputs.project_id + count = var.inputs.should_enable_apis_on_apply ? 1 : 0 + disable_on_destroy = var.inputs.should_disable_apis_on_destroy +} diff --git a/java-vision/.cloud/variables.tf b/java-vision/.cloud/variables.tf new file mode 100644 index 000000000000..f78a8e9f648c --- /dev/null +++ b/java-vision/.cloud/variables.tf @@ -0,0 +1,7 @@ +variable "inputs" { + type = object({ + project_id = string + should_enable_apis_on_apply = bool + should_disable_apis_on_destroy = bool + }) +} \ No newline at end of file diff --git a/java-vision/google-cloud-vision/src/test/java/com/google/cloud/vision/it/ITSystemTest.java b/java-vision/google-cloud-vision/src/test/java/com/google/cloud/vision/it/ITSystemTest.java index 35837ecf52a6..ce6ca73c26d7 100644 --- a/java-vision/google-cloud-vision/src/test/java/com/google/cloud/vision/it/ITSystemTest.java +++ b/java-vision/google-cloud-vision/src/test/java/com/google/cloud/vision/it/ITSystemTest.java @@ -77,8 +77,10 @@ import java.util.Arrays; import java.util.List; import java.util.UUID; +import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -313,22 +315,33 @@ public void detectLandmarksUrlTest() throws Exception { AnnotateImageRequest request = AnnotateImageRequest.newBuilder().addFeatures(feat).setImage(img).build(); - List actual = new ArrayList<>(); - - int tryCount = 0; int maxTries = 3; + List actual = + retryIfEmptyOrStatusRuntimeException(() -> addResponsesToList(request), maxTries); + assertThat(actual).contains("Palace of Fine Arts"); + } - while (tryCount < maxTries) { - try { - actual = addResponsesToList(request); - break; - } catch (StatusRuntimeException ex) { - tryCount++; - System.out.println("retrying due to request throttling or DOS prevention..."); + private static List retryIfEmptyOrStatusRuntimeException( + Callable> callable, int retries) throws Exception { + Assert.assertTrue("No more retries available.", retries >= 0); + + try { + List result = callable.call(); + if (result.isEmpty()) { + System.out.println("Retrying due to empty response list... "); TimeUnit.SECONDS.sleep(30); + return retryIfEmptyOrStatusRuntimeException(callable, retries - 1); + } + return result; + + } catch (StatusRuntimeException ex) { + if (retries == 0) { + throw ex; } + System.out.println("Retrying due to request throttling or DOS prevention... "); + TimeUnit.SECONDS.sleep(30); + return retryIfEmptyOrStatusRuntimeException(callable, retries - 1); } - assertThat(actual).contains("Palace of Fine Arts"); } private List addResponsesToList(AnnotateImageRequest request) {