From ae8e207f1dee1a3fe37441bc9b5e995c1b8ffb05 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 12 Apr 2024 17:14:16 -0400 Subject: [PATCH 01/16] chore: update generation workflow --- .../generate-from-configuration.yaml | 61 ++--------------- .../hermetic_library_generation.yaml | 67 +++++++++++++++---- 2 files changed, 60 insertions(+), 68 deletions(-) diff --git a/.github/workflows/generate-from-configuration.yaml b/.github/workflows/generate-from-configuration.yaml index c1f84ea2e78b..44e05f64987c 100644 --- a/.github/workflows/generate-from-configuration.yaml +++ b/.github/workflows/generate-from-configuration.yaml @@ -6,24 +6,14 @@ on: jobs: - generate-from-configuration: + update-googleapis-commit: runs-on: ubuntu-22.04 env: - # the branch into which pull request is created. - base_branch: main # the branch with which the pull request is associated. branch_name: generate-libraries-main - library_generation_image_tag: latest - repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" steps: - uses: actions/checkout@v4 - - name: get baseline commit - id: get-baseline - shell: bash - run: | - echo "baseline_commit=$(grep googleapis_commitish generation_config.yaml | cut -d " " -f2 | xargs)" >> "$GITHUB_ENV" - - name: setup branch for pull request - id: setup-branch + - name: Setup branch for pull request shell: bash run: | [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" @@ -40,17 +30,7 @@ jobs: echo "pr_num=${pr_num}" >> "$GITHUB_ENV" env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - name: setup docker environment - shell: bash - run: | - # we create a volume pointing to `pwd` (google-cloud-java) that will - # be referenced by the container and its children - if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then - docker volume rm repo-google-cloud-java - fi - docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - - name: update googleapis commit to latest - id: update-commit + - name: Update googleapis commit to latest shell: bash run: | mkdir tmp-googleapis @@ -62,39 +42,11 @@ jobs: popd rm -rf tmp-googleapis sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" generation_config.yaml - - name: generate from configuration - shell: bash - run: | - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/generate_repo.py generate --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml --repository-path=/workspace/google-cloud-java - - name: generate pull request description - id: generate-description - shell: bash - run: | - # the pr description (body) will be available in pr-description.txt after - # running this command. - set -x - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/generate_pr_description.py generate --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml --baseline-commit="${baseline_commit}" - env: - baseline_commit: ${{ env.baseline_commit }} - name: create or update the pull request shell: bash run: | - title="chore: generate libraries at $(date)" - git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt generation_config.yaml + title="chore: update googleapis commit at $(date)" + git add generation_config.yaml # use --allow-empty because (rarely) there's no change. git commit --allow-empty -m "${title}" if [ -z "${pr_num}" ]; then @@ -102,11 +54,8 @@ jobs: git fetch -q --unshallow monorepo git push -f monorepo "${branch_name}" set -x - gh pr create --base "${base_branch}" --title "${title}" --head "${branch_name}" --body "$(cat pr_description.txt)" else git push - gh pr edit "${pr_num}" --title "${title}" - gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" fi env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 7d71749ec671..2a0eb467d831 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -1,10 +1,8 @@ name: Hermetic library generation upon generation config change through pull requests on: pull_request: - types: - - synchronize paths: - - generation_config.yaml + - "generation_config.yaml" jobs: library_generation: runs-on: ubuntu-latest @@ -14,14 +12,59 @@ jobs: steps: - uses: actions/checkout@v4 with: - ref: ${{ github.head_ref }} - - name: get baseline generation config - uses: actions/checkout@v4 - with: - ref: ${{ github.base_ref }} - path: baseline - sparse-checkout: generation_config.yaml - - name: diff + fetch-depth: 0 + - name: Checkout the target branch + shell: bash + run: | + git checkout ${{ github.base_ref }} + - name: Checkout the current branch + shell: bash + run: | + git checkout ${{ github.head_ref }} + - name: Get baseline generation config from target branch + shell: bash + run: | + git show ${{ github.base_ref }}:generation_config.yaml > baseline_generation_config.yaml + - name: Show configuration diff + shell: bash + run: | + diff generation_config.yaml baseline_generation_config.yaml || echo "config diff" + - name: Setup docker environment + shell: bash + run: | + set -x + # we create a volume pointing to `pwd` (google-cloud-java) that will + # be referenced by the container and its children + if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then + docker volume rm repo-google-cloud-java + fi + docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" + - name: Generate changed libraries + shell: bash + run: | + docker run --rm \ + ${repo_volumes} \ + -v /tmp:/tmp \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e "RUNNING_IN_DOCKER=true" \ + -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ + python /src/cli/entry_point.py generate \ + --baseline-generation-config=/workspace/google-cloud-java/baseline_generation_config.yaml \ + --current-generation-config=/workspace/google-cloud-java/generation_config.yaml \ + --repository-path=/workspace/google-cloud-java + - name: Push commit to the pull request shell: bash run: | - diff generation_config.yaml baseline/generation_config.yaml + [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot" + git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt generation_config.yaml + message="chore: generate libraries at $(date)" + git commit --allow-empty -m "${message}" + git push + if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H ${{ github.head_ref }} -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" + fi + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From b20b0c5305163e12aed0ff989c9847176abc1dca Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 16 Apr 2024 08:37:29 -0400 Subject: [PATCH 02/16] update workflow --- .../hermetic_library_generation.yaml | 22 +++++++++---------- ...ion.yaml => update-googleapis-commit.yaml} | 6 ++--- 2 files changed, 14 insertions(+), 14 deletions(-) rename .github/workflows/{generate-from-configuration.yaml => update-googleapis-commit.yaml} (94%) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 2a0eb467d831..79dbb8fe4d24 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -2,7 +2,7 @@ name: Hermetic library generation upon generation config change through pull req on: pull_request: paths: - - "generation_config.yaml" + - "generation_config.yaml" jobs: library_generation: runs-on: ubuntu-latest @@ -13,23 +13,23 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Checkout the target branch + - name: checkout the target branch shell: bash run: | git checkout ${{ github.base_ref }} - - name: Checkout the current branch + - name: checkout the current branch shell: bash run: | git checkout ${{ github.head_ref }} - - name: Get baseline generation config from target branch + - name: get baseline generation config shell: bash run: | git show ${{ github.base_ref }}:generation_config.yaml > baseline_generation_config.yaml - - name: Show configuration diff + - name: show configuration diff shell: bash run: | diff generation_config.yaml baseline_generation_config.yaml || echo "config diff" - - name: Setup docker environment + - name: create a docker volume shell: bash run: | set -x @@ -39,7 +39,7 @@ jobs: docker volume rm repo-google-cloud-java fi docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - - name: Generate changed libraries + - name: generate changed libraries shell: bash run: | docker run --rm \ @@ -50,10 +50,10 @@ jobs: -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ python /src/cli/entry_point.py generate \ - --baseline-generation-config=/workspace/google-cloud-java/baseline_generation_config.yaml \ - --current-generation-config=/workspace/google-cloud-java/generation_config.yaml \ + --baseline-generation-config-path=/workspace/google-cloud-java/baseline_generation_config.yaml \ + --current-generation-config-path=/workspace/google-cloud-java/generation_config.yaml \ --repository-path=/workspace/google-cloud-java - - name: Push commit to the pull request + - name: push commit to the pull request shell: bash run: | [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" @@ -61,7 +61,7 @@ jobs: git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt generation_config.yaml message="chore: generate libraries at $(date)" git commit --allow-empty -m "${message}" - git push + git push -f if [[ -f "pr_description.txt" ]]; then pr_num=$(gh pr list -s open -H ${{ github.head_ref }} -q . --json number | jq ".[] | .number") gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" diff --git a/.github/workflows/generate-from-configuration.yaml b/.github/workflows/update-googleapis-commit.yaml similarity index 94% rename from .github/workflows/generate-from-configuration.yaml rename to .github/workflows/update-googleapis-commit.yaml index 44e05f64987c..279a762c1a8a 100644 --- a/.github/workflows/generate-from-configuration.yaml +++ b/.github/workflows/update-googleapis-commit.yaml @@ -1,7 +1,7 @@ -name: Generate GAPIC libraries from configuration +name: Update googleapis commit on: schedule: - - cron: '0 2 * * *' # nightly at 2 am UTC + - cron: '* 2 * * *' workflow_dispatch: @@ -53,7 +53,7 @@ jobs: git remote add monorepo https://cloud-java-bot:${GH_TOKEN}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo git push -f monorepo "${branch_name}" - set -x + gh pr create --title "${title}" --head "${branch_name}" --body "${title}" else git push fi From 8cef63a3b8e47686b076a1659e555ab77874d7be Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 16 Apr 2024 08:42:45 -0400 Subject: [PATCH 03/16] update new client creation workflow --- .../generate_new_client_hermetic_build.yaml | 48 +------------------ 1 file changed, 2 insertions(+), 46 deletions(-) diff --git a/.github/workflows/generate_new_client_hermetic_build.yaml b/.github/workflows/generate_new_client_hermetic_build.yaml index 493fb7b93964..8e59983346cb 100644 --- a/.github/workflows/generate_new_client_hermetic_build.yaml +++ b/.github/workflows/generate_new_client_hermetic_build.yaml @@ -1,7 +1,7 @@ name: Generate new GAPIC client library (Hermetic Build) on: workflow_dispatch: - # some inputs are ommited due to limit of 10 input arguments + # some inputs are emitted due to limit of 10 input arguments inputs: api_shortname: required: true @@ -53,7 +53,7 @@ jobs: generate: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-python@v4 with: python-version: '3.9' @@ -78,36 +78,6 @@ jobs: API_DESCRIPTION: ${{ github.event.inputs.api_description }} LIBRARY_NAME: ${{ github.event.inputs.library_name }} DISTRIBUTION_NAME: ${{ github.event.inputs.distribution_name }} - - name: setup docker environment - shell: bash - run: | - set -x - # we create a volume pointing to `pwd` (google-cloud-java) that will - # be referenced by the container and its children - if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then - docker volume rm repo-google-cloud-java - fi - docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - - name: generate from configuration - id: generation - shell: bash - run: | - set -x - repo_volumes="-v repo-google-cloud-java:/workspace/google-cloud-java" - echo "::set-output name=repo_volumes::${repo_volumes}" - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:latest \ - python /src/generate_repo.py generate \ - --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \ - --repository-path=/workspace/google-cloud-java \ - --target-library-names=${API_SHORTNAME} - env: - API_SHORTNAME: ${{ github.event.inputs.api_shortname }} - name: Push to branch and create PR run: | set -x @@ -133,19 +103,6 @@ jobs: \`\`\` python generation/new_client_hermetic_build/add-new-client-config.py add-new-client ${GENERATION_ARGUMENTS} - - docker run --rm \\ - ${DOCKER_VOLUMES} \\ - -v /tmp:/tmp \\ - -v /var/run/docker.sock:/var/run/docker.sock \\ - -e \"RUNNING_IN_DOCKER=true\" \\ - -e \"REPO_BINDING_VOLUMES=${DOCKER_VOLUMES}\" \\ - gcr.io/cloud-devrel-public-resources/java-library-generation:latest \\ - python /src/generate_repo.py generate \\ - --generation-config-yaml=/workspace/google-cloud-java/generation_config.yaml \\ - --repository-path=/workspace/google-cloud-java \\ - --target-library-names=${API_SHORTNAME} - \`\`\`" gh pr create --title "${commit_message}" --label "owlbot:run" --head "${branch_name}" --body "${pr_body}" env: @@ -154,4 +111,3 @@ jobs: GENERATION_ARGUMENTS: ${{ steps.config_generation.outputs.new_library_args }} DOCKER_VOLUMES: ${{ steps.generation.outputs.repo_volumes }} GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - From 018be46b75de9512b84dd81edfa4c530bc983ffd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 16 Apr 2024 09:39:45 -0400 Subject: [PATCH 04/16] add base branch --- .github/workflows/update-googleapis-commit.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/update-googleapis-commit.yaml b/.github/workflows/update-googleapis-commit.yaml index 279a762c1a8a..956009e43395 100644 --- a/.github/workflows/update-googleapis-commit.yaml +++ b/.github/workflows/update-googleapis-commit.yaml @@ -9,8 +9,8 @@ jobs: update-googleapis-commit: runs-on: ubuntu-22.04 env: - # the branch with which the pull request is associated. - branch_name: generate-libraries-main + # the branch into which the pull request is merged + base_branch: main steps: - uses: actions/checkout@v4 - name: Setup branch for pull request @@ -19,11 +19,11 @@ jobs: [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot" # try to find a open pull request associated with the branch - pr_num=$(gh pr list -s open -H "${branch_name}" -q . --json number | jq ".[] | .number") + pr_num=$(gh pr list -s open -H "generate-libraries-${base_branch}" -q . --json number | jq ".[] | .number") # create a branch if there's no open pull request associated with the # branch; otherwise checkout the pull request. if [ -z "${pr_num}" ]; then - git checkout -b "${branch_name}" + git checkout -b "generate-libraries-${base_branch}" else gh pr checkout "${pr_num}" fi @@ -52,8 +52,8 @@ jobs: if [ -z "${pr_num}" ]; then git remote add monorepo https://cloud-java-bot:${GH_TOKEN}@github.com/${{ github.repository }}.git git fetch -q --unshallow monorepo - git push -f monorepo "${branch_name}" - gh pr create --title "${title}" --head "${branch_name}" --body "${title}" + git push -f monorepo "generate-libraries-${base_branch}" + gh pr create --title "${title}" --head "generate-libraries-${base_branch}" --body "${title}" --base "${base_branch}" else git push fi From 932d1a199dfbcda2b6b05d76323c06cb24ed7c04 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Wed, 17 Apr 2024 20:20:17 -0400 Subject: [PATCH 05/16] use shell script --- .../hermetic_library_generation.yaml | 57 ++-------- generation/hermetic_library_generation.sh | 100 ++++++++++++++++++ 2 files changed, 106 insertions(+), 51 deletions(-) create mode 100755 generation/hermetic_library_generation.sh diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 79dbb8fe4d24..32df3f50a8d2 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -3,68 +3,23 @@ on: pull_request: paths: - "generation_config.yaml" + jobs: library_generation: runs-on: ubuntu-latest env: library_generation_image_tag: latest - repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: checkout the target branch - shell: bash - run: | - git checkout ${{ github.base_ref }} - - name: checkout the current branch - shell: bash - run: | - git checkout ${{ github.head_ref }} - - name: get baseline generation config - shell: bash - run: | - git show ${{ github.base_ref }}:generation_config.yaml > baseline_generation_config.yaml - - name: show configuration diff - shell: bash - run: | - diff generation_config.yaml baseline_generation_config.yaml || echo "config diff" - - name: create a docker volume + - name: Generate changed libraries shell: bash run: | set -x - # we create a volume pointing to `pwd` (google-cloud-java) that will - # be referenced by the container and its children - if [[ $(docker volume inspect repo-google-cloud-java) != '[]' ]]; then - docker volume rm repo-google-cloud-java - fi - docker volume create --name "repo-google-cloud-java" --opt "type=none" --opt "device=$(pwd)" --opt "o=bind" - - name: generate changed libraries - shell: bash - run: | - docker run --rm \ - ${repo_volumes} \ - -v /tmp:/tmp \ - -v /var/run/docker.sock:/var/run/docker.sock \ - -e "RUNNING_IN_DOCKER=true" \ - -e "REPO_BINDING_VOLUMES=${repo_volumes}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${library_generation_image_tag}" \ - python /src/cli/entry_point.py generate \ - --baseline-generation-config-path=/workspace/google-cloud-java/baseline_generation_config.yaml \ - --current-generation-config-path=/workspace/google-cloud-java/generation_config.yaml \ - --repository-path=/workspace/google-cloud-java - - name: push commit to the pull request - shell: bash - run: | - [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot" - git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt generation_config.yaml - message="chore: generate libraries at $(date)" - git commit --allow-empty -m "${message}" - git push -f - if [[ -f "pr_description.txt" ]]; then - pr_num=$(gh pr list -s open -H ${{ github.head_ref }} -q . --json number | jq ".[] | .number") - gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" - fi + bash generation/hermetic_library_generation.sh \ + --target_branch ${{ github.base_ref }} \ + --current_branch ${{ github.head_ref }} \ + --image_tag "${library_generation_image_tag}" env: GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh new file mode 100755 index 000000000000..14c5380661c3 --- /dev/null +++ b/generation/hermetic_library_generation.sh @@ -0,0 +1,100 @@ +#!/bin/bash +# This script should be run at the root directory of google-cloud-java +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 2. generate changed libraries using library_generation image; +# 3. commit the change to the pull request +# 4. edit the pr body with generated pull request description, if applicable. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --image_tag) + image_tag="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${image_tag}" ]; then + echo "missing required argument --image_tag" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Use default generation config: ${generation_config}" +fi + +volume_name="repo-google-cloud-java" +repo_volumes="${volume_name}:/workspace/google-cloud-java" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +diff "${generation_config}" "${baseline_generation_config}" || echo "config diff" +# bind docker volume to include google-cloud-java in docker running environment. +if [[ $(docker volume inspect ${volume_name}) != '[]' ]]; then + docker volume rm ${volume_name} +fi +docker volume create \ + --name ${volume_name} \ + --opt "type=none" \ + --opt "device=$(pwd)" \ + --opt "o=bind" +# run hermetic code generation docker image. +docker run \ + --rm \ + -v "${repo_volumes}" \ + -v /tmp:/tmp \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -e "RUNNING_IN_DOCKER=true" \ + -e "REPO_BINDING_VOLUMES=-v ${repo_volumes}" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + python /src/cli/entry_point.py generate \ + --baseline-generation-config-path=/workspace/google-cloud-java/"${baseline_generation_config}" \ + --current-generation-config-path=/workspace/google-cloud-java/"${generation_config}" \ + --repository-path=/workspace/google-cloud-java +# commit the change to the pull request. +[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" +[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" +git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt "${generation_config}" +git commit --allow-empty -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi From 8a4293eee344fa117a11ee7e4e0451dd9b648107 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 18 Apr 2024 13:12:36 -0400 Subject: [PATCH 06/16] generalize repo name --- generation/hermetic_library_generation.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index 14c5380661c3..dd84cb038234 100755 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -55,8 +55,9 @@ if [ -z "${generation_config}" ]; then echo "Use default generation config: ${generation_config}" fi -volume_name="repo-google-cloud-java" -repo_volumes="${volume_name}:/workspace/google-cloud-java" +volume_name="repo" +workspace_name="/workspace/repo" +repo_volumes="${volume_name}:${workspace_name}" baseline_generation_config="baseline_generation_config.yaml" message="chore: generate libraries at $(date)" @@ -84,9 +85,9 @@ docker run \ -e "REPO_BINDING_VOLUMES=-v ${repo_volumes}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ python /src/cli/entry_point.py generate \ - --baseline-generation-config-path=/workspace/google-cloud-java/"${baseline_generation_config}" \ - --current-generation-config-path=/workspace/google-cloud-java/"${generation_config}" \ - --repository-path=/workspace/google-cloud-java + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" \ + --repository-path="${workspace_name}" # commit the change to the pull request. [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" From 064fff20b3a8f5169862b641fa0bafaea23af066 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 10:38:54 -0400 Subject: [PATCH 07/16] add script to update googleapis commit --- generation/update_googleapis_commit.sh | 94 ++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 generation/update_googleapis_commit.sh diff --git a/generation/update_googleapis_commit.sh b/generation/update_googleapis_commit.sh new file mode 100644 index 000000000000..26c6a8ff802e --- /dev/null +++ b/generation/update_googleapis_commit.sh @@ -0,0 +1,94 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis commit to latest in generation +# configuration at the time of running and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh + +# The pull request is created by cloud-java-bot@google.com, make sure you +# have set the token of this account via environment variable (GH_TOKEN) +# before running the script locally. + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: update googleapis commit at $(date)" + +[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" +[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}" + + +git add "${generation_config}" +# use --allow-empty because (rarely) there's no change. +git commit --allow-empty -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push +fi From 9ffd66d0edd872777f10d4e3233360b7b517892c Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 10:39:28 -0400 Subject: [PATCH 08/16] update comments --- generation/hermetic_library_generation.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index dd84cb038234..e5a8fafec66a 100755 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -1,5 +1,5 @@ #!/bin/bash -# This script should be run at the root directory of google-cloud-java +# This script should be run at the root of the repository. # This script is used to, when a pull request changes the generation # configuration (generation_config.yaml by default): # 1. compare generation configurations in the current branch (with which the @@ -8,6 +8,18 @@ # 2. generate changed libraries using library_generation image; # 3. commit the change to the pull request # 4. edit the pr body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. while [[ $# -gt 0 ]]; do key="$1" case "${key}" in From 19c791241715547d539a1e2142a40e0932b2dba4 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 10:40:46 -0400 Subject: [PATCH 09/16] update workflow file --- .../workflows/update-googleapis-commit.yaml | 62 ------------------- .../workflows/update_googleapis_commit.yaml | 24 +++++++ 2 files changed, 24 insertions(+), 62 deletions(-) delete mode 100644 .github/workflows/update-googleapis-commit.yaml create mode 100644 .github/workflows/update_googleapis_commit.yaml diff --git a/.github/workflows/update-googleapis-commit.yaml b/.github/workflows/update-googleapis-commit.yaml deleted file mode 100644 index 956009e43395..000000000000 --- a/.github/workflows/update-googleapis-commit.yaml +++ /dev/null @@ -1,62 +0,0 @@ -name: Update googleapis commit -on: - schedule: - - cron: '* 2 * * *' - workflow_dispatch: - - -jobs: - update-googleapis-commit: - runs-on: ubuntu-22.04 - env: - # the branch into which the pull request is merged - base_branch: main - steps: - - uses: actions/checkout@v4 - - name: Setup branch for pull request - shell: bash - run: | - [ -z "`git config user.email`" ] && git config --global user.email "cloud-java-bot@google.com" - [ -z "`git config user.name`" ] && git config --global user.name "cloud-java-bot" - # try to find a open pull request associated with the branch - pr_num=$(gh pr list -s open -H "generate-libraries-${base_branch}" -q . --json number | jq ".[] | .number") - # create a branch if there's no open pull request associated with the - # branch; otherwise checkout the pull request. - if [ -z "${pr_num}" ]; then - git checkout -b "generate-libraries-${base_branch}" - else - gh pr checkout "${pr_num}" - fi - echo "pr_num=${pr_num}" >> "$GITHUB_ENV" - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - name: Update googleapis commit to latest - shell: bash - run: | - mkdir tmp-googleapis - # use partial clone because only commit history is needed. - git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis - pushd tmp-googleapis - git pull - latest_commit=$(git rev-parse HEAD) - popd - rm -rf tmp-googleapis - sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" generation_config.yaml - - name: create or update the pull request - shell: bash - run: | - title="chore: update googleapis commit at $(date)" - git add generation_config.yaml - # use --allow-empty because (rarely) there's no change. - git commit --allow-empty -m "${title}" - if [ -z "${pr_num}" ]; then - git remote add monorepo https://cloud-java-bot:${GH_TOKEN}@github.com/${{ github.repository }}.git - git fetch -q --unshallow monorepo - git push -f monorepo "generate-libraries-${base_branch}" - gh pr create --title "${title}" --head "generate-libraries-${base_branch}" --body "${title}" --base "${base_branch}" - else - git push - fi - env: - GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - pr_num: ${{ env.pr_num }} diff --git a/.github/workflows/update_googleapis_commit.yaml b/.github/workflows/update_googleapis_commit.yaml new file mode 100644 index 000000000000..cd2059e855a8 --- /dev/null +++ b/.github/workflows/update_googleapis_commit.yaml @@ -0,0 +1,24 @@ +name: Update googleapis commit +on: + schedule: + - cron: '* 2 * * *' + workflow_dispatch: + + +jobs: + update-googleapis-commit: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: lts-11 + steps: + - uses: actions/checkout@v4 + - name: Update googleapis commit to latest + shell: bash + run: | + set -x + bash generation/update_googleapis_commit.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} From cca96ea5e7c52a695b1e0eefc8a5f3e58d625066 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 10:42:18 -0400 Subject: [PATCH 10/16] update image tag --- .github/workflows/generated_files_sync.yaml | 2 +- .github/workflows/hermetic_library_generation.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generated_files_sync.yaml b/.github/workflows/generated_files_sync.yaml index 1c0e25ce125c..4708ed760ab2 100644 --- a/.github/workflows/generated_files_sync.yaml +++ b/.github/workflows/generated_files_sync.yaml @@ -20,7 +20,7 @@ on: pull_request: name: generation diff env: - library_generation_image_tag: latest + library_generation_image_tag: 2.39.0 repo_volumes: "-v repo-google-cloud-java:/workspace/google-cloud-java" jobs: root-pom: diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 32df3f50a8d2..d9e440d1dd84 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -8,7 +8,7 @@ jobs: library_generation: runs-on: ubuntu-latest env: - library_generation_image_tag: latest + library_generation_image_tag: 2.39.0 steps: - uses: actions/checkout@v4 with: From 11e2255cd0f43c11c93c0f30c3966af1bb285acd Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 11:02:52 -0400 Subject: [PATCH 11/16] try to fix workflow error --- generation/new_client/new-client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation/new_client/new-client.py b/generation/new_client/new-client.py index d108a61e49f2..6ff3ba091ec1 100644 --- a/generation/new_client/new-client.py +++ b/generation/new_client/new-client.py @@ -250,7 +250,7 @@ def generate( ) # In monorepo, .OwlBot.yaml needs to be in the directory of the module. - owlbot_yaml_location_from_module = ".OwlBot.yaml" + owlbot_yaml_location_from_module = ".OwlBot-hermetic.yaml" # create owlbot config templates.render( template_name="owlbot.yaml.monorepo.j2", From 314d136feeb1ca6aac19c305f48d08868ab60b5d Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 11:50:13 -0400 Subject: [PATCH 12/16] change comment --- .github/workflows/generate_new_client_hermetic_build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/generate_new_client_hermetic_build.yaml b/.github/workflows/generate_new_client_hermetic_build.yaml index 8e59983346cb..f66f597890df 100644 --- a/.github/workflows/generate_new_client_hermetic_build.yaml +++ b/.github/workflows/generate_new_client_hermetic_build.yaml @@ -1,7 +1,7 @@ name: Generate new GAPIC client library (Hermetic Build) on: workflow_dispatch: - # some inputs are emitted due to limit of 10 input arguments + # some inputs are omitted due to limit of 10 input arguments inputs: api_shortname: required: true From fc8556055f656e4c8564a6e2893e1fb2c2448cf8 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 11:52:29 -0400 Subject: [PATCH 13/16] update comment --- .github/workflows/generated_files_sync.yaml | 2 +- .../workflows/hermetic_library_generation.yaml | 15 +++++++++++++++ .github/workflows/update_googleapis_commit.yaml | 15 +++++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/generated_files_sync.yaml b/.github/workflows/generated_files_sync.yaml index 4708ed760ab2..f3a48566d93f 100644 --- a/.github/workflows/generated_files_sync.yaml +++ b/.github/workflows/generated_files_sync.yaml @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# Github action job to test core java library features on +# GitHub action job to test core java library features on # downstream client libraries before they are released. on: push: diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index d9e440d1dd84..cce5525791f4 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -1,3 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. name: Hermetic library generation upon generation config change through pull requests on: pull_request: diff --git a/.github/workflows/update_googleapis_commit.yaml b/.github/workflows/update_googleapis_commit.yaml index cd2059e855a8..3087d99564d5 100644 --- a/.github/workflows/update_googleapis_commit.yaml +++ b/.github/workflows/update_googleapis_commit.yaml @@ -1,3 +1,18 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. name: Update googleapis commit on: schedule: From 9b612c87eece5132e0b2c2dc695e14d5435e5d4e Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 15:02:38 -0400 Subject: [PATCH 14/16] refactor according to code review --- .github/workflows/hermetic_library_generation.yaml | 2 ++ .github/workflows/update_googleapis_commit.yaml | 2 ++ generation/hermetic_library_generation.sh | 4 +--- generation/update_googleapis_commit.sh | 6 ------ 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index cce5525791f4..24c666e9e05e 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -32,6 +32,8 @@ jobs: shell: bash run: | set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash generation/hermetic_library_generation.sh \ --target_branch ${{ github.base_ref }} \ --current_branch ${{ github.head_ref }} \ diff --git a/.github/workflows/update_googleapis_commit.yaml b/.github/workflows/update_googleapis_commit.yaml index 3087d99564d5..f3f31724dd4a 100644 --- a/.github/workflows/update_googleapis_commit.yaml +++ b/.github/workflows/update_googleapis_commit.yaml @@ -32,6 +32,8 @@ jobs: shell: bash run: | set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash generation/update_googleapis_commit.sh \ --base_branch "${base_branch}"\ --repo ${{ github.repository }} diff --git a/generation/hermetic_library_generation.sh b/generation/hermetic_library_generation.sh index e5a8fafec66a..5fe34e47f26d 100755 --- a/generation/hermetic_library_generation.sh +++ b/generation/hermetic_library_generation.sh @@ -78,7 +78,7 @@ git checkout "${current_branch}" # copy generation configuration from target branch to current branch. git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" diff "${generation_config}" "${baseline_generation_config}" || echo "config diff" -# bind docker volume to include google-cloud-java in docker running environment. +# bind docker volume to include the repository in docker running environment. if [[ $(docker volume inspect ${volume_name}) != '[]' ]]; then docker volume rm ${volume_name} fi @@ -101,8 +101,6 @@ docker run \ --current-generation-config-path="${workspace_name}/${generation_config}" \ --repository-path="${workspace_name}" # commit the change to the pull request. -[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" -[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt "${generation_config}" git commit --allow-empty -m "${message}" git push diff --git a/generation/update_googleapis_commit.sh b/generation/update_googleapis_commit.sh index 26c6a8ff802e..365eb0e4db86 100644 --- a/generation/update_googleapis_commit.sh +++ b/generation/update_googleapis_commit.sh @@ -8,10 +8,6 @@ set -e # 1. git # 2. gh -# The pull request is created by cloud-java-bot@google.com, make sure you -# have set the token of this account via environment variable (GH_TOKEN) -# before running the script locally. - # The parameters of this script is: # 1. base_branch, the base branch of the result pull request. # 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java @@ -58,8 +54,6 @@ fi current_branch="generate-libraries-${base_branch}" title="chore: update googleapis commit at $(date)" -[ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" -[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" # try to find a open pull request associated with the branch pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") # create a branch if there's no open pull request associated with the From 3acdfdee44a1bbf6f9ce8c56910ef11403a2cbaa Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 15:04:37 -0400 Subject: [PATCH 15/16] restore change --- generation/new_client/new-client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generation/new_client/new-client.py b/generation/new_client/new-client.py index 6ff3ba091ec1..d108a61e49f2 100644 --- a/generation/new_client/new-client.py +++ b/generation/new_client/new-client.py @@ -250,7 +250,7 @@ def generate( ) # In monorepo, .OwlBot.yaml needs to be in the directory of the module. - owlbot_yaml_location_from_module = ".OwlBot-hermetic.yaml" + owlbot_yaml_location_from_module = ".OwlBot.yaml" # create owlbot config templates.render( template_name="owlbot.yaml.monorepo.j2", From 102b46cc7cba6de3075614418ac5b00da4bfc114 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Fri, 19 Apr 2024 18:20:56 -0400 Subject: [PATCH 16/16] update branch --- .github/workflows/update_googleapis_commit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_googleapis_commit.yaml b/.github/workflows/update_googleapis_commit.yaml index f3f31724dd4a..3efb1ad5ce54 100644 --- a/.github/workflows/update_googleapis_commit.yaml +++ b/.github/workflows/update_googleapis_commit.yaml @@ -25,7 +25,7 @@ jobs: runs-on: ubuntu-22.04 env: # the branch into which the pull request is merged - base_branch: lts-11 + base_branch: main steps: - uses: actions/checkout@v4 - name: Update googleapis commit to latest