From a27aff80929c6eecd86f5752e242fcd17a789366 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Mon, 5 Jul 2021 16:34:51 +0200 Subject: [PATCH 01/13] feat: Adding pagination sample. --- samples/snippets/quickstart.py | 8 ++- samples/snippets/sample_pagination.py | 73 +++++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/sample_pagination.py diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index fb67bfab0..6877d68e9 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -73,9 +73,15 @@ def list_all_instances( iterable collections of Instance objects as values. """ instance_client = compute_v1.InstancesClient() - agg_list = instance_client.aggregated_list(project=project_id) + # Requesting a list of Instances, aggregated by the zone they are located in. + # The max_results parameter tells the API to return no more than 5 results per response page. + request = compute_v1.AggregatedListInstancesRequest(project=project_id, max_results=5) + agg_list = instance_client.aggregated_list(request=request) all_instances = {} print("Instances found:") + # Despite telling the API to return only 5 instances per page, you don't need to handle the + # pagination system yourself. The returned AggregatedListPager object handles it automatically + # for you, requesting next pages as you iterate over the results. for zone, response in agg_list: if response.instances: all_instances[zone] = response.instances diff --git a/samples/snippets/sample_pagination.py b/samples/snippets/sample_pagination.py new file mode 100644 index 000000000..4e559c38c --- /dev/null +++ b/samples/snippets/sample_pagination.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import google.cloud.compute_v1 as compute_v1 + + +def print_images_list(project: str) -> None: + """ + Prints a list of all non-deprecated image names available in given project. + + Args: + project: project ID or project number of the Cloud project you want to list images from. + + Returns: + None + """ + images_client = compute_v1.ImagesClient() + # Listing only non-deprecated images to reduce the size of the reply. + images_list_request = compute_v1.ListImagesRequest(project=project, max_results=3, + filter="deprecated.state != DEPRECATED") + + # Although the max_results parameter is specified as 3 in the request, the iterable returned + # by the `list()` method hides the pagination mechanic and allows you to simply iterate over + # all the images, while the library makes multiple requests to the API for you. + for img in images_client.list(request=images_list_request): + print(f" - {img.name}") + + +def print_images_list_by_page(project: str, page_size: int = 10) -> None: + """ + Prints a list of all non-deprecated image names available in given project, + divided into pages, as returned by the GCE API. + + Args: + project: project ID or project number of the Cloud project you want to list images from. + page_size: size of the pages you want the API to return on each call. + + Returns: + None + """ + images_client = compute_v1.ImagesClient() + # Listing only non-deprecated images to reduce the size of the reply. + images_list_request = compute_v1.ListImagesRequest(project=project, max_results=page_size, + filter="deprecated.state != DEPRECATED") + + # By using the `pages` attribute of returned iterable, you can have more granular control over + # the way you iterate over paginated results retrieved from the API. Each time you want to + # access next page, the library retrieves it from the API. + for page_num, page in enumerate(images_client.list(request=images_list_request).pages, start=1): + print(f"Page {page_num}: ") + for img in page.items: + print(f" - {img.name}") + + +if __name__ == '__main__': + print("=================== Flat list of images ===================") + print_images_list('windows-sql-cloud') + print("================= Paginated list of images ================") + print_images_list_by_page('windows-sql-cloud', 5) From 4ff53a89a874a798bc0bc7a9b610b666d462baf5 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Mon, 19 Jul 2021 14:47:49 +0200 Subject: [PATCH 02/13] feat: Adding regions and tests to pagination samples --- samples/snippets/sample_pagination.py | 9 ++++++- samples/snippets/test_sample_pagination.py | 29 ++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/test_sample_pagination.py diff --git a/samples/snippets/sample_pagination.py b/samples/snippets/sample_pagination.py index 4e559c38c..64034a7a6 100644 --- a/samples/snippets/sample_pagination.py +++ b/samples/snippets/sample_pagination.py @@ -14,10 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. - +# [START compute_images_list_page ] +# [START compute_images_list ] import google.cloud.compute_v1 as compute_v1 +# [END compute_images_list ] +# [END compute_images_list_page ] +# [START compute_images_list ] def print_images_list(project: str) -> None: """ Prints a list of all non-deprecated image names available in given project. @@ -38,8 +42,10 @@ def print_images_list(project: str) -> None: # all the images, while the library makes multiple requests to the API for you. for img in images_client.list(request=images_list_request): print(f" - {img.name}") +# [END compute_images_list ] +# [START compute_images_list_page ] def print_images_list_by_page(project: str, page_size: int = 10) -> None: """ Prints a list of all non-deprecated image names available in given project, @@ -64,6 +70,7 @@ def print_images_list_by_page(project: str, page_size: int = 10) -> None: print(f"Page {page_num}: ") for img in page.items: print(f" - {img.name}") +# [END compute_images_list_page ] if __name__ == '__main__': diff --git a/samples/snippets/test_sample_pagination.py b/samples/snippets/test_sample_pagination.py new file mode 100644 index 000000000..d2088fed0 --- /dev/null +++ b/samples/snippets/test_sample_pagination.py @@ -0,0 +1,29 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import typing + +from sample_pagination import print_images_list, print_images_list_by_page + +PROJECT = 'windows-sql-cloud' + + +def test_pagination(capsys: typing.Any) -> None: + print_images_list(PROJECT) + out, _ = capsys.readouterr() + assert(len(out.splitlines()) > 2) + +def test_pagination_page(capsys: typing.Any) -> None: + print_images_list_by_page(PROJECT, 2) + out, _ = capsys.readouterr() + assert("Page 2" in out) From c7fe34632d0b67fb1676377b9cac7a7d2874526e Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Mon, 19 Jul 2021 14:57:14 +0200 Subject: [PATCH 03/13] chore: lint fix --- samples/snippets/test_sample_pagination.py | 1 + 1 file changed, 1 insertion(+) diff --git a/samples/snippets/test_sample_pagination.py b/samples/snippets/test_sample_pagination.py index d2088fed0..c881ccd1a 100644 --- a/samples/snippets/test_sample_pagination.py +++ b/samples/snippets/test_sample_pagination.py @@ -23,6 +23,7 @@ def test_pagination(capsys: typing.Any) -> None: out, _ = capsys.readouterr() assert(len(out.splitlines()) > 2) + def test_pagination_page(capsys: typing.Any) -> None: print_images_list_by_page(PROJECT, 2) out, _ = capsys.readouterr() From 0693ec21f5944eead35a12e5966a61e192ec9420 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Thu, 22 Jul 2021 13:51:45 +0200 Subject: [PATCH 04/13] Apply suggestions from code review Co-authored-by: Jonathan L. --- samples/snippets/quickstart.py | 8 ++++---- samples/snippets/sample_pagination.py | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 6877d68e9..2e9a12128 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -74,14 +74,14 @@ def list_all_instances( """ instance_client = compute_v1.InstancesClient() # Requesting a list of Instances, aggregated by the zone they are located in. - # The max_results parameter tells the API to return no more than 5 results per response page. + # Use the `max_results` parameter to limit the number of results that the API returns per response page. request = compute_v1.AggregatedListInstancesRequest(project=project_id, max_results=5) agg_list = instance_client.aggregated_list(request=request) all_instances = {} print("Instances found:") - # Despite telling the API to return only 5 instances per page, you don't need to handle the - # pagination system yourself. The returned AggregatedListPager object handles it automatically - # for you, requesting next pages as you iterate over the results. + # Despite using the `max_results` parameter, you don't need to handle the pagination + # yourself. The returned `AggregatedListPager` object handles pagination + # automatically, requesting next pages as you iterate over the results. for zone, response in agg_list: if response.instances: all_instances[zone] = response.instances diff --git a/samples/snippets/sample_pagination.py b/samples/snippets/sample_pagination.py index 64034a7a6..7c661bd83 100644 --- a/samples/snippets/sample_pagination.py +++ b/samples/snippets/sample_pagination.py @@ -37,9 +37,9 @@ def print_images_list(project: str) -> None: images_list_request = compute_v1.ListImagesRequest(project=project, max_results=3, filter="deprecated.state != DEPRECATED") - # Although the max_results parameter is specified as 3 in the request, the iterable returned - # by the `list()` method hides the pagination mechanic and allows you to simply iterate over - # all the images, while the library makes multiple requests to the API for you. + # Although the `max_results` parameter is specified in the request, the iterable returned + # by the `list()` method hides the pagination mechanic. The library makes multiple + # requests to the API for you, so you can simply iterate over all the images. for img in images_client.list(request=images_list_request): print(f" - {img.name}") # [END compute_images_list ] @@ -48,8 +48,8 @@ def print_images_list(project: str) -> None: # [START compute_images_list_page ] def print_images_list_by_page(project: str, page_size: int = 10) -> None: """ - Prints a list of all non-deprecated image names available in given project, - divided into pages, as returned by the GCE API. + Prints a list of all non-deprecated image names available in a given project, + divided into pages as returned by the Compute Engine API. Args: project: project ID or project number of the Cloud project you want to list images from. @@ -63,9 +63,9 @@ def print_images_list_by_page(project: str, page_size: int = 10) -> None: images_list_request = compute_v1.ListImagesRequest(project=project, max_results=page_size, filter="deprecated.state != DEPRECATED") - # By using the `pages` attribute of returned iterable, you can have more granular control over - # the way you iterate over paginated results retrieved from the API. Each time you want to - # access next page, the library retrieves it from the API. + # Use the `pages` attribute of returned iterable to have more granular control of + # iteration over paginated results from the API. Each time you want to access the + # next page, the library retrieves that page from the API. for page_num, page in enumerate(images_client.list(request=images_list_request).pages, start=1): print(f"Page {page_num}: ") for img in page.items: From 62073537009d08741ed129f50d22f1e3d863f506 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Thu, 22 Jul 2021 13:56:05 +0200 Subject: [PATCH 05/13] chore: applying review comments. --- samples/snippets/quickstart.py | 1 - 1 file changed, 1 deletion(-) diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index 2e9a12128..d43c09dbc 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -73,7 +73,6 @@ def list_all_instances( iterable collections of Instance objects as values. """ instance_client = compute_v1.InstancesClient() - # Requesting a list of Instances, aggregated by the zone they are located in. # Use the `max_results` parameter to limit the number of results that the API returns per response page. request = compute_v1.AggregatedListInstancesRequest(project=project_id, max_results=5) agg_list = instance_client.aggregated_list(request=request) From d60a00c4ac4fa30ebc781703c0297aa5f5a49366 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Thu, 22 Jul 2021 14:32:16 +0200 Subject: [PATCH 06/13] fix: Make tests run in separate projects. --- samples/snippets/noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 6a8ccdae2..2776c1e51 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -49,8 +49,8 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # If you need to use a specific version of pip, # change pip_version_override to the string representation # of the version number, for example, "20.2.4" From 98ade2cb24c23e4380e29e404a3b23a66ada22d1 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Thu, 22 Jul 2021 15:07:18 +0200 Subject: [PATCH 07/13] Revert "fix: Make tests run in separate projects." This reverts commit d60a00c4ac4fa30ebc781703c0297aa5f5a49366. --- samples/snippets/noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 2776c1e51..6a8ccdae2 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -49,8 +49,8 @@ # to 'BUILD_SPECIFIC_GCLOUD_PROJECT' if you want to opt in using a # build specific Cloud project. You can also use your own string # to use your own Cloud project. - # 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', - 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', + 'gcloud_project_env': 'GOOGLE_CLOUD_PROJECT', + # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # If you need to use a specific version of pip, # change pip_version_override to the string representation # of the version number, for example, "20.2.4" From 856f244ac9334e25ba1f8581c7c0d6cad8efb892 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 23 Jul 2021 13:23:30 +0200 Subject: [PATCH 08/13] trying to figure out the noxfile_config location --- samples/snippets/noxfile_config.py => noxfile_config.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename samples/snippets/noxfile_config.py => noxfile_config.py (100%) diff --git a/samples/snippets/noxfile_config.py b/noxfile_config.py similarity index 100% rename from samples/snippets/noxfile_config.py rename to noxfile_config.py From aed9059458f3321ff30fdc9e0d9ed9d613de9d76 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 23 Jul 2021 13:32:44 +0200 Subject: [PATCH 09/13] Revert "trying to figure out the noxfile_config location" This reverts commit 856f244ac9334e25ba1f8581c7c0d6cad8efb892. --- noxfile_config.py => samples/snippets/noxfile_config.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename noxfile_config.py => samples/snippets/noxfile_config.py (100%) diff --git a/noxfile_config.py b/samples/snippets/noxfile_config.py similarity index 100% rename from noxfile_config.py rename to samples/snippets/noxfile_config.py From e2a3fb849465a8c4d170c65321070db25b0f90f7 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 23 Jul 2021 13:35:44 +0200 Subject: [PATCH 10/13] Testing nox stuff. --- samples/snippets/noxfile.py | 3 +++ samples/snippets/noxfile_config.py | 1 + 2 files changed, 4 insertions(+) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 6a8ccdae2..67a94b73b 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -71,6 +71,7 @@ # Update the TEST_CONFIG with the user supplied values. TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) +print('Using test config: ', TEST_CONFIG) def get_pytest_env_vars() -> Dict[str, str]: @@ -196,6 +197,8 @@ def _session_tests(session: nox.sessions.Session, post_install: Callable = None) if post_install: post_install(session) + print('Testing with envs: ', get_pytest_env_vars()) + session.run( "pytest", *(PYTEST_COMMON_ARGS + session.posargs), diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py index 16c4c694e..fd026b595 100644 --- a/samples/snippets/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -16,3 +16,4 @@ # Tests in test_sample_default_values.py require separate projects to not interfere with each other. 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', } +print('Noxfile_config loaded') From c26385d1ddf440ddf3f0747a66b6321cc9474df4 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Fri, 23 Jul 2021 13:59:07 +0200 Subject: [PATCH 11/13] Revert "Testing nox stuff." This reverts commit e2a3fb849465a8c4d170c65321070db25b0f90f7. --- samples/snippets/noxfile.py | 3 --- samples/snippets/noxfile_config.py | 1 - 2 files changed, 4 deletions(-) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 67a94b73b..6a8ccdae2 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -71,7 +71,6 @@ # Update the TEST_CONFIG with the user supplied values. TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) -print('Using test config: ', TEST_CONFIG) def get_pytest_env_vars() -> Dict[str, str]: @@ -197,8 +196,6 @@ def _session_tests(session: nox.sessions.Session, post_install: Callable = None) if post_install: post_install(session) - print('Testing with envs: ', get_pytest_env_vars()) - session.run( "pytest", *(PYTEST_COMMON_ARGS + session.posargs), diff --git a/samples/snippets/noxfile_config.py b/samples/snippets/noxfile_config.py index fd026b595..16c4c694e 100644 --- a/samples/snippets/noxfile_config.py +++ b/samples/snippets/noxfile_config.py @@ -16,4 +16,3 @@ # Tests in test_sample_default_values.py require separate projects to not interfere with each other. 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', } -print('Noxfile_config loaded') From 55d229be6e76516132a83e805824c79f84d1b810 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Mon, 2 Aug 2021 13:45:08 +0200 Subject: [PATCH 12/13] chore: Making the default_values tests be more stable. --- samples/snippets/test_sample_default_values.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/samples/snippets/test_sample_default_values.py b/samples/snippets/test_sample_default_values.py index b6d2f0acc..efed3196c 100644 --- a/samples/snippets/test_sample_default_values.py +++ b/samples/snippets/test_sample_default_values.py @@ -13,6 +13,7 @@ # limitations under the License. import typing import uuid +import time import google.auth import google.cloud.storage as storage @@ -37,6 +38,7 @@ def temp_bucket(): def test_set_usage_export_bucket_default(capsys: typing.Any, temp_bucket: storage.Bucket) -> None: set_usage_export_bucket(project_id=PROJECT, bucket_name=temp_bucket.name) + time.sleep(5) # To make sure the settings are properly updated uel = get_usage_export_bucket(project_id=PROJECT) assert(uel.bucket_name == temp_bucket.name) assert(uel.report_name_prefix == 'usage_gce') @@ -44,6 +46,7 @@ def test_set_usage_export_bucket_default(capsys: typing.Any, assert('default prefix of `usage_gce`.' in out) disable_usage_export(project_id=PROJECT) + time.sleep(5) # To make sure the settings are properly updated uel = get_usage_export_bucket(project_id=PROJECT) assert(uel.bucket_name == '') assert(uel.report_name_prefix == '') @@ -53,6 +56,7 @@ def test_set_usage_export_bucket_custom(capsys: typing.Any, temp_bucket: storage.Bucket) -> None: set_usage_export_bucket(project_id=PROJECT, bucket_name=temp_bucket.name, report_name_prefix=TEST_PREFIX) + time.sleep(5) # To make sure the settings are properly updated uel = get_usage_export_bucket(project_id=PROJECT) assert(uel.bucket_name == temp_bucket.name) assert(uel.report_name_prefix == TEST_PREFIX) @@ -60,6 +64,7 @@ def test_set_usage_export_bucket_custom(capsys: typing.Any, assert('usage_gce' not in out) disable_usage_export(project_id=PROJECT) + time.sleep(5) # To make sure the settings are properly updated uel = get_usage_export_bucket(project_id=PROJECT) assert(uel.bucket_name == '') assert(uel.report_name_prefix == '') From 16f39a6b6acf587d488b3634c51e93edbafb4054 Mon Sep 17 00:00:00 2001 From: Maciej Strzelczyk Date: Mon, 9 Aug 2021 12:37:03 +0200 Subject: [PATCH 13/13] Apply suggestions from code review Co-authored-by: Dan Lee <71398022+dandhlee@users.noreply.github.com> --- samples/snippets/quickstart.py | 2 +- samples/snippets/sample_pagination.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/quickstart.py b/samples/snippets/quickstart.py index d43c09dbc..b4e68cdf7 100644 --- a/samples/snippets/quickstart.py +++ b/samples/snippets/quickstart.py @@ -80,7 +80,7 @@ def list_all_instances( print("Instances found:") # Despite using the `max_results` parameter, you don't need to handle the pagination # yourself. The returned `AggregatedListPager` object handles pagination - # automatically, requesting next pages as you iterate over the results. + # automatically, returning separated pages as you iterate over the results. for zone, response in agg_list: if response.instances: all_instances[zone] = response.instances diff --git a/samples/snippets/sample_pagination.py b/samples/snippets/sample_pagination.py index 7c661bd83..99ab80b09 100644 --- a/samples/snippets/sample_pagination.py +++ b/samples/snippets/sample_pagination.py @@ -30,7 +30,7 @@ def print_images_list(project: str) -> None: project: project ID or project number of the Cloud project you want to list images from. Returns: - None + None. """ images_client = compute_v1.ImagesClient() # Listing only non-deprecated images to reduce the size of the reply. @@ -56,7 +56,7 @@ def print_images_list_by_page(project: str, page_size: int = 10) -> None: page_size: size of the pages you want the API to return on each call. Returns: - None + None. """ images_client = compute_v1.ImagesClient() # Listing only non-deprecated images to reduce the size of the reply.