From 6e5ae0db23198a20c162c8e09dbd2f9b2d29c973 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Tue, 19 Mar 2019 17:33:00 -0700 Subject: [PATCH 1/7] Add Snippets for security center list_assets call --- securitycenter/docs/snippets_list_assets.py | 160 ++++++++++++++++++++ securitycenter/noxfile.py | 23 +++ 2 files changed, 183 insertions(+) create mode 100644 securitycenter/docs/snippets_list_assets.py diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py new file mode 100644 index 000000000000..ba395b284063 --- /dev/null +++ b/securitycenter/docs/snippets_list_assets.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python + +# Copyright 2018, Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Examples of listing assets in Cloud Security Command Center.""" +import os +from datetime import datetime, timedelta + + +# [START organization_id] +# The numeric identifier for the organization. +ORGANIZATION_ID = os.environ["GCLOUD_ORGANIZATION"] +# [END organization_id] + +# [START asset_resource_project_filter] +PROJECT_ASSET_FILTER = ( + "security_center_properties.resource_type=" + + '"google.cloud.resourcemanager.Project"' +) +# [END asset_resource_project_filter] + + +def org_name(org_id): + """Returns the relative resource name (i.e. organizatoin/[org_id) for + the given ord_id. + + Args: + org_id (str) - The organizations unique numerical ID. + """ + return "organizations/{org_id}".format(org_id=org_id) + + +def to_timestamp_pb2(dt): + """Converts the given timezone aware datetime to a protocol buffer + Timestamp. + + Args: + dt (:class:`datetime.datetime`): The datetime to convert. + """ + from google.api_core.datetime_helpers import to_microseconds + from google.protobuf.timestamp_pb2 import Timestamp + + micros = to_microseconds(dt) + MICRO_PER_SEC = 1000000 + return Timestamp( + seconds=micros // MICRO_PER_SEC, nanos=(micros % MICRO_PER_SEC) * 1000 + ) + + +def to_duration_pb2(delta): + """Converts the given timedelta protocol buffer Duration. + + Args: + delta (:class:`datetime.timedelta`): The duration to convert. + """ + from google.api_core.datetime_helpers import to_microseconds + from google.protobuf.duration_pb2 import Duration + + secs = int(delta.total_seconds()) + nanos = delta.microseconds * 1000 + return Duration(seconds=secs, nanos=nanos) + + +def test_list_all_assets(): + """Demonstrate listing and printing all assets.""" + from google.cloud import securitycenter_v1beta1 as securitycenter + + # [START demo_list_all_assets] + client = securitycenter.SecurityCenterClient() + # list_assets returns an iterator. We convert it to a list + # here for demonstration purposes only. Processing each element + # from the iterator is recommended. + # [END demo_list_all_assets] + assets = list(client.list_assets(org_name(ORGANIZATION_ID))) + assert len(assets) > 0 + # [START demo_list_all_assets] + print(assets) + # [END demo_list_all_assets] + + +def test_list_assets_with_filters(): + """Demonstrate listing assets with a filter.""" + from google.cloud import securitycenter_v1beta1 as securitycenter + + # [START demo_list_assets_with_filter] + client = securitycenter.SecurityCenterClient() + + # list_assets returns an iterator. We convert it to a list + # here for demonstration purposes only. Processing each element + # from the iterator is recommended. + assets = list( + client.list_assets(org_name(ORGANIZATION_ID), filter_=PROJECT_ASSET_FILTER) + ) + # [END demo_list_assets_with_filter] + assert len(assets) > 0 + # [START demo_list_assets_with_filter] + print(assets) + # [END demo_list_assets_with_filter] + + +def test_list_assets_with_filters_and_read_time(): + """Demonstrate listing assets with a filter.""" + from google.cloud import securitycenter_v1beta1 as securitycenter + + # [START demo_list_assets_with_filter_and_time] + client = securitycenter.SecurityCenterClient() + + # Lists assets as of yesterday. + read_time = to_timestamp_pb2(datetime.utcnow() - timedelta(days=1)) + # list_assets returns an iterator. We convert it to a list + # here for demonstration purposes only. Processing each element + # from the iterator is recommended. + assets = list( + client.list_assets( + org_name(ORGANIZATION_ID), filter_=PROJECT_ASSET_FILTER, read_time=read_time + ) + ) + # [END demo_list_assets_with_filter_and_time] + assert len(assets) > 0 + # [START demo_list_assets_with_filter_and_time] + print(assets) + # [END demo_list_assets_with_filter_and_time] + + +def test_list_point_in_time_changes(): + """Demonstrate listing assets along with their state changes.""" + from google.cloud import securitycenter_v1beta1 as securitycenter + + # [START demo_list_assets_changes] + client = securitycenter.SecurityCenterClient() + + # Lists assets as of yesterday. + read_time = datetime(2019, 3, 18) + one_month_before = read_time - datetime(2019, 2, 18) + # list_assets returns an iterator. We convert it to a list + # here for demonstration purposes only. Processing each element + # from the iterator is recommended. + assets = list( + client.list_assets( + org_name(ORGANIZATION_ID), + filter_=PROJECT_ASSET_FILTER, + read_time=to_timestamp_pb2(read_time), + compare_duration=to_duration_pb2(one_month_before), + ) + ) + # [END demo_list_assets_changes] + assert len(assets) > 0 + # [START demo_list_assets_changes] + print(assets) + # [END demo_list_assets_changes] diff --git a/securitycenter/noxfile.py b/securitycenter/noxfile.py index d692cf37f39c..ec2d908c3803 100644 --- a/securitycenter/noxfile.py +++ b/securitycenter/noxfile.py @@ -138,3 +138,26 @@ def cover(session): session.run("coverage", "report", "--show-missing", "--fail-under=100") session.run("coverage", "erase") + + +@nox.session(python=["2.7", "3.5", "3.6", "3.7"]) +def snippets(session): + """Run the documentation example snippets.""" + # Sanity check: Only run snippets system tests if the environment variable + # is set. + if not os.environ.get('GOOGLE_APPLICATION_CREDENTIALS', ''): + session.skip('Credentials must be set via environment variable.') + if not os.environ.get('GCLOUD_ORGANIZATION', ''): + session.skip('Credentials must be set via environment variable.') + + + # Install all test dependencies, then install local packages in place. + session.install('mock', 'pytest') + session.install('-e', '../test_utils/') + session.install('-e', '.') + session.run( + 'py.test', + '--quiet', + os.path.join('docs', 'snippets_list_assets.py'), + *session.posargs + ) From a5deccceb5e2f54dd9682c364c77cb3c456ff919 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Thu, 21 Mar 2019 09:42:04 -0700 Subject: [PATCH 2/7] use well known types --- securitycenter/docs/snippets_list_assets.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py index ba395b284063..36e5503e23e3 100644 --- a/securitycenter/docs/snippets_list_assets.py +++ b/securitycenter/docs/snippets_list_assets.py @@ -50,11 +50,9 @@ def to_timestamp_pb2(dt): from google.api_core.datetime_helpers import to_microseconds from google.protobuf.timestamp_pb2 import Timestamp - micros = to_microseconds(dt) - MICRO_PER_SEC = 1000000 - return Timestamp( - seconds=micros // MICRO_PER_SEC, nanos=(micros % MICRO_PER_SEC) * 1000 - ) + timestamp = Timestamp() + timestamp.FromDatetime(dt) + return timestamp def to_duration_pb2(delta): @@ -66,9 +64,9 @@ def to_duration_pb2(delta): from google.api_core.datetime_helpers import to_microseconds from google.protobuf.duration_pb2 import Duration - secs = int(delta.total_seconds()) - nanos = delta.microseconds * 1000 - return Duration(seconds=secs, nanos=nanos) + duration = Duration() + duration.FromTimedelta(delta) + return duration def test_list_all_assets(): From 616a41422114f4f6b693b7ddcff3c9ab00b9979b Mon Sep 17 00:00:00 2001 From: emkornfield Date: Thu, 21 Mar 2019 13:34:46 -0700 Subject: [PATCH 3/7] fix doc --- securitycenter/docs/snippets_list_assets.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py index 36e5503e23e3..d1f5492b8d7a 100644 --- a/securitycenter/docs/snippets_list_assets.py +++ b/securitycenter/docs/snippets_list_assets.py @@ -136,8 +136,7 @@ def test_list_point_in_time_changes(): # [START demo_list_assets_changes] client = securitycenter.SecurityCenterClient() - - # Lists assets as of yesterday. + # List assets and their state change over a month read_time = datetime(2019, 3, 18) one_month_before = read_time - datetime(2019, 2, 18) # list_assets returns an iterator. We convert it to a list From ab952b3c97d11edff8d104f3c97b148dbb2cdb6f Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Thu, 21 Mar 2019 23:18:46 -0700 Subject: [PATCH 4/7] inline everything per rubrics --- securitycenter/docs/snippets_list_assets.py | 106 ++++++++------------ 1 file changed, 43 insertions(+), 63 deletions(-) diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py index d1f5492b8d7a..204b4c42f86c 100644 --- a/securitycenter/docs/snippets_list_assets.py +++ b/securitycenter/docs/snippets_list_assets.py @@ -17,56 +17,8 @@ from datetime import datetime, timedelta -# [START organization_id] # The numeric identifier for the organization. ORGANIZATION_ID = os.environ["GCLOUD_ORGANIZATION"] -# [END organization_id] - -# [START asset_resource_project_filter] -PROJECT_ASSET_FILTER = ( - "security_center_properties.resource_type=" - + '"google.cloud.resourcemanager.Project"' -) -# [END asset_resource_project_filter] - - -def org_name(org_id): - """Returns the relative resource name (i.e. organizatoin/[org_id) for - the given ord_id. - - Args: - org_id (str) - The organizations unique numerical ID. - """ - return "organizations/{org_id}".format(org_id=org_id) - - -def to_timestamp_pb2(dt): - """Converts the given timezone aware datetime to a protocol buffer - Timestamp. - - Args: - dt (:class:`datetime.datetime`): The datetime to convert. - """ - from google.api_core.datetime_helpers import to_microseconds - from google.protobuf.timestamp_pb2 import Timestamp - - timestamp = Timestamp() - timestamp.FromDatetime(dt) - return timestamp - - -def to_duration_pb2(delta): - """Converts the given timedelta protocol buffer Duration. - - Args: - delta (:class:`datetime.timedelta`): The duration to convert. - """ - from google.api_core.datetime_helpers import to_microseconds - from google.protobuf.duration_pb2 import Duration - - duration = Duration() - duration.FromTimedelta(delta) - return duration def test_list_all_assets(): @@ -75,11 +27,13 @@ def test_list_all_assets(): # [START demo_list_all_assets] client = securitycenter.SecurityCenterClient() + # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) + org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) # list_assets returns an iterator. We convert it to a list # here for demonstration purposes only. Processing each element # from the iterator is recommended. + assets = list(client.list_assets(org_name)) # [END demo_list_all_assets] - assets = list(client.list_assets(org_name(ORGANIZATION_ID))) assert len(assets) > 0 # [START demo_list_all_assets] print(assets) @@ -93,12 +47,17 @@ def test_list_assets_with_filters(): # [START demo_list_assets_with_filter] client = securitycenter.SecurityCenterClient() + # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) + org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) + + project_filter = ( + "security_center_properties.resource_type=" + + '"google.cloud.resourcemanager.Project"' + ) # list_assets returns an iterator. We convert it to a list # here for demonstration purposes only. Processing each element # from the iterator is recommended. - assets = list( - client.list_assets(org_name(ORGANIZATION_ID), filter_=PROJECT_ASSET_FILTER) - ) + assets = list(client.list_assets(org_name, filter_=project_filter)) # [END demo_list_assets_with_filter] assert len(assets) > 0 # [START demo_list_assets_with_filter] @@ -108,20 +67,31 @@ def test_list_assets_with_filters(): def test_list_assets_with_filters_and_read_time(): """Demonstrate listing assets with a filter.""" + from datetime import datetime, timedelta from google.cloud import securitycenter_v1beta1 as securitycenter + from google.protobuf.timestamp_pb2 import Timestamp # [START demo_list_assets_with_filter_and_time] client = securitycenter.SecurityCenterClient() + # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) + org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) + + project_filter = ( + "security_center_properties.resource_type=" + + '"google.cloud.resourcemanager.Project"' + ) + # Lists assets as of yesterday. - read_time = to_timestamp_pb2(datetime.utcnow() - timedelta(days=1)) + read_time = datetime.utcnow() - timedelta(days=1) + timestamp_proto = Timestamp() + timestamp_proto.FromDatetime(read_time) + # list_assets returns an iterator. We convert it to a list # here for demonstration purposes only. Processing each element # from the iterator is recommended. assets = list( - client.list_assets( - org_name(ORGANIZATION_ID), filter_=PROJECT_ASSET_FILTER, read_time=read_time - ) + client.list_assets(org_name, filter_=project_filter, read_time=timestamp_proto) ) # [END demo_list_assets_with_filter_and_time] assert len(assets) > 0 @@ -133,21 +103,31 @@ def test_list_assets_with_filters_and_read_time(): def test_list_point_in_time_changes(): """Demonstrate listing assets along with their state changes.""" from google.cloud import securitycenter_v1beta1 as securitycenter + from google.protobuf.duration_pb2 import Duration + from datetime import timedelta # [START demo_list_assets_changes] client = securitycenter.SecurityCenterClient() - # List assets and their state change over a month - read_time = datetime(2019, 3, 18) - one_month_before = read_time - datetime(2019, 2, 18) + + # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) + org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) + + project_filter = ( + "security_center_properties.resource_type=" + + '"google.cloud.resourcemanager.Project"' + ) + + # List assets and their state change the last 30 days + compare_delta = timedelta(days=30) + # Convert the timedelta to a Duration + duration_proto = Duration() + duration_proto.FromTimedelta(compare_delta) # list_assets returns an iterator. We convert it to a list # here for demonstration purposes only. Processing each element # from the iterator is recommended. assets = list( client.list_assets( - org_name(ORGANIZATION_ID), - filter_=PROJECT_ASSET_FILTER, - read_time=to_timestamp_pb2(read_time), - compare_duration=to_duration_pb2(one_month_before), + org_name, filter_=project_filter, compare_duration=duration_proto ) ) # [END demo_list_assets_changes] From 816693eb8b67da2669c698734df50fb52ee53710 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Tue, 26 Mar 2019 14:11:29 -0700 Subject: [PATCH 5/7] fix apache license --- securitycenter/docs/snippets_list_assets.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py index 204b4c42f86c..9d08e25a9520 100644 --- a/securitycenter/docs/snippets_list_assets.py +++ b/securitycenter/docs/snippets_list_assets.py @@ -1,17 +1,19 @@ #!/usr/bin/env python - -# Copyright 2018, Google LLC +# +# Copyright 2019 Google LLC +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + """ Examples of listing assets in Cloud Security Command Center.""" import os from datetime import datetime, timedelta From 0e82a95f115f6b3e44750585d10cc9623aff3e24 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Tue, 26 Mar 2019 14:59:53 -0700 Subject: [PATCH 6/7] iterate instead of aggregate to list --- securitycenter/docs/snippets_list_assets.py | 56 ++++++++------------- 1 file changed, 22 insertions(+), 34 deletions(-) diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py index 9d08e25a9520..c83708748fa2 100644 --- a/securitycenter/docs/snippets_list_assets.py +++ b/securitycenter/docs/snippets_list_assets.py @@ -34,12 +34,11 @@ def test_list_all_assets(): # list_assets returns an iterator. We convert it to a list # here for demonstration purposes only. Processing each element # from the iterator is recommended. - assets = list(client.list_assets(org_name)) - # [END demo_list_all_assets] - assert len(assets) > 0 - # [START demo_list_all_assets] - print(assets) + asset_iterator = client.list_assets(org_name) + for i, asset_result in enumerate(asset_iterator): + print(i, asset_result) # [END demo_list_all_assets] + assert i > 0 def test_list_assets_with_filters(): @@ -56,15 +55,12 @@ def test_list_assets_with_filters(): "security_center_properties.resource_type=" + '"google.cloud.resourcemanager.Project"' ) - # list_assets returns an iterator. We convert it to a list - # here for demonstration purposes only. Processing each element - # from the iterator is recommended. - assets = list(client.list_assets(org_name, filter_=project_filter)) - # [END demo_list_assets_with_filter] - assert len(assets) > 0 - # [START demo_list_assets_with_filter] - print(assets) + # Call the API and print results. + asset_iterator = client.list_assets(org_name, filter_=project_filter) + for i, asset_result in enumerate(asset_iterator): + print(i, asset_result) # [END demo_list_assets_with_filter] + assert i > 0 def test_list_assets_with_filters_and_read_time(): @@ -89,17 +85,14 @@ def test_list_assets_with_filters_and_read_time(): timestamp_proto = Timestamp() timestamp_proto.FromDatetime(read_time) - # list_assets returns an iterator. We convert it to a list - # here for demonstration purposes only. Processing each element - # from the iterator is recommended. - assets = list( - client.list_assets(org_name, filter_=project_filter, read_time=timestamp_proto) + # Call the API and print results. + asset_iterator = client.list_assets( + org_name, filter_=project_filter, read_time=timestamp_proto ) + for i, asset_result in enumerate(asset_iterator): + print(i, asset_result) # [END demo_list_assets_with_filter_and_time] - assert len(assets) > 0 - # [START demo_list_assets_with_filter_and_time] - print(assets) - # [END demo_list_assets_with_filter_and_time] + assert i > 0 def test_list_point_in_time_changes(): @@ -113,7 +106,6 @@ def test_list_point_in_time_changes(): # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) - project_filter = ( "security_center_properties.resource_type=" + '"google.cloud.resourcemanager.Project"' @@ -124,16 +116,12 @@ def test_list_point_in_time_changes(): # Convert the timedelta to a Duration duration_proto = Duration() duration_proto.FromTimedelta(compare_delta) - # list_assets returns an iterator. We convert it to a list - # here for demonstration purposes only. Processing each element - # from the iterator is recommended. - assets = list( - client.list_assets( - org_name, filter_=project_filter, compare_duration=duration_proto - ) + # Call the API and print results. + asset_iterator = client.list_assets( + org_name, filter_=project_filter, compare_duration=duration_proto ) + for i, asset in enumerate(asset_iterator): + print(i, asset) + # [END demo_list_assets_changes] - assert len(assets) > 0 - # [START demo_list_assets_changes] - print(assets) - # [END demo_list_assets_changes] + assert i > 0 From d7905b3582e0620aca083ec44f8adb801403e431 Mon Sep 17 00:00:00 2001 From: Micah Kornfield Date: Tue, 26 Mar 2019 16:12:15 -0700 Subject: [PATCH 7/7] fix out of date doc --- securitycenter/docs/snippets_list_assets.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/securitycenter/docs/snippets_list_assets.py b/securitycenter/docs/snippets_list_assets.py index c83708748fa2..e75b5cfa9f86 100644 --- a/securitycenter/docs/snippets_list_assets.py +++ b/securitycenter/docs/snippets_list_assets.py @@ -31,9 +31,8 @@ def test_list_all_assets(): client = securitycenter.SecurityCenterClient() # ORGANIZATION_ID is the numeric ID of the organization (e.g. 123213123121) org_name = "organizations/{org_id}".format(org_id=ORGANIZATION_ID) - # list_assets returns an iterator. We convert it to a list - # here for demonstration purposes only. Processing each element - # from the iterator is recommended. + + # Call the API and print results. asset_iterator = client.list_assets(org_name) for i, asset_result in enumerate(asset_iterator): print(i, asset_result)