Upgrade dependencies (CDK, Pylint, and others) (#364)

sbkok · web-flow · commit df2e5962c7fe · 2021-07-16T15:49:04.000+02:00
* Fix duplicate notification endpoint setup in pipeline generation **Why?** When generating a pipeline, the code optimization introduced by PR #285 broke the notification_endpoint configuration. When set, it would try to configure the notification in the main generator as well as in the default ADF pipeline generator. **What?** Removing the notification endpoint configuration from the main template, as that is replaced by the default ADF pipeline generator implementation. * Upgrade to latest CDK, pylint, and others Upgrade to CDK v1.105 and latest version of pylint and other libraries. Upgrade of pylint resulted in new errors as listed below. These errors have been fixed as part of this commit too.
diff --git a/requirements.txt b/requirements.txt
@@ -1,10 +1,10 @@
-astroid~=2.4.2
-botocore==1.17.63
-boto3==1.14.63
-isort==5.6.4
+astroid~=2.5.6
+botocore==1.20.74
+boto3==1.17.74
+isort==5.8.0
 mock~=4.0.3
-pylint~=2.6.0
-pytest~=6.2.1
-pyyaml>=5.3
-schema~=0.7.2
-tox==3.20.1
+pylint~=2.8.2
+pytest~=6.2.4
+pyyaml>=5.4.1
+schema~=0.7.4
+tox==3.23.1
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml
@@ -672,7 +672,7 @@ Resources:
                 nodejs: 12
             pre_build:
               commands:
-                - npm install cdk@1.88 -g -y --quiet --no-progress
+                - npm install cdk@1.105 -g -y --quiet --no-progress
                 - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet
                 - pip install -r adf-build/requirements.txt -q -t ./adf-build
             build:
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/requirements.txt
@@ -1,3 +1,3 @@
 Jinja2~=2.11.3
-boto3==1.14.63
+boto3==1.17.74
 cfn-custom-resource~=1.0.1
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py
@@ -133,7 +133,8 @@ def send_message(url, payload):
         data=params,
         headers={'content-type': 'application/json'}
     )
-    return urllib.request.urlopen(req)
+    with urllib.request.urlopen(req) as response:
+        return response.read()
 
 def lambda_handler(event, _):
     message = extract_message(event)
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py
@@ -161,6 +161,11 @@ def apply(self, organizations, parameter_store, config):  # pylint: disable=R091
                         path)
                     organizations.attach_policy(
                         policy_id, organization_mapping[path])
+                except organizations.client.exceptions.DuplicatePolicyAttachmentException:
+                    LOGGER.info(
+                        'Policy (%s) for %s exists and is attached already.',
+                        policy,
+                        organization_mapping[path])
                 except organizations.client.exceptions.DuplicatePolicyException:
                     LOGGER.info(
                         'Policy (%s) for %s exists ensuring attached.',
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt
@@ -1,15 +1,14 @@
 # Install libs here that you might want in AWS CodeBuild (On Master Account)
-astroid~=2.4.2
-aws-sam-cli==1.15.0
-awscli==1.18.140
-boto3==1.14.63
-botocore==1.17.63
-jsii<1.20.0,>=1.16.0
+astroid~=2.5.6
+aws-sam-cli==1.23.0
+awscli==1.19.74
+boto3==1.17.74
+botocore==1.20.74
 mock~=4.0.3
-pip~=20.2.3
-pylint~=2.6.0
-pytest~=6.2.1
-pyyaml>=5.3
-six~=1.15.0
+pip~=21.1.1
+pylint~=2.8.2
+pytest~=6.2.4
+pyyaml>=5.4.1
+six~=1.16.0
 tenacity==6.3.0
 urllib3~=1.25.11
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/main.py
@@ -7,7 +7,6 @@
 from aws_cdk import (
     core
 )
-from cdk_constructs import adf_notifications
 from logger import configure_logger
 
 from cdk_stacks.adf_default_pipeline import generate_adf_default_pipeline as generate_default_pipeline, PIPELINE_TYPE as DEFAULT_PIPELINE
@@ -21,9 +20,6 @@ def __init__(self, scope: core.Construct, stack_input: dict, **kwargs) -> None:
         LOGGER.info('Pipeline creation/update of %s commenced', stack_input['input']['name'])
         _pipeline_type = stack_input['input'].get('params', {}).get('type', DEFAULT_PIPELINE).lower()
 
-        if stack_input['input'].get('params', {}).get('notification_endpoint'):
-            stack_input['input']["topic_arn"] = adf_notifications.Notifications(self, 'adf_notifications', stack_input['input']).topic_arn
-
         self.generate_pipeline(_pipeline_type, stack_input)
 
     def generate_pipeline(self, _pipeline_type, stack_input):
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py
@@ -113,6 +113,7 @@ def _parse(filename):
                     return yaml.load(file, Loader=yaml.FullLoader)
             except yaml.scanner.ScannerError:
                 LOGGER.exception('Invalid Yaml for %s.yml', filename)
+                raise
             except FileNotFoundError:
                 return {'Parameters': {}, 'Tags': {}}
 
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/requirements.txt
@@ -1,3 +1,3 @@
-boto3==1.14.63
-botocore==1.17.63
+botocore==1.20.74
+boto3==1.17.74
 docopt~=0.6.2
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/cloudformation.py
@@ -224,27 +224,25 @@ def _change_set_failed_due_to_empty(status, reason):
 
     def _update_stack_termination_protection(self):
         try:
-            return self.client.update_termination_protection(
+            self.client.update_termination_protection(
                 EnableTerminationProtection=STACK_TERMINATION_PROTECTION == "True",
                 StackName=self.stack_name
             )
         except ClientError as e:
             LOGGER.error(
                 '%s | %s, Error: %s',
                 self.account_id, self.stack_name, e)
-            pass
 
     def _delete_change_set(self):
         try:
-            return self.client.delete_change_set(
+            self.client.delete_change_set(
                 ChangeSetName=self.stack_name,
                 StackName=self.stack_name
             )
-        except ClientError as e:
+        except ClientError as client_error:
             LOGGER.info(
                 '%s | %s, Error: %s',
-                self.account_id, self.stack_name, e)
-            pass
+                self.account_id, self.stack_name, client_error)
 
     def _execute_change_set(self, waiter):
         LOGGER.info(
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py
@@ -71,16 +71,19 @@ def update_policy(self, content, policy_id):
         )
 
     def create_policy(self, content, ou_path, policy_type="SERVICE_CONTROL_POLICY"):
-        try:
-            response = self.client.create_policy(
-                Content=content,
-                Description='ADF Managed {0}'.format(policy_type),
-                Name='adf-{0}-{1}'.format('scp' if policy_type == "SERVICE_CONTROL_POLICY" else 'tagging-policy', ou_path),
-                Type=policy_type
-            )
-            return response['Policy']['PolicySummary']['Id']
-        except self.client.exceptions.DuplicatePolicyAttachmentException:
-            pass
+        response = self.client.create_policy(
+            Content=content,
+            Description='ADF Managed {0}'.format(policy_type),
+            Name='adf-{0}-{1}'.format(
+                (
+                    'scp' if policy_type == "SERVICE_CONTROL_POLICY"
+                    else 'tagging-policy'
+                ),
+                ou_path,
+            ),
+            Type=policy_type
+        )
+        return response['Policy']['PolicySummary']['Id']
 
     @staticmethod
     def get_policy_body(path):
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py
@@ -45,12 +45,11 @@ def put_parameter(self, name, value, tier='Standard'):
     def delete_parameter(self, name):
         try:
             LOGGER.debug('Deleting Parameter %s', name)
-            return self.client.delete_parameter(
+            self.client.delete_parameter(
                 Name=name
             )
         except self.client.exceptions.ParameterNotFound:
             LOGGER.debug('Attempted to delete Parameter %s but it was not found', name)
-            pass
 
     def fetch_parameters_by_path(self, path):
         """Gets a Parameter(s) by Path from Parameter Store (Recursively)
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/requirements.txt
@@ -1,60 +1,59 @@
 # Install libs here that you might want in AWS CodeBuild
-aws-cdk-assets==1.88
-aws-cdk.aws-apigateway==1.88
-aws-cdk.aws-applicationautoscaling==1.88
-aws-cdk.aws-autoscaling==1.88
-aws-cdk.aws-autoscaling_common==1.88
-aws-cdk.aws-autoscaling_hooktargets==1.88
-aws-cdk.aws-certificatemanager==1.88
-aws-cdk.aws-cloudformation==1.88
-aws-cdk.aws-cloudfront==1.88
-aws-cdk.aws-cloudwatch==1.88
-aws-cdk.aws-codebuild==1.88
-aws-cdk.aws-codecommit==1.88
-aws-cdk.aws-codedeploy==1.88
-aws-cdk.aws-codepipeline-actions==1.88
-aws-cdk.aws-codepipeline==1.88
-aws-cdk.aws-dynamodb==1.88
-aws-cdk.aws-ec2==1.88
-aws-cdk.aws-ecr==1.88
-aws-cdk.aws-ecr==1.88
-aws-cdk.aws-ecr_assets==1.88
-aws-cdk.aws-ecs==1.88
-aws-cdk.aws-elasticloadbalancing==1.88
-aws-cdk.aws-elasticloadbalancingv2==1.88
-aws-cdk.aws-events==1.88
-aws-cdk.aws-events_targets==1.88
-aws-cdk.aws-iam==1.88
-aws-cdk.aws-kinesis==1.88
-aws-cdk.aws-kms==1.88
-aws-cdk.aws-lambda==1.88
-aws-cdk.aws-lambda_event_sources==1.88
-aws-cdk.aws-logs==1.88
-aws-cdk.aws-route53==1.88
-aws-cdk.aws-route53_targets==1.88
-aws-cdk.aws-s3==1.88
-aws-cdk.aws-s3_assets==1.88
-aws-cdk.aws-s3_assets==1.88
-aws-cdk.aws-s3_notifications==1.88
-aws-cdk.aws-sam==1.88
-aws-cdk.aws-secretsmanager==1.88
-aws-cdk.aws-servicediscovery==1.88
-aws-cdk.aws-sns==1.88
-aws-cdk.aws-sns==1.88
-aws-cdk.aws-sns_subscriptions==1.88
-aws-cdk.aws-sqs==1.88
-aws-cdk.aws-ssm==1.88
-aws-cdk.aws-stepfunctions==1.88
-aws-cdk.core==1.88
-aws-cdk.cx-api==1.88
-aws-cdk.region-info==1.88
-awscli==1.18.140
-aws-sam-cli==1.15.0
-boto3==1.14.63
-botocore==1.17.63
-jsii<1.20.0,>=1.16.0
+aws-cdk-assets==1.105
+aws-cdk.aws-apigateway==1.105
+aws-cdk.aws-applicationautoscaling==1.105
+aws-cdk.aws-autoscaling==1.105
+aws-cdk.aws-autoscaling_common==1.105
+aws-cdk.aws-autoscaling_hooktargets==1.105
+aws-cdk.aws-certificatemanager==1.105
+aws-cdk.aws-cloudformation==1.105
+aws-cdk.aws-cloudfront==1.105
+aws-cdk.aws-cloudwatch==1.105
+aws-cdk.aws-codebuild==1.105
+aws-cdk.aws-codecommit==1.105
+aws-cdk.aws-codedeploy==1.105
+aws-cdk.aws-codepipeline-actions==1.105
+aws-cdk.aws-codepipeline==1.105
+aws-cdk.aws-dynamodb==1.105
+aws-cdk.aws-ec2==1.105
+aws-cdk.aws-ecr==1.105
+aws-cdk.aws-ecr==1.105
+aws-cdk.aws-ecr_assets==1.105
+aws-cdk.aws-ecs==1.105
+aws-cdk.aws-elasticloadbalancing==1.105
+aws-cdk.aws-elasticloadbalancingv2==1.105
+aws-cdk.aws-events==1.105
+aws-cdk.aws-events_targets==1.105
+aws-cdk.aws-iam==1.105
+aws-cdk.aws-kinesis==1.105
+aws-cdk.aws-kms==1.105
+aws-cdk.aws-lambda==1.105
+aws-cdk.aws-lambda_event_sources==1.105
+aws-cdk.aws-logs==1.105
+aws-cdk.aws-route53==1.105
+aws-cdk.aws-route53_targets==1.105
+aws-cdk.aws-s3==1.105
+aws-cdk.aws-s3_assets==1.105
+aws-cdk.aws-s3_assets==1.105
+aws-cdk.aws-s3_notifications==1.105
+aws-cdk.aws-sam==1.105
+aws-cdk.aws-secretsmanager==1.105
+aws-cdk.aws-servicediscovery==1.105
+aws-cdk.aws-sns==1.105
+aws-cdk.aws-sns==1.105
+aws-cdk.aws-sns_subscriptions==1.105
+aws-cdk.aws-sqs==1.105
+aws-cdk.aws-ssm==1.105
+aws-cdk.aws-stepfunctions==1.105
+aws-cdk.core==1.105
+aws-cdk.cx-api==1.105
+aws-cdk.region-info==1.105
+awscli==1.19.74
+aws-sam-cli==1.23.0
+botocore==1.20.74
+boto3==1.17.74
 mock~=4.0.3
-pytest~=6.2.1
-pyyaml>=5.3
-schema~=0.7.2
+pytest~=6.2.4
+pyyaml>=5.4.1
+schema~=0.7.4
 urllib3~=1.25.11
diff --git a/src/lambda_codebase/initial_commit/requirements.txt b/src/lambda_codebase/initial_commit/requirements.txt
@@ -1,3 +1,3 @@
 Jinja2~=2.11.3
-boto3==1.14.63
+boto3==1.17.74
 cfn-custom-resource~=1.0.1

Original file line number	Diff line number	Diff line change
`@@ -133,7 +133,8 @@ def send_message(url, payload):`
`133`	`133`	`data=params,`
`134`	`134`	`headers={'content-type': 'application/json'}`
`135`	`135`	`)`
`136`		`- return urllib.request.urlopen(req)`
	`136`	`+ with urllib.request.urlopen(req) as response:`
	`137`	`+ return response.read()`
`137`	`138`
`138`	`139`	`def lambda_handler(event, _):`
`139`	`140`	`message = extract_message(event)`