Move sync_to_s3 to shared helpers

sbkok · sbkok · commit 35e1e0875959 · 2022-09-14T15:26:28.000+02:00
diff --git a/Makefile b/Makefile
@@ -10,6 +10,7 @@ test:
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pytest.ini
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/initial_commit/pytest.ini
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/pytest.ini
+	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/pytest.ini
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/pytest.ini
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/pytest.ini
 	pytest src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared -vvv -s -c src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/pytest.ini
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml
@@ -767,10 +767,13 @@ Resources:
                 - npm install cdk@1.169 -g -y --quiet --no-progress
                 - aws s3 cp s3://$SHARED_MODULES_BUCKET/adf-build/ ./adf-build/ --recursive --quiet
                 - pip install -r adf-build/requirements.txt -q -t ./adf-build
+            pre_build:
+              commands:
+                - mkdir -p deployment_maps
             build:
               commands:
-                - python adf-build/sync_to_s3.py --upload-with-metadata execution_id=${!CODEBUILD_BUILD_NUMBER} deployment_map.yml s3://$ADF_PIPELINES_BUCKET/deployment_map.yml
-                - python adf-build/sync_to_s3.py --extension .yml --upload-with-metadata execution_id=${!CODEBUILD_BUILD_NUMBER} --recursive deployment_maps '**/*.yml' s3://$ADF_PIPELINES_BUCKET/deployment_maps
+                - python adf-build/helpers/sync_to_s3.py --upload-with-metadata execution_id=${!CODEBUILD_BUILD_NUMBER} deployment_map.yml s3://$ADF_PIPELINES_BUCKET/deployment_map.yml
+                - python adf-build/helpers/sync_to_s3.py --extension .yml --upload-with-metadata execution_id=${!CODEBUILD_BUILD_NUMBER} --recursive deployment_maps s3://$ADF_PIPELINES_BUCKET/deployment_maps
             post_build:
               commands:
                 - echo "Pipelines are updated in the AWS Step Functions ADFPipelineManagementStateMachine."
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/determine_default_branch/requirements.txt
@@ -1,2 +1,2 @@
-boto3==1.18.2
+boto3==1.24.59
 cfn-custom-resource~=1.0.1
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/requirements.txt
@@ -5,7 +5,6 @@ awscli==1.25.60
 boto3==1.24.59
 botocore==1.27.59
 aws-sam-cli==1.55.0
-docopt~=0.6.2
 mock~=4.0.3
 pip~=21.3.1
 pylint~=2.13.9
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/__init__.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/__init__.py
@@ -0,0 +1,2 @@
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/pytest.ini b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/pytest.ini
@@ -0,0 +1,3 @@
+[pytest]
+testpaths = tests
+norecursedirs = terraform
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/sync_to_s3.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/sync_to_s3.py
@@ -118,14 +118,15 @@
 import boto3
 from docopt import docopt
 
-from logger import configure_logger
-
 
 ADF_VERSION = os.environ.get("ADF_VERSION")
 ADF_LOG_LEVEL = os.environ.get("ADF_LOG_LEVEL", "INFO")
-LOGGER = configure_logger(__name__)
 NON_RECURSIVE_KEY = '%%-single-match-%%'
 
+logging.basicConfig(level=logging.INFO)
+LOGGER = logging.getLogger(__name__)
+LOGGER.setLevel(ADF_LOG_LEVEL)
+
 
 class GenericFileData(TypedDict):
     """
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/tests/__init__.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/tests/__init__.py
@@ -0,0 +1,11 @@
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0
+
+"""
+__init__ for tests module
+"""
+
+import sys
+import os
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/tests/test_sync_to_s3.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/tests/test_sync_to_s3.py
diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/pytest.ini b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/pytest.ini
@@ -1,3 +1,3 @@
 [pytest]
 testpaths = tests
-norecursedirs = python cdk
+norecursedirs = python cdk helpers
diff --git a/src/template.yml b/src/template.yml
@@ -1288,7 +1288,7 @@ Resources:
                 # Base templates:
                 - aws s3 sync . s3://$S3_BUCKET --quiet --delete
                 # Upload account files to the ACCOUNT_BUCKET
-                - python adf-build/sync_to_s3.py --extension .yml --upload-with-metadata execution_id=${CODEBUILD_BUILD_NUMBER} --recursive adf-accounts s3://$ACCOUNT_BUCKET
+                - python adf-build/shared/helpers/sync_to_s3.py --extension .yml --upload-with-metadata execution_id=${CODEBUILD_BUILD_NUMBER} --recursive adf-accounts s3://$ACCOUNT_BUCKET
                 # Updates config, updates (or creates) base stacks:
                 - python adf-build/main.py
         Type: CODEPIPELINE

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-boto3==1.18.2`
	`1`	`+boto3==1.24.59`
`2`	`2`	`cfn-custom-resource~=1.0.1`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.`
	`2`	`+# SPDX-License-Identifier: MIT-0`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+[pytest]`
	`2`	`+testpaths = tests`
	`3`	`+norecursedirs = terraform`