diff --git a/samples/sample-etl-pipeline/README.md b/samples/sample-etl-pipeline/README.md index 828ab36c7..483c7568d 100644 --- a/samples/sample-etl-pipeline/README.md +++ b/samples/sample-etl-pipeline/README.md @@ -11,6 +11,8 @@ account_id: 111111111111 bucket_name: banking-etl-bucket-source object_key: input.zip + build: + enabled: False deploy: provider: s3 targets: diff --git a/samples/sample-etl-pipeline/scripts/some_etl_script.sh b/samples/sample-etl-pipeline/scripts/some_etl_script.sh index fdbb2cfba..3e4c189c2 100644 --- a/samples/sample-etl-pipeline/scripts/some_etl_script.sh +++ b/samples/sample-etl-pipeline/scripts/some_etl_script.sh @@ -7,3 +7,4 @@ echo "Doing some ETL tasks... This could also be done with a custom CodeBuild Im cat big_data.txt echo "You can optionally bundle the buildspec.yml in the source zip and have the commands executed that way.." +echo "Don't forget to enable the build stage to support this" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py index 7672f9651..e3368f930 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py @@ -275,62 +275,93 @@ def generate(self): "region": self.region or ADF_DEPLOYMENT_REGION, "run_order": self.run_order } + input_artifacts = self._get_input_artifacts() + if input_artifacts: + action_props["input_artifacts"] = input_artifacts + output_artifacts = self._get_output_artifacts() + if output_artifacts: + action_props["output_artifacts"] = output_artifacts if _role: action_props["role_arn"] = _role if self.category == 'Manual': del action_props['region'] - if self.category == 'Build' and not self.target: - action_props["input_artifacts"] = [ - _codepipeline.CfnPipeline.InputArtifactProperty( - name="output-source" - ) - ] - action_props["output_artifacts"] = [ - _codepipeline.CfnPipeline.OutputArtifactProperty( - name="{0}-build".format(self.map_params['name']) - ) - ] - if self.category == 'Build' and self.target: - action_props["input_artifacts"] = [ - _codepipeline.CfnPipeline.InputArtifactProperty( - name="{0}-build".format(self.map_params['name']) - ) - ] - if not self.map_params.get('default_providers', {}).get('build', {}).get('enabled', True): - action_props["input_artifacts"] = [ - _codepipeline.CfnPipeline.InputArtifactProperty( - name="output-source" - ) - ] + + return _codepipeline.CfnPipeline.ActionDeclarationProperty( + **action_props + ) + + def _get_base_input_artifact_name(self): + """ + Determine the name for the input artifact for this action. + + Returns: + str: The output artifact name as a string + """ + use_output_source = ( + not self.target or + not self.map_params.get('default_providers', {}).get('build', {}).get('enabled', True) + ) + if use_output_source: + return "output-source" + return "{0}-build".format(self.map_params['name']) + + def _get_input_artifacts(self): + """ + Generate the list of input artifacts that are required for this action + + Returns: + list: The Input Artifacts + """ + if not self.category in ['Build', 'Deploy']: + return [] + input_artifacts = [ + _codepipeline.CfnPipeline.InputArtifactProperty( + name=self._get_base_input_artifact_name(), + ), + ] if self.category == 'Deploy': - action_props["input_artifacts"] = [ - _codepipeline.CfnPipeline.InputArtifactProperty( - name="{0}-build".format(self.map_params['name']) - ) - ] - if self.provider == "CloudFormation" and self.target.get('properties', {}).get('outputs') and self.action_mode != 'CHANGE_SET_REPLACE': - action_props["output_artifacts"] = [ - _codepipeline.CfnPipeline.OutputArtifactProperty( - name=self.target.get('properties', {}).get('outputs') - ) - ] for override in self.target.get('properties', {}).get('param_overrides', []): if self.provider == "CloudFormation" and override.get('inputs') and self.action_mode != "CHANGE_SET_EXECUTE": - action_props["input_artifacts"].append( + input_artifacts.append( _codepipeline.CfnPipeline.InputArtifactProperty( name=override.get('inputs') ) ) + return input_artifacts + + def _get_base_output_artifact_name(self): + """ + Determine the name for the output artifact for this action. + + Returns: + str: The output artifact name as a string + """ if self.category == 'Source': - action_props["output_artifacts"] = [ + return "output-source" + if self.category == 'Build' and not self.target: + return "{0}-build".format(self.map_params['name']) + if self.category == 'Deploy' and self.provider == "CloudFormation": + outputs_name = self.target.get('properties', {}).get('outputs', '') + if outputs_name and self.action_mode != 'CHANGE_SET_REPLACE': + return outputs_name + return '' + + def _get_output_artifacts(self): + """ + Generate the list of output artifacts that are required for this action + + Returns: + list: The Output Artifacts + """ + output_artifact_name = self._get_base_output_artifact_name() + if output_artifact_name: + return [ _codepipeline.CfnPipeline.OutputArtifactProperty( - name="output-source" - ) + name=output_artifact_name + ), ] + return [] - return _codepipeline.CfnPipeline.ActionDeclarationProperty( - **action_props - ) class Pipeline(core.Construct): _import_arns = [ diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/adf_codepipeline_test_constants.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/adf_codepipeline_test_constants.py new file mode 100644 index 000000000..cd3e510bb --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/adf_codepipeline_test_constants.py @@ -0,0 +1,19 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +# pylint: skip-file + +BASE_MAP_PARAMS = { + 'default_providers': { + 'source': { + 'properties': { + 'account_id': 123456123456, + } + }, + 'build': {}, + 'deploy': {}, + }, + 'name': 'name', +} + + diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_generate.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_generate.py new file mode 100644 index 000000000..c59eb3cb2 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_generate.py @@ -0,0 +1,42 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +# pylint: skip-file + +from mock import patch +from cdk_constructs.adf_codepipeline import Action +from adf_codepipeline_test_constants import BASE_MAP_PARAMS + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_output_artifacts') +@patch('cdk_constructs.adf_codepipeline.Action._get_input_artifacts') +def test_generates_with_input_and_output_artifacts_when_given(input_mock, output_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + mocked_input_value = 'InputArtifacts' + mocked_output_value = 'OutputArtifacts' + input_mock.return_value = mocked_input_value + output_mock.return_value = mocked_output_value + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild', + ) + assert action.config['input_artifacts'] == mocked_input_value + assert action.config['output_artifacts'] == mocked_output_value + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_output_artifacts') +@patch('cdk_constructs.adf_codepipeline.Action._get_input_artifacts') +def test_generates_without_input_and_output_artifacts(input_mock, output_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + mocked_value = None + input_mock.return_value = mocked_value + output_mock.return_value = mocked_value + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild', + ) + assert not 'input_artifacts' in action.config + assert not 'output_artifacts' in action.config diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_input_artifacts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_input_artifacts.py new file mode 100644 index 000000000..41bf97fde --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_input_artifacts.py @@ -0,0 +1,205 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +# pylint: skip-file + +from mock import patch +from copy import deepcopy +from cdk_constructs.adf_codepipeline import Action +from aws_cdk import ( aws_codepipeline ) +from adf_codepipeline_test_constants import BASE_MAP_PARAMS + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_input_artifacts_no_build_deploy(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + for category in ['Approval', 'Source']: + action = Action( + map_params=BASE_MAP_PARAMS, + category=category, + provider='Manual' if category == 'Approval' else 'CodeCommit', + ) + assert action._get_input_artifacts() == [] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_input_artifact_name') +def test_get_input_artifacts_build(base_input_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_input_name_mocked_value = 'BaseInputName' + base_input_name_mock.return_value = base_input_name_mocked_value + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild', + ) + assert action.config['input_artifacts'] == [ + aws_codepipeline.CfnPipeline.InputArtifactProperty( + name=base_input_name_mocked_value, + ) + ] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_input_artifact_name') +def test_get_input_artifacts_deploy_simple(base_input_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_input_name_mocked_value = 'BaseInputName' + base_input_name_mock.return_value = base_input_name_mocked_value + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CodeBuild', + target={ + 'properties': { + 'param_overrides': [], + }, + }, + ) + assert action.config['input_artifacts'] == [ + aws_codepipeline.CfnPipeline.InputArtifactProperty( + name=base_input_name_mocked_value, + ) + ] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_input_artifact_name') +def test_get_input_artifacts_deploy_with_cb_param_overrides(base_input_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_input_name_mocked_value = 'BaseInputName' + base_input_name_mock.return_value = base_input_name_mocked_value + override_mocked_value = 'OverrideName' + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CodeBuild', + target={ + 'properties': { + 'param_overrides': [ + { + 'param': 'SomeParam', + 'inputs': override_mocked_value, + 'key_name': 'SomeKeyName', + }, + ], + }, + }, + ) + assert action.config['input_artifacts'] == [ + aws_codepipeline.CfnPipeline.InputArtifactProperty( + name=base_input_name_mocked_value, + ) + ] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_input_artifact_name') +def test_get_input_artifacts_deploy_with_cfn_param_overrides_is_cse(base_input_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_input_name_mocked_value = 'BaseInputName' + base_input_name_mock.return_value = base_input_name_mocked_value + override_mocked_value = 'OverrideName' + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CloudFormation', + action_mode='CHANGE_SET_EXECUTE', + target={ + 'name': 'targetname', + 'id': 'someid', + 'properties': { + 'param_overrides': [ + { + 'param': 'SomeParam', + 'inputs': override_mocked_value, + 'key_name': 'SomeKeyName', + }, + ], + }, + }, + ) + assert action.config['input_artifacts'] == [ + aws_codepipeline.CfnPipeline.InputArtifactProperty( + name=base_input_name_mocked_value, + ) + ] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_input_artifact_name') +def test_get_input_artifacts_deploy_with_cfn_param_overrides_not_cse(base_input_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_input_name_mocked_value = 'BaseInputName' + base_input_name_mock.return_value = base_input_name_mocked_value + override_mocked_value = 'OverrideName' + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CloudFormation', + action_mode='CHANGE_SET_REPLACE', + target={ + 'name': 'targetname', + 'id': 'someid', + 'properties': { + 'param_overrides': [ + { + 'param': 'SomeParam', + 'inputs': override_mocked_value, + 'key_name': 'SomeKeyName', + }, + ], + }, + }, + ) + assert action.config['input_artifacts'] == [ + aws_codepipeline.CfnPipeline.InputArtifactProperty( + name=base_input_name_mocked_value, + ), + aws_codepipeline.CfnPipeline.InputArtifactProperty( + name=override_mocked_value, + ) + ] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_input_artifact_name_build_enabled(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild' + ) + assert action._get_base_input_artifact_name() == 'output-source' + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_input_artifact_name_deploy_build_disabled(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + map_params = deepcopy(BASE_MAP_PARAMS) + map_params['default_providers']['build']['enabled'] = False + action = Action( + map_params=map_params, + category='Deploy', + provider='CodeBuild', + target={ + 'name': 'targetname', + 'id': 'someid', + }, + ) + assert action._get_base_input_artifact_name() == 'output-source' + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_input_artifact_name_deploy_build_enabled(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CodeBuild', + target={ + 'name': 'targetname', + 'id': 'someid', + }, + ) + assert action._get_base_input_artifact_name() == '{0}-build'.format(BASE_MAP_PARAMS['name']) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_output_artifacts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_output_artifacts.py new file mode 100644 index 000000000..399162c15 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/tests/test_adf_codepipeline_output_artifacts.py @@ -0,0 +1,133 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +# pylint: skip-file + +from mock import patch +from copy import deepcopy +from cdk_constructs.adf_codepipeline import Action +from aws_cdk import ( aws_codepipeline ) +from adf_codepipeline_test_constants import BASE_MAP_PARAMS + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_output_artifact_name') +def test_get_output_artifacts_no_base_output(base_output_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_output_name_mock.return_value = '' + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild', + ) + assert not 'output_artifacts' in action.config + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +@patch('cdk_constructs.adf_codepipeline.Action._get_base_output_artifact_name') +def test_get_output_artifacts_with_base_output(base_output_name_mock, action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + base_output_name_mocked_value = 'BaseOutputName' + base_output_name_mock.return_value = base_output_name_mocked_value + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild', + ) + assert action.config['output_artifacts'] == [ + aws_codepipeline.CfnPipeline.OutputArtifactProperty( + name=base_output_name_mocked_value, + ) + ] + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_output_artifact_name_source(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + action = Action( + map_params=BASE_MAP_PARAMS, + category='Source', + provider='CodeCommit' + ) + assert action._get_base_output_artifact_name() == 'output-source' + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_output_artifact_name_build(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + action = Action( + map_params=BASE_MAP_PARAMS, + category='Build', + provider='CodeBuild', + ) + assert action._get_base_output_artifact_name() == '{0}-build'.format(BASE_MAP_PARAMS['name']) + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_output_artifact_name_deploy_codebuild(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CodeBuild', + target={ + 'name': 'targetname', + 'id': 'someid', + }, + ) + assert action._get_base_output_artifact_name() == '' + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_output_artifact_name_deploy_cfn_without_outputs(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CloudFormation', + target={ + 'name': 'targetname', + 'id': 'someid', + }, + ) + assert action._get_base_output_artifact_name() == '' + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_output_artifact_name_deploy_cfn_with_outputs_csr(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + override_outputs_mocked_value = 'OverrideName' + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CloudFormation', + action_mode='CHANGE_SET_REPLACE', + target={ + 'name': 'targetname', + 'id': 'someid', + 'properties': { + 'outputs': override_outputs_mocked_value, + }, + }, + ) + assert action._get_base_output_artifact_name() == '' + + +@patch('cdk_constructs.adf_codepipeline._codepipeline.CfnPipeline.ActionDeclarationProperty') +def test_get_base_output_artifact_name_deploy_cfn_with_outputs_cse(action_decl_mock): + action_decl_mock.side_effect = lambda **x: x + override_outputs_mocked_value = 'OverrideName' + action = Action( + map_params=BASE_MAP_PARAMS, + category='Deploy', + provider='CloudFormation', + action_mode='CHANGE_SET_EXECUTE', + target={ + 'name': 'targetname', + 'id': 'someid', + 'properties': { + 'outputs': override_outputs_mocked_value, + }, + }, + ) + assert action._get_base_output_artifact_name() == override_outputs_mocked_value