Feature: Wave Config for Targets

docs/user-guide.md

@@ -188,7 +188,25 @@ targets:
     name: production_step
     provider: ...
     properties: ...
-```
+  - path: /my_ou/production/some_path
+    regions: [eu-central-1, us-west-1]
+    name: another_step
+    wave:
+      size: 30 # (Optional) This forces the pipeline to split this OU into seperate stages, each stage containing up to X accounts
+    exclude: 
+      - 9999999999 # (Optional) List of accounts to exclude from this target. Currently only supports account Ids 
+    properties: ...    
+```
+
+CodePipeline has a limit of 50 actions per stage.
+A stage is identified in the above list of targets with a new entry in the array, using `-`.
+
+To workaround this limit, ADF will split the accounts x regions that are selected as part of one stage over multiple stages when required.
+A new stage is introduced for every 50 accounts/region deployments by default. The default of 50 will make sense for most pipelines.
+However, in some situations, you would like to limit the rate at which an update is rolled out to the list of accounts/regions.
+This can be configured using the `wave/size` target property. Setting these to `30` as shown above, will introduce a new stage for every 30 accounts/regions.
+If the `/my_ou/production/some_path` OU would contain 25 accounts (actually 26, but account `9999999999` is excluded by the setup above), multiplied by the two regions it targets in the last step, the total of account/region deployment actions required would be 50.
+Since the configuration is set to 30, the first 30 accounts will be deployed to in the first stage. If all of these successfully deploy, the pipeline will continue to the next stage, deploying to the remaining 20 account/regions.
 
 ### Params
 

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/adf_default_pipeline.py

@@ -96,7 +96,7 @@ def generate_targets_for_pipeline(_stages, scope, stack_input):
     for index, targets in enumerate(
             stack_input["input"].get("environments", {}).get("targets", [])
     ):
-        _actions = []
+
         top_level_deployment_type = (
             stack_input["input"]
             .get("default_providers", {})
@@ -112,74 +112,72 @@ def generate_targets_for_pipeline(_stages, scope, stack_input):
             .get("action", "")
         )
 
-        for target in targets:
-            target_stage_override = target.get("provider") or top_level_deployment_type
-            if target.get("name") == "approval" or target.get("provider", "") == "approval":
-                _actions.extend(
-                    [
-                        adf_codepipeline.Action(
-                            name="{0}".format(target["name"]),
-                            provider="Manual",
-                            category="Approval",
-                            target=target,
-                            run_order=1,
-                            map_params=stack_input["input"],
-                            action_name="{0}".format(target["name"]),
-                        ).config
-                    ]
-                )
-                continue
+        for wave_index, wave in enumerate(targets):
+            _actions = []
+            _is_approval = (
+                wave[0].get("name", "").startswith("approval")
+                or wave[0].get("provider", "") == "approval"
+            )
+            _action_type_name = "approval" if _is_approval else "deployment"
+            _stage_name = (
+                # 0th Index since step names are for entire stages not
+                # per target.
+                f"{wave[0].get('step_name')}-{wave_index}"
+                if wave[0].get("step_name") else f"{_action_type_name}-stage-{index + 1}-wave-{wave_index}"
+            )
 
-            if "codebuild" in target_stage_override:
-                _actions.extend(
-                    [
-                        adf_codebuild.CodeBuild(
-                            scope,
-                            # Use the name of the pipeline for CodeBuild
-                            # instead of the target name as it will always
-                            # operate from the deployment account.
-                            "{pipeline_name}-stage-{index}".format(
-                                pipeline_name=stack_input["input"]["name"],
-                                index=index + 1,
-                            ),
-                            stack_input["ssm_params"][ADF_DEPLOYMENT_REGION]["modules"],
-                            stack_input["ssm_params"][ADF_DEPLOYMENT_REGION]["kms"],
-                            stack_input["input"],
-                            target,
-                        ).deploy
-                    ]
+            for target in wave:
+                target_stage_override = target.get("provider") or top_level_deployment_type
+                if target.get("name") == "approval" or target.get("provider", "") == "approval":
+                    _actions.extend(
+                        [
+                            adf_codepipeline.Action(
+                                name=f"wave-{wave_index}-{target.get('name')}".format(target["name"]),
+                                provider="Manual",
+                                category="Approval",
+                                target=target,
+                                run_order=1,
+                                map_params=stack_input["input"],
+                                action_name=f"{target.get('name')}",
+                            ).config
+                        ]
+                    )
+                    continue
+
+                if "codebuild" in target_stage_override:
+                    _actions.extend(
+                        [
+                            adf_codebuild.CodeBuild(
+                                scope,
+                                # Use the name of the pipeline for CodeBuild
+                                # instead of the target name as it will always
+                                # operate from the deployment account.
+                                f"{stack_input['input']['name']}-target-{index + 1}-wave-{wave_index}",
+                                stack_input["ssm_params"][ADF_DEPLOYMENT_REGION]["modules"],
+                                stack_input["ssm_params"][ADF_DEPLOYMENT_REGION]["kms"],
+                                stack_input["input"],
+                                target,
+                            ).deploy
+                        ]
+                    )
+                    continue
+
+                regions = target.get("regions", [])
+                generate_deployment_action_per_region(
+                    _actions,
+                    regions,
+                    stack_input,
+                    target,
+                    target_stage_override,
+                    top_level_action,
                 )
-                continue
 
-            regions = target.get("regions", [])
-            generate_deployment_action_per_region(
-                _actions,
-                regions,
-                stack_input,
-                target,
-                target_stage_override,
-                top_level_action,
-            )
-        _is_approval = (
-            targets[0].get("name", "").startswith("approval")
-            or targets[0].get("provider", "") == "approval"
-        )
-        _action_type_name = "approval" if _is_approval else "deployment"
-        _stage_name = (
-            # 0th Index since step names are for entire stages not
-            # per target.
-            targets[0].get("step_name")
-            or "{action_type_name}-stage-{index}".format(
-                action_type_name=_action_type_name,
-                index=index + 1,
-            )
-        )
-        _stages.append(
-            _codepipeline.CfnPipeline.StageDeclarationProperty(
-                name=_stage_name,
-                actions=_actions,
+            _stages.append(
+                _codepipeline.CfnPipeline.StageDeclarationProperty(
+                    name=_stage_name,
+                    actions=_actions,
+                )
             )
-        )
 
 
 def generate_deployment_action_per_region(_actions,

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_stacks/tests/test_default_pipeline_type.py

@@ -0,0 +1,140 @@
+# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT-0
+
+# pylint: skip-file
+
+from aws_cdk import core
+from cdk_stacks.main import PipelineStack
+
+
+def test_pipeline_creation_outputs_as_expected_when_input_has_1_target_with_2_waves():
+    region_name = "eu-central-1"
+    acount_id = "123456789012"
+
+    stack_input = {
+        "input": {
+            "params": {},
+            "default_providers": {"deploy": {"provider": "codedeploy"}},
+            "regions": {},
+        },
+        "ssm_params": {"fake-region": {}},
+    }
+
+    stack_input["input"]["name"] = "test-stack"
+    stack_input["input"]["environments"] = {
+        "targets": [
+            [
+                [
+                    {"name": "account-1", "id": "001", "regions": ["eu-west-1"]},
+                    {"name": "account-2", "id": "002", "regions": ["eu-west-1"]},
+                    {"name": "account-3", "id": "003", "regions": ["eu-west-1"]},
+                ],
+                [
+                    {"name": "account-4", "id": "004", "regions": ["eu-west-1"]},
+                    {"name": "account-5", "id": "005", "regions": ["eu-west-1"]},
+                    {"name": "account-6", "id": "006", "regions": ["eu-west-1"]},
+                ],
+            ],
+        ]
+    }
+
+    stack_input["input"]["default_providers"]["source"] = {
+        "provider": "codecommit",
+        "properties": {"account_id": "123456789012"},
+    }
+    stack_input["input"]["default_providers"]["build"] = {
+        "provider": "codebuild",
+        "properties": {"account_id": "123456789012"},
+    }
+
+    stack_input["ssm_params"][region_name] = {
+        "modules": "fake-bucket-name",
+        "kms": f"arn:aws:kms:{region_name}:{acount_id}:key/my-unique-kms-key-id",
+    }
+    app = core.App()
+    PipelineStack(app, stack_input)
+
+    cloud_assembly = app.synth()
+    resources = {
+        k[0:-8]: v for k, v in cloud_assembly.stacks[0].template["Resources"].items()
+    }
+    code_pipeline = resources["codepipeline"]
+    assert code_pipeline["Type"] == "AWS::CodePipeline::Pipeline"
+    assert len(code_pipeline["Properties"]["Stages"]) == 4
+
+    target_1_wave_1 = code_pipeline["Properties"]["Stages"][2]
+    assert target_1_wave_1["Name"] == "deployment-stage-1-wave-0"
+    assert len(target_1_wave_1["Actions"]) == 3
+
+    target_1_wave_2 = code_pipeline["Properties"]["Stages"][3]
+    assert target_1_wave_2["Name"] == "deployment-stage-1-wave-1"
+    assert len(target_1_wave_2["Actions"]) == 3
+
+
+def test_pipeline_creation_outputs_as_expected_when_input_has_2_targets_with_2_waves_and_1_wave():
+    region_name = "eu-central-1"
+    acount_id = "123456789012"
+
+    stack_input = {
+        "input": {
+            "params": {},
+            "default_providers": {"deploy": {"provider": "codedeploy"}},
+            "regions": {},
+        },
+        "ssm_params": {"fake-region": {}},
+    }
+
+    stack_input["input"]["name"] = "test-stack"
+    stack_input["input"]["environments"] = {
+        "targets": [
+            [
+                [
+                    {"name": "account-1", "id": "001", "regions": ["eu-west-1"]},
+                    {"name": "account-2", "id": "002", "regions": ["eu-west-1"]},
+                    {"name": "account-3", "id": "003", "regions": ["eu-west-1"]},
+                ],
+                [
+                    {"name": "account-4", "id": "004", "regions": ["eu-west-1"]},
+                    {"name": "account-5", "id": "005", "regions": ["eu-west-1"]},
+                    {"name": "account-6", "id": "006", "regions": ["eu-west-1"]},
+                ],
+            ],
+            [[{"name": "account-7", "id": "007", "regions": ["eu-west-2"]}]],
+        ]
+    }
+
+    stack_input["input"]["default_providers"]["source"] = {
+        "provider": "codecommit",
+        "properties": {"account_id": "123456789012"},
+    }
+    stack_input["input"]["default_providers"]["build"] = {
+        "provider": "codebuild",
+        "properties": {"account_id": "123456789012"},
+    }
+
+    stack_input["ssm_params"][region_name] = {
+        "modules": "fake-bucket-name",
+        "kms": f"arn:aws:kms:{region_name}:{acount_id}:key/my-unique-kms-key-id",
+    }
+    app = core.App()
+    PipelineStack(app, stack_input)
+
+    cloud_assembly = app.synth()
+    resources = {
+        k[0:-8]: v for k, v in cloud_assembly.stacks[0].template["Resources"].items()
+    }
+    code_pipeline = resources["codepipeline"]
+    assert code_pipeline["Type"] == "AWS::CodePipeline::Pipeline"
+    assert len(code_pipeline["Properties"]["Stages"]) == 5
+
+    target_1_wave_1 = code_pipeline["Properties"]["Stages"][2]
+    assert target_1_wave_1["Name"] == "deployment-stage-1-wave-0"
+    assert len(target_1_wave_1["Actions"]) == 3
+
+    target_1_wave_2 = code_pipeline["Properties"]["Stages"][3]
+    assert target_1_wave_2["Name"] == "deployment-stage-1-wave-1"
+    assert len(target_1_wave_2["Actions"]) == 3
+
+    target_2_wave_1 = code_pipeline["Properties"]["Stages"][4]
+    assert target_2_wave_1["Name"] == "deployment-stage-2-wave-0"
+    assert len(target_2_wave_1["Actions"]) == 1    

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/generate_pipeline_inputs.py

@@ -113,8 +113,8 @@ def worker_thread(p, organizations, auto_create_repositories, deployment_map, pa
                 pipeline.stage_regions.append(regions)
                 pipeline_target = Target(path_or_tag, target_structure, organizations, step, regions)
                 pipeline_target.fetch_accounts_for_target()
-        pipeline.template_dictionary["targets"].append(
-            target_structure.account_list)
+
+            pipeline.template_dictionary["targets"].append(target.target_structure.generate_waves())
 
     if DEPLOYMENT_ACCOUNT_REGION not in regions:
         pipeline.stage_regions.append(DEPLOYMENT_ACCOUNT_REGION)

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/schema_validation.py

@@ -302,6 +302,10 @@
     int
 )]
 
+TARGET_WAVE_SCHEME = {
+    Optional("size", default=50): int,
+}
+
 # Pipeline Params
 
 TARGET_SCHEMA = {
@@ -311,7 +315,9 @@
     Optional("name"): str,
     Optional("provider"): Or('lambda', 's3', 'codedeploy', 'cloudformation', 'service_catalog', 'approval', 'codebuild', 'jenkins'),
     Optional("properties"): Or(CODEBUILD_PROPS, JENKINS_PROPS, CLOUDFORMATION_PROPS, CODEDEPLOY_PROPS, S3_DEPLOY_PROPS, SERVICECATALOG_PROPS, LAMBDA_PROPS, APPROVAL_PROPS),
-    Optional("regions"): REGION_SCHEMA
+    Optional("regions"): REGION_SCHEMA,
+    Optional("exclude", default=[]): [str],
+    Optional("wave", default={"size": 50}): TARGET_WAVE_SCHEME
 }
 COMPLETION_TRIGGERS_SCHEMA = {
     "pipelines": [str]

src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/target.py

@@ -23,6 +23,8 @@ class TargetStructure:
     def __init__(self, target):
         self.target = TargetStructure._define_target_type(target)
         self.account_list = []
+        self.wave = target.get('wave', {}) if isinstance(target, dict) else {}
+        self.exclude = target.get('exclude', []) if isinstance(target, dict) else []
 
     @staticmethod
     def _define_target_type(target):
@@ -45,6 +47,14 @@ def _define_target_type(target):
             target = [target]
         return target
 
+    def generate_waves(self):
+        waves = []
+        wave_size = self.wave.get('size', 50)
+        length = len(self.account_list)
+        for index in range(0, length, wave_size):
+            yield self.account_list[index:min(index + wave_size, length)]
+            waves.append(self.account_list[index:min(index + wave_size, length)])
+        return waves
 
 class Target:
     def __init__(self, path, target_structure, organizations, step, regions):
@@ -83,7 +93,7 @@ def _create_response_object(self, responses):
         _entities = 0
         for response in responses:
             _entities += 1
-            if Target._account_is_active(response):
+            if Target._account_is_active(response) and not response.get('Id') in self.target_structure.exclude:
                 self.target_structure.account_list.append(
                     self._create_target_info(
                         response.get('Name'),
@@ -103,8 +113,11 @@ def _target_is_tags(self):
         responses = self.organizations.get_account_ids_for_tags(self.path)
         accounts = []
         for response in responses:
-            account = self.organizations.client.describe_account(AccountId=response).get('Account')
-            accounts.append(account)
+            if response.startswith('ou-'):
+                accounts.extend(self.organizations.get_accounts_for_parent(response))
+            else:
+                account = self.organizations.client.describe_account(AccountId=response).get('Account')
+                accounts.append(account)
         self._create_response_object(accounts)
 
     def _target_is_ou_id(self):