diff --git a/Makefile.tox b/Makefile.tox index 1cffe0b6f..0c8d8521a 100644 --- a/Makefile.tox +++ b/Makefile.tox @@ -11,10 +11,8 @@ all: test lint test: # Run unit tests - ( \ - for config in $(TEST_CONFIGS); do \ - pytest $$(dirname $$config) -vvv -s -c $$config; \ - done \ + @ $(foreach config,$(TEST_CONFIGS), \ + pytest $$(dirname $(config)) -vvv -s -c $(config) || exit 1; \ ) lint: diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 4626a626a..be88a8f58 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -709,7 +709,7 @@ You can read more about creating a Token Once the token has been created you can store that in AWS Secrets Manager on the Deployment Account. The Webhook Secret is a value you define and store in AWS Secrets Manager with a path of `/adf/my_teams_token`. By Default, ADF only -has read access access to Secrets with a path that starts with `/adf/`. +has read access to Secrets with a path that starts with `/adf/`. Once the values are stored, you can create the Repository in GitHub as per normal. Once its created you do not need to do anything else on GitHub's side diff --git a/src/lambda_codebase/account/main.py b/src/lambda_codebase/account/main.py index 77ae7ee6c..cc855d58a 100644 --- a/src/lambda_codebase/account/main.py +++ b/src/lambda_codebase/account/main.py @@ -12,6 +12,7 @@ import time import json import boto3 +from botocore.exceptions import ClientError from cfn_custom_resource import ( # pylint: disable=unused-import lambda_handler, create, @@ -19,6 +20,9 @@ delete, ) +# ADF Imports +from organizations import Organizations + # Type aliases: Data = Mapping[str, str] PhysicalResourceId = str @@ -28,10 +32,16 @@ # Globals: ORGANIZATION_CLIENT = boto3.client("organizations") SSM_CLIENT = boto3.client("ssm") +TAGGING_CLIENT = boto3.client("resourcegroupstaggingapi") LOGGER = logging.getLogger(__name__) LOGGER.setLevel(os.environ.get("ADF_LOG_LEVEL", logging.INFO)) logging.basicConfig(level=logging.INFO) MAX_RETRIES = 120 # => 120 retries * 5 seconds = 10 minutes +DEPLOYMENT_OU_PATH = '/deployment' +DEPLOYMENT_ACCOUNT_ID_PARAM_PATH = "/adf/deployment_account_id" +SSM_PARAMETER_ADF_DESCRIPTION = ( + "DO NOT EDIT - Used by The AWS Deployment Framework" +) class InvalidPhysicalResourceId(Exception): @@ -76,6 +86,7 @@ def create_(event: Mapping[str, Any], _context: Any) -> CloudFormationResponse: account_name, account_email, cross_account_access_role_name, + is_update=False, ) return PhysicalResource( account_id, account_name, account_email, created @@ -96,6 +107,7 @@ def update_(event: Mapping[str, Any], _context: Any) -> CloudFormationResponse: account_name, account_email, cross_account_access_role_name, + is_update=True, ) return PhysicalResource( account_id, account_name, account_email, created or previously_created @@ -118,24 +130,136 @@ def delete_(event, _context): return +def _set_deployment_account_id_parameter(deployment_account_id: str): + SSM_CLIENT.put_parameter( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + Value=deployment_account_id, + Description=SSM_PARAMETER_ADF_DESCRIPTION, + Type="String", + Overwrite=True, + ) + + +def _find_deployment_account_via_orgs_api() -> str: + try: + organizations = Organizations( + org_client=ORGANIZATION_CLIENT, + tagging_client=TAGGING_CLIENT, + ) + accounts_found = organizations.get_accounts_in_path( + DEPLOYMENT_OU_PATH, + ) + active_accounts = list(filter( + lambda account: account.get("Status") == "ACTIVE", + accounts_found, + )) + number_of_deployment_accounts = len(active_accounts) + if number_of_deployment_accounts > 1: + raise RuntimeError( + "Failed to determine Deployment account to setup, as " + f"{number_of_deployment_accounts} AWS Accounts were found " + f"in the {DEPLOYMENT_OU_PATH} organization unit (OU). " + "Please ensure there is only one account in the " + f"{DEPLOYMENT_OU_PATH} OU path. " + "Move all AWS accounts you don't want to be bootstrapped as " + f"the ADF deployment account out of the {DEPLOYMENT_OU_PATH} " + "OU. In case there are no accounts in the " + f"{DEPLOYMENT_OU_PATH} OU, ADF will automatically create a " + "new AWS account for you, or move the deployment account as " + "specified at install time of ADF to the respective OU.", + ) + if number_of_deployment_accounts == 1: + deployment_account_id = str(active_accounts[0].get("Id")) + _set_deployment_account_id_parameter(deployment_account_id) + return deployment_account_id + LOGGER.debug( + "No active AWS Accounts found in the %s OU path.", + DEPLOYMENT_OU_PATH, + ) + except ClientError as client_error: + LOGGER.debug( + "Retrieving the accounts in %s failed due to %s." + "Most likely the %s OU does not exist, if so, you can ignore this " + "error as it will create it later on automatically.", + DEPLOYMENT_OU_PATH, + str(client_error), + DEPLOYMENT_OU_PATH, + ) + return "" + + +def _find_deployment_account_via_ssm_params() -> str: + try: + get_parameter = SSM_CLIENT.get_parameter( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + return get_parameter["Parameter"]["Value"] + except SSM_CLIENT.exceptions.ParameterNotFound: + LOGGER.debug( + "SSM Parameter at %s does not exist. This is expected behavior " + "when you install ADF the first time or upgraded ADF while the " + "parameter store path was changed.", + DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + return "" + + def ensure_account( existing_account_id: str, account_name: str, account_email: str, cross_account_access_role_name: str, no_retries: int = 0, + is_update: bool = False, ) -> Tuple[AccountId, bool]: # If an existing account ID was provided, use that: + ssm_deployment_account_id = _find_deployment_account_via_ssm_params() if existing_account_id: + LOGGER.info( + "Using existing deployment account as specified %s.", + existing_account_id, + ) + if is_update and not ssm_deployment_account_id: + LOGGER.info( + "The %s param was not found, creating it as we are " + "updating ADF", + DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + _set_deployment_account_id_parameter(existing_account_id) return existing_account_id, False - # If no existing account ID was provided, check if the ID is stores in + # If no existing account ID was provided, check if the ID is stored in # parameter store: - try: - get_parameter = SSM_CLIENT.get_parameter(Name="deployment_account_id") - return get_parameter["Parameter"]["Value"], False - except SSM_CLIENT.exceptions.ParameterNotFound: - pass # Carry on with creating the account + if ssm_deployment_account_id: + LOGGER.info( + "Using deployment account as specified with param %s : %s.", + DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ssm_deployment_account_id, + ) + return ssm_deployment_account_id, False + + if is_update: + # If no existing account ID was provided and Parameter Store did not + # contain the account id, check if the /deployment OU exists and + # whether that has a single account inside. + deployment_account_id = _find_deployment_account_via_orgs_api() + if deployment_account_id: + LOGGER.info( + "Using deployment account %s as found in AWS Organization %s.", + deployment_account_id, + DEPLOYMENT_OU_PATH, + ) + _set_deployment_account_id_parameter(deployment_account_id) + return deployment_account_id, False + + error_msg = ( + "When updating ADF should not be required to create a deployment " + "account. If your previous installation failed and you try to fix " + "it via an update, please delete the ADF stack first and run it " + "as a fresh installation." + ) + LOGGER.error(error_msg) + raise RuntimeError(error_msg) # No existing account found: create one LOGGER.info("Creating account ...") @@ -147,9 +271,8 @@ def ensure_account( IamUserAccessToBilling="ALLOW", ) except ORGANIZATION_CLIENT.exceptions.ConcurrentModificationException as err: - return handle_concurrent_modification( + return _handle_concurrent_modification( err, - existing_account_id, account_name, account_email, cross_account_access_role_name, @@ -159,10 +282,13 @@ def ensure_account( request_id = create_account["CreateAccountStatus"]["Id"] LOGGER.info("Account creation requested, request ID: %s", request_id) - return wait_on_account_creation(request_id) + LOGGER.info("Waiting for account creation to complete...") + deployment_account_id = _wait_on_account_creation(request_id) + LOGGER.info("Account created, using %s", deployment_account_id) + return deployment_account_id, True -def wait_on_account_creation(request_id: str) -> Tuple[AccountId, bool]: +def _wait_on_account_creation(request_id: str) -> AccountId: while True: account_status = ORGANIZATION_CLIENT.describe_create_account_status( CreateAccountRequestId=request_id @@ -180,12 +306,11 @@ def wait_on_account_creation(request_id: str) -> Tuple[AccountId, bool]: else: account_id = account_status["CreateAccountStatus"]["AccountId"] LOGGER.info("Account created: %s", account_id) - return account_id, True + return account_id -def handle_concurrent_modification( +def _handle_concurrent_modification( error: Exception, - existing_account_id: str, account_name: str, account_email: str, cross_account_access_role_name: str, @@ -199,6 +324,7 @@ def handle_concurrent_modification( ) raise error time.sleep(5) + existing_account_id = "" return ensure_account( existing_account_id, account_name, diff --git a/src/lambda_codebase/account/tests/test_main.py b/src/lambda_codebase/account/tests/test_main.py index 597a2d857..fb5e83821 100644 --- a/src/lambda_codebase/account/tests/test_main.py +++ b/src/lambda_codebase/account/tests/test_main.py @@ -5,10 +5,15 @@ import pytest from mock import patch, call + from main import ( ensure_account, - handle_concurrent_modification, - wait_on_account_creation, + _handle_concurrent_modification, + _find_deployment_account_via_orgs_api, + _wait_on_account_creation, + DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + DEPLOYMENT_OU_PATH, + SSM_PARAMETER_ADF_DESCRIPTION, MAX_RETRIES, ) @@ -27,11 +32,12 @@ class OtherException(Exception): @patch("main.ORGANIZATION_CLIENT") @patch("main.SSM_CLIENT") -@patch("main.wait_on_account_creation") -@patch("main.handle_concurrent_modification") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") @patch("main.LOGGER") def test_deployment_account_given( - logger, concur_mod_fn, wait_on_fn, ssm_client, org_client + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client ): account_id = "123456789012" account_name = "test-deployment-account" @@ -42,29 +48,147 @@ def test_deployment_account_given( ConcurrentModificationException ) + ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = "" + returned_account_id, created = ensure_account( account_id, account_name, account_email, cross_account_access_role_name, + is_update=False, ) assert returned_account_id == account_id assert not created - logger.info.assert_not_called() + logger.info.assert_called_once_with( + 'Using existing deployment account as specified %s.', + account_id, + ) + concur_mod_fn.assert_not_called() + wait_on_fn.assert_not_called() + ssm_client.get_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + ssm_client.put_parameter.assert_not_called() + find_orgs_api.assert_not_called() + org_client.create_account.assert_not_called() + + +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") +@patch("main.LOGGER") +def test_deployment_account_given_on_update_no_params( + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client +): + account_id = "123456789012" + account_name = "test-deployment-account" + account_email = "test@amazon.com" + cross_account_access_role_name = "some-role" + ssm_client.exceptions.ParameterNotFound = ParameterNotFound + org_client.exceptions.ConcurrentModificationException = ( + ConcurrentModificationException + ) + + ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = account_id + + returned_account_id, created = ensure_account( + account_id, + account_name, + account_email, + cross_account_access_role_name, + is_update=True, + ) + + assert returned_account_id == account_id + assert not created + logger.info.assert_has_calls([ + call( + 'Using existing deployment account as specified %s.', + account_id, + ), + call( + 'The %s param was not found, creating it as we are updating ADF', + DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ), + ]) + concur_mod_fn.assert_not_called() + wait_on_fn.assert_not_called() + ssm_client.get_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + ssm_client.put_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + Value=account_id, + Description=SSM_PARAMETER_ADF_DESCRIPTION, + Type="String", + Overwrite=True, + ) + find_orgs_api.assert_not_called() + org_client.create_account.assert_not_called() + + +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") +@patch("main.LOGGER") +def test_deployment_account_given_on_update_with_params( + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client +): + account_id = "123456789012" + account_name = "test-deployment-account" + account_email = "test@amazon.com" + cross_account_access_role_name = "some-role" + ssm_client.exceptions.ParameterNotFound = ParameterNotFound + org_client.exceptions.ConcurrentModificationException = ( + ConcurrentModificationException + ) + + ssm_client.get_parameter.return_value = { + "Parameter": { + "Value": account_id, + } + } + find_orgs_api.return_value = account_id + + returned_account_id, created = ensure_account( + account_id, + account_name, + account_email, + cross_account_access_role_name, + is_update=True, + ) + + assert returned_account_id == account_id + assert not created + logger.info.assert_called_once_with( + 'Using existing deployment account as specified %s.', + account_id, + ) concur_mod_fn.assert_not_called() wait_on_fn.assert_not_called() - ssm_client.get_parameter.assert_not_called() + ssm_client.get_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + ssm_client.put_parameter.assert_not_called() + find_orgs_api.assert_not_called() org_client.create_account.assert_not_called() @patch("main.ORGANIZATION_CLIENT") @patch("main.SSM_CLIENT") -@patch("main.wait_on_account_creation") -@patch("main.handle_concurrent_modification") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") @patch("main.LOGGER") def test_deployment_account_found_with_ssm( - logger, concur_mod_fn, wait_on_fn, ssm_client, org_client + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client ): account_id = "123456789012" given_account_id = "" @@ -81,33 +205,146 @@ def test_deployment_account_found_with_ssm( "Value": account_id, } } + find_orgs_api.return_value = account_id returned_account_id, created = ensure_account( given_account_id, account_name, account_email, cross_account_access_role_name, + is_update=True, ) assert returned_account_id != given_account_id assert returned_account_id == account_id assert not created + logger.info.assert_called_once_with( + 'Using deployment account as specified with param %s : %s.', + DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + account_id, + ) + concur_mod_fn.assert_not_called() + wait_on_fn.assert_not_called() + ssm_client.get_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + ssm_client.put_parameter.assert_not_called() + find_orgs_api.assert_not_called() + org_client.create_account.assert_not_called() + + +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") +@patch("main.LOGGER") +def test_deployment_account_not_found_via_orgs_api_on_update( + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client +): + account_id = "123456789012" + given_account_id = "" + account_name = "test-deployment-account" + account_email = "test@amazon.com" + cross_account_access_role_name = "some-role" + ssm_client.exceptions.ParameterNotFound = ParameterNotFound + org_client.exceptions.ConcurrentModificationException = ( + ConcurrentModificationException + ) + + ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = account_id + + returned_account_id, created = ensure_account( + given_account_id, + account_name, + account_email, + cross_account_access_role_name, + is_update=True, + ) + + assert returned_account_id != given_account_id + assert returned_account_id == account_id + assert not created + logger.info.assert_called_once_with( + "Using deployment account %s as found in AWS Organization %s.", + account_id, + DEPLOYMENT_OU_PATH, + ) + concur_mod_fn.assert_not_called() + wait_on_fn.assert_not_called() + ssm_client.get_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + ) + ssm_client.put_parameter.assert_called_once_with( + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, + Value=account_id, + Description=SSM_PARAMETER_ADF_DESCRIPTION, + Type="String", + Overwrite=True, + ) + find_orgs_api.assert_called_once_with() + org_client.create_account.assert_not_called() + + +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") +@patch("main.LOGGER") +def test_deployment_account_found_via_orgs_api_on_update( + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client +): + given_account_id = "" + account_name = "test-deployment-account" + account_email = "test@amazon.com" + cross_account_access_role_name = "some-role" + ssm_client.exceptions.ParameterNotFound = ParameterNotFound + org_client.exceptions.ConcurrentModificationException = ( + ConcurrentModificationException + ) + + ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = "" + + with pytest.raises(RuntimeError) as exc_info: + ensure_account( + given_account_id, + account_name, + account_email, + cross_account_access_role_name, + is_update=True, + ) + + error_msg = ( + "When updating ADF should not be required to create a deployment " + "account. If your previous installation failed and you try to fix " + "it via an update, please delete the ADF stack first and run it " + "as a fresh installation." + ) + assert str(exc_info.value) == error_msg + logger.info.assert_not_called() + logger.error.assert_called_once_with(error_msg) concur_mod_fn.assert_not_called() wait_on_fn.assert_not_called() ssm_client.get_parameter.assert_called_once_with( - Name="deployment_account_id", + Name=DEPLOYMENT_ACCOUNT_ID_PARAM_PATH, ) + ssm_client.put_parameter.assert_not_called() + find_orgs_api.assert_called_once_with() org_client.create_account.assert_not_called() @patch("main.ORGANIZATION_CLIENT") @patch("main.SSM_CLIENT") -@patch("main.wait_on_account_creation") -@patch("main.handle_concurrent_modification") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") @patch("main.LOGGER") def test_deployment_account_create_success( - logger, concur_mod_fn, wait_on_fn, ssm_client, org_client + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client ): request_id = "random-request-id" account_id = "123456789012" @@ -121,18 +358,20 @@ def test_deployment_account_create_success( ) ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = "" org_client.create_account.return_value = { "CreateAccountStatus": { "Id": request_id, } } - wait_on_fn.return_value = (account_id, True) + wait_on_fn.return_value = account_id returned_account_id, created = ensure_account( given_account_id, account_name, account_email, cross_account_access_role_name, + is_update=False, ) assert returned_account_id != given_account_id @@ -147,8 +386,9 @@ def test_deployment_account_create_success( concur_mod_fn.assert_not_called() wait_on_fn.assert_called_once_with(request_id) ssm_client.get_parameter.assert_called_once_with( - Name="deployment_account_id", + Name="/adf/deployment_account_id", ) + find_orgs_api.assert_not_called() org_client.create_account.assert_called_once_with( Email=account_email, AccountName=account_name, @@ -159,13 +399,13 @@ def test_deployment_account_create_success( @patch("main.ORGANIZATION_CLIENT") @patch("main.SSM_CLIENT") -@patch("main.wait_on_account_creation") -@patch("main.handle_concurrent_modification") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") @patch("main.LOGGER") def test_deployment_account_create_failed_concur( - logger, concur_mod_fn, wait_on_fn, ssm_client, org_client + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client ): - request_id = "random-request-id" account_id = "123456789012" given_account_id = "" account_name = "test-deployment-account" @@ -177,6 +417,7 @@ def test_deployment_account_create_failed_concur( ) ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = "" org_client.create_account.side_effect = ConcurrentModificationException( "Test", ) @@ -187,6 +428,7 @@ def test_deployment_account_create_failed_concur( account_name, account_email, cross_account_access_role_name, + is_update=False, ) assert returned_account_id != given_account_id @@ -199,7 +441,6 @@ def test_deployment_account_create_failed_concur( ) concur_mod_fn.assert_called_once_with( org_client.create_account.side_effect, - given_account_id, account_name, account_email, cross_account_access_role_name, @@ -207,8 +448,9 @@ def test_deployment_account_create_failed_concur( ) wait_on_fn.assert_not_called() ssm_client.get_parameter.assert_called_once_with( - Name="deployment_account_id", + Name="/adf/deployment_account_id", ) + find_orgs_api.assert_not_called() org_client.create_account.assert_called_once_with( Email=account_email, AccountName=account_name, @@ -219,14 +461,13 @@ def test_deployment_account_create_failed_concur( @patch("main.ORGANIZATION_CLIENT") @patch("main.SSM_CLIENT") -@patch("main.wait_on_account_creation") -@patch("main.handle_concurrent_modification") +@patch("main._find_deployment_account_via_orgs_api") +@patch("main._wait_on_account_creation") +@patch("main._handle_concurrent_modification") @patch("main.LOGGER") def test_deployment_account_create_failed_other( - logger, concur_mod_fn, wait_on_fn, ssm_client, org_client + logger, concur_mod_fn, wait_on_fn, find_orgs_api, ssm_client, org_client ): - request_id = "random-request-id" - account_id = "123456789012" given_account_id = "" account_name = "test-deployment-account" account_email = "test@amazon.com" @@ -238,6 +479,7 @@ def test_deployment_account_create_failed_other( ) ssm_client.get_parameter.side_effect = ParameterNotFound("Test") + find_orgs_api.return_value = "" org_client.create_account.side_effect = OtherException(correct_error_message) with pytest.raises(OtherException) as excinfo: @@ -246,6 +488,7 @@ def test_deployment_account_create_failed_other( account_name, account_email, cross_account_access_role_name, + is_update=False, ) error_message = str(excinfo.value) @@ -259,8 +502,9 @@ def test_deployment_account_create_failed_other( concur_mod_fn.assert_not_called() wait_on_fn.assert_not_called() ssm_client.get_parameter.assert_called_once_with( - Name="deployment_account_id", + Name="/adf/deployment_account_id", ) + find_orgs_api.assert_not_called() org_client.create_account.assert_called_once_with( Email=account_email, AccountName=account_name, @@ -280,7 +524,7 @@ def test_deployment_account_wait_exception(logger, time_mock, org_client): ) with pytest.raises(OtherException) as excinfo: - wait_on_account_creation( + _wait_on_account_creation( request_id, ) @@ -294,6 +538,130 @@ def test_deployment_account_wait_exception(logger, time_mock, org_client): ) +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main.TAGGING_CLIENT") +@patch("main.Organizations") +@patch("main.LOGGER") +def test_deployment_account_find_via_orgs_api_one_found( + logger, org_cls, tag_client, ssm_client, org_client +): + account_id = "123456789012" + org_instance = org_cls.return_value + org_instance.get_accounts_in_path.return_value = [ + { + "Id": "111111111111", + "Status": "SUSPENDED", + }, + { + "Id": account_id, + "Status": "ACTIVE", + }, + { + "Id": "111111111111", + "Status": "PENDING_CLOSURE", + }, + ] + + returned_account_id = _find_deployment_account_via_orgs_api() + + assert returned_account_id == account_id + + logger.debug.assert_not_called() + org_cls.assert_called_once_with( + org_client=org_client, + tagging_client=tag_client, + ) + org_instance.get_accounts_in_path.assert_called_once_with( + DEPLOYMENT_OU_PATH, + ) + + +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main.TAGGING_CLIENT") +@patch("main.Organizations") +@patch("main.LOGGER") +def test_deployment_account_find_via_orgs_api_none_found( + logger, org_cls, tag_client, ssm_client, org_client +): + org_instance = org_cls.return_value + org_instance.get_accounts_in_path.return_value = [ + { + "Id": "111111111111", + "Status": "SUSPENDED", + }, + { + "Id": "111111111111", + "Status": "PENDING_CLOSURE", + }, + ] + + returned_account_id = _find_deployment_account_via_orgs_api() + + assert returned_account_id == "" + + logger.debug.assert_called_once_with( + "No active AWS Accounts found in the %s OU path.", + DEPLOYMENT_OU_PATH, + ) + org_cls.assert_called_once_with( + org_client=org_client, + tagging_client=tag_client, + ) + org_instance.get_accounts_in_path.assert_called_once_with( + DEPLOYMENT_OU_PATH, + ) + + +@patch("main.ORGANIZATION_CLIENT") +@patch("main.SSM_CLIENT") +@patch("main.TAGGING_CLIENT") +@patch("main.Organizations") +@patch("main.LOGGER") +def test_deployment_account_find_via_orgs_api_multiple_found( + logger, org_cls, tag_client, ssm_client, org_client +): + org_instance = org_cls.return_value + org_instance.get_accounts_in_path.return_value = [ + { + "Id": "111111111111", + "Status": "ACTIVE", + }, + { + "Id": "222222222222", + "Status": "ACTIVE", + }, + ] + + with pytest.raises(RuntimeError) as exc_info: + _find_deployment_account_via_orgs_api() + + correct_error_message = ( + "Failed to determine Deployment account to setup, as " + f"2 AWS Accounts were found " + f"in the {DEPLOYMENT_OU_PATH} organization unit (OU). " + "Please ensure there is only one account in the " + f"{DEPLOYMENT_OU_PATH} OU path. " + "Move all AWS accounts you don't want to be bootstrapped as " + f"the ADF deployment account out of the {DEPLOYMENT_OU_PATH} " + "OU. In case there are no accounts in the " + f"{DEPLOYMENT_OU_PATH} OU, ADF will automatically create a " + "new AWS account for you, or move the deployment account as " + "specified at install time of ADF to the respective OU." + ) + assert str(exc_info.value) == correct_error_message + + logger.debug.assert_not_called() + org_cls.assert_called_once_with( + org_client=org_client, + tagging_client=tag_client, + ) + org_instance.get_accounts_in_path.assert_called_once_with( + DEPLOYMENT_OU_PATH, + ) + + @patch("main.ORGANIZATION_CLIENT") @patch("main.time") @patch("main.LOGGER") @@ -306,10 +674,10 @@ def test_deployment_account_wait_on_failed(logger, time_mock, org_client): "FailureReason": failure_reason, } } - correct_error_message = "Failed to create account because %s" % failure_reason + correct_error_message = f"Failed to create account because {failure_reason}" with pytest.raises(Exception) as excinfo: - wait_on_account_creation( + _wait_on_account_creation( request_id, ) @@ -343,12 +711,11 @@ def test_deployment_account_wait_in_progress_success(logger, time_mock, org_clie }, ] - returned_account_id, created = wait_on_account_creation( + returned_account_id = _wait_on_account_creation( request_id, ) assert returned_account_id == account_id - assert created logger.info.assert_has_calls( [ @@ -390,9 +757,8 @@ def test_deployment_account_handle_concurrent_last_try( ensure_fn.return_value = (account_id, True) - returned_account_id, created = handle_concurrent_modification( + returned_account_id, created = _handle_concurrent_modification( err, - given_account_id, account_name, account_email, cross_account_access_role_name, @@ -440,9 +806,8 @@ def test_deployment_account_handle_concurrent_too_many_tries( ensure_fn.return_value = (account_id, True) with pytest.raises(ConcurrentModificationException) as excinfo: - handle_concurrent_modification( + _handle_concurrent_modification( err, - given_account_id, account_name, account_email, cross_account_access_role_name, diff --git a/src/lambda_codebase/account_bootstrap.py b/src/lambda_codebase/account_bootstrap.py index 5cc4b0701..a7a610552 100644 --- a/src/lambda_codebase/account_bootstrap.py +++ b/src/lambda_codebase/account_bootstrap.py @@ -62,10 +62,10 @@ def configure_generic_account(sts, event, region, role): role, ) kms_arn = parameter_store_deployment_account.fetch_parameter( - f'/cross_region/kms_arn/{region}', + f'cross_region/kms_arn/{region}', ) bucket_name = parameter_store_deployment_account.fetch_parameter( - f'/cross_region/s3_regional_bucket/{region}', + f'cross_region/s3_regional_bucket/{region}', ) org_stage = parameter_store_deployment_account.fetch_parameter( '/adf/org/stage' @@ -85,17 +85,17 @@ def configure_generic_account(sts, event, region, role): parameter_store_target_account.put_parameter('/adf/org/stage', org_stage) -def configure_master_account_parameters(event): +def configure_management_account_parameters(event): """ Update the management account parameter store in us-east-1 with the deployment_account_id then updates the main deployment region with that same value """ - parameter_store_master_account_region = ParameterStore( + parameter_store_management_account_region = ParameterStore( os.environ["AWS_REGION"], boto3, ) - parameter_store_master_account_region.put_parameter( + parameter_store_management_account_region.put_parameter( 'deployment_account_id', event['account_id'], ) @@ -124,10 +124,7 @@ def configure_deployment_account_parameters(event, role): for region in regions: parameter_store = ParameterStore(region, role) for key, value in event['deployment_account_parameters'].items(): - parameter_store.put_parameter( - key, - value - ) + parameter_store.put_parameter(key, value) def is_inter_ou_account_move(event): @@ -163,7 +160,7 @@ def _lambda_handler(event, context): ) if event['is_deployment_account']: - configure_master_account_parameters(event) + configure_management_account_parameters(event) configure_deployment_account_parameters(event, role) s3 = S3( diff --git a/src/lambda_codebase/account_processing/configure_account_regions.py b/src/lambda_codebase/account_processing/configure_account_regions.py index 0ced5af80..caa0d1f89 100644 --- a/src/lambda_codebase/account_processing/configure_account_regions.py +++ b/src/lambda_codebase/account_processing/configure_account_regions.py @@ -17,7 +17,10 @@ def get_regions_from_ssm(ssm_client): - regions = ssm_client.get_parameter(Name="target_regions")["Parameter"].get("Value") + regions = ( + ssm_client.get_parameter(Name="/adf/target_regions")["Parameter"] + .get("Value") + ) regions = literal_eval(regions) return regions diff --git a/src/lambda_codebase/cross_region_bucket/main.py b/src/lambda_codebase/cross_region_bucket/main.py index 3c241e053..c02f45d8c 100644 --- a/src/lambda_codebase/cross_region_bucket/main.py +++ b/src/lambda_codebase/cross_region_bucket/main.py @@ -136,7 +136,7 @@ def determine_region(event: Mapping[str, Any]): return event["ResourceProperties"]["Region"] try: get_parameter = SSM_CLIENT.get_parameter( - Name="deployment_account_region", + Name="/adf/deployment_account_region", ) return get_parameter["Parameter"]["Value"] except SSM_CLIENT.exceptions.ParameterNotFound: @@ -148,7 +148,9 @@ def determine_region(event: Mapping[str, Any]): def ensure_bucket(region: str, bucket_name_prefix: str) -> Tuple[BucketName, Created]: try: - get_parameter = SSM_CLIENT.get_parameter(Name="shared_modules_bucket") + get_parameter = SSM_CLIENT.get_parameter( + Name="/adf/shared_modules_bucket", + ) return get_parameter["Parameter"]["Value"], False except SSM_CLIENT.exceptions.ParameterNotFound: pass # Carry on with creating the bucket diff --git a/src/lambda_codebase/deployment_account_config.py b/src/lambda_codebase/deployment_account_config.py index c5d77ba38..61b32ca3c 100644 --- a/src/lambda_codebase/deployment_account_config.py +++ b/src/lambda_codebase/deployment_account_config.py @@ -21,7 +21,7 @@ from s3 import S3 S3_BUCKET = os.environ["S3_BUCKET_NAME"] -MASTER_ACCOUNT_ID = os.environ["MASTER_ACCOUNT_ID"] +MANAGEMENT_ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] REGION_DEFAULT = os.environ["AWS_REGION"] diff --git a/src/lambda_codebase/event.py b/src/lambda_codebase/event.py index 429c5b1f8..c16e323f5 100644 --- a/src/lambda_codebase/event.py +++ b/src/lambda_codebase/event.py @@ -27,9 +27,7 @@ class Event: def __init__(self, event, parameter_store, organizations, account_id): self.parameter_store = parameter_store self.config = ast.literal_eval( - parameter_store.fetch_parameter( - 'config' - ) + parameter_store.fetch_parameter('config'), ) self.account_id = account_id self.organizations = organizations @@ -123,28 +121,20 @@ def create_output_object(self, account_path): 'account_id': self.account_id, 'cross_account_access_role': self.cross_account_access_role, 'deployment_account_id': self.deployment_account_id, - 'regions': self.regions, - 'deployment_account_region': self.deployment_account_region, - 'moved_to_root': self.moved_to_root, - 'moved_to_protected': self.moved_to_protected, - 'is_deployment_account': self.is_deployment_account, - 'ou_name': self.destination_ou_name, - 'full_path': "ROOT" if self.moved_to_root else account_path, - 'destination_ou_id': self.destination_ou_id, - 'source_ou_id': self.source_ou_id, 'deployment_account_parameters': { - 'organization_id': organization_information.get( - "organization_id" - ), - 'master_account_id': organization_information.get( - "organization_master_account_id" + 'adf_log_level': ADF_LOG_LEVEL, + 'adf_version': ADF_VERSION, + 'cross_account_access_role': self.cross_account_access_role, + 'deployment_account_bucket': DEPLOYMENT_ACCOUNT_S3_BUCKET, + 'deployment_account_id': self.deployment_account_id, + 'management_account_id': organization_information.get( + "organization_management_account_id" ), 'notification_endpoint': self.main_notification_endpoint, 'notification_type': self.notification_type, - 'cross_account_access_role': self.cross_account_access_role, - 'deployment_account_bucket': DEPLOYMENT_ACCOUNT_S3_BUCKET, - 'adf_version': ADF_VERSION, - 'adf_log_level': ADF_LOG_LEVEL, + 'organization_id': organization_information.get( + "organization_id" + ), 'extensions/terraform/enabled': ( self._read_parameter( 'extensions/terraform/enabled', @@ -152,4 +142,13 @@ def create_output_object(self, account_path): ) ), }, + 'deployment_account_region': self.deployment_account_region, + 'destination_ou_id': self.destination_ou_id, + 'full_path': "ROOT" if self.moved_to_root else account_path, + 'is_deployment_account': self.is_deployment_account, + 'moved_to_protected': self.moved_to_protected, + 'moved_to_root': self.moved_to_root, + 'ou_name': self.destination_ou_name, + 'regions': self.regions, + 'source_ou_id': self.source_ou_id, } diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index a984cf3e3..9f16c5655 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -8,27 +8,27 @@ Description: ADF CloudFormation Template (Global) for Deployment Account Parameters: ADFVersion: Type: "AWS::SSM::Parameter::Value" - Default: adf_version + Default: /adf/adf_version ADFLogLevel: Type: "AWS::SSM::Parameter::Value" - Default: adf_log_level + Default: /adf/adf_log_level - MasterAccountId: + ManagementAccountId: Type: "AWS::SSM::Parameter::Value" - Default: master_account_id + Default: /adf/management_account_id SharedModulesBucket: Type: "AWS::SSM::Parameter::Value" - Default: deployment_account_bucket + Default: /adf/deployment_account_bucket OrganizationId: Type: "AWS::SSM::Parameter::Value" - Default: organization_id + Default: /adf/organization_id CrossAccountAccessRole: Type: "AWS::SSM::Parameter::Value" - Default: cross_account_access_role + Default: /adf/cross_account_access_role Image: Description: The Image you wish to use for CodeBuild (defaults to Ubuntu - standard:7.0). @@ -46,11 +46,11 @@ Parameters: NotificationEndpoint: Type: "AWS::SSM::Parameter::Value" - Default: notification_endpoint + Default: /adf/notification_endpoint NotificationType: Type: "AWS::SSM::Parameter::Value" - Default: notification_type + Default: /adf/notification_type PipelinePrefix: Description: The Prefix that will be attached to pipeline stacks and names for ADF. @@ -84,7 +84,7 @@ Resources: ContentUri: "../../adf-build/shared/python" CompatibleRuntimes: - python3.12 - Description: "Shared Lambda Layer between master and deployment account" + Description: "Shared Lambda Layer between management and deployment account" LayerName: adf_shared_layer Metadata: BuildMethod: python3.12 @@ -187,7 +187,7 @@ Resources: OrganizationId: !Ref OrganizationId CrossAccountAccessRole: !Ref CrossAccountAccessRole PipelineBucket: !Ref PipelineBucket - RootAccountId: !Ref MasterAccountId + RootAccountId: !Ref ManagementAccountId CodeBuildImage: !Ref Image CodeBuildComputeType: !Ref ComputeType SharedModulesBucket: !Ref SharedModulesBucket @@ -335,14 +335,14 @@ Resources: - "ssm:GetParameter" - "ssm:GetParameters" Resource: - - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/* - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}::parameter/aws/service/* - Effect: Deny Action: - "sts:AssumeRole" Resource: - !GetAtt CloudFormationDeploymentRole.Arn - - !Sub arn:${AWS::Partition}:iam::${MasterAccountId}:role/${CrossAccountAccessRole} + - !Sub arn:${AWS::Partition}:iam::${ManagementAccountId}:role/${CrossAccountAccessRole} - Effect: Allow Action: - "logs:CreateLogGroup" @@ -636,9 +636,9 @@ Resources: - "ssm:GetParameters" - "ssm:GetParameter" Resource: - - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/bucket_name" - - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/deployment_account_id" - - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/kms_arn" + - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/adf/bucket_name" + - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/adf/deployment_account_id" + - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/adf/kms_arn" Roles: - !Ref AdfAutomationRole @@ -1087,7 +1087,7 @@ Resources: Action: - "ssm:GetParameter" Resource: - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/notification_endpoint/*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/notification_endpoint/*" - Effect: Allow Action: - "secretsmanager:GetSecretValue" @@ -1151,8 +1151,8 @@ Resources: Action: - "ssm:GetParameter" Resource: - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/cross_region/kms_arn/*" - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/cross_region/s3_regional_bucket/*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/cross_region/kms_arn/*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/cross_region/s3_regional_bucket/*" - Effect: Allow Action: - "codepipeline:GetPipelineState" @@ -1292,6 +1292,7 @@ Resources: } }, "ItemsPath": "$.account_ids", + "MaxConcurrency": 1, "Parameters": { "deployment_account_region.$": "$.deployment_account_region", "deployment_account_id.$": "$.deployment_account_id", @@ -1428,7 +1429,7 @@ Resources: KmsKeyArnParameter: Type: "AWS::SSM::Parameter" Properties: - Name: "kms_arn" + Name: "/adf/kms_arn" Type: "String" Value: !GetAtt KMSKey.Arn diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/enable_cross_account_access.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/enable_cross_account_access.py index c0b1f560d..394a8fb09 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/enable_cross_account_access.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/enable_cross_account_access.py @@ -113,11 +113,11 @@ def lambda_handler(event, _): ) ): kms_key_arn = parameter_store.fetch_parameter( - f"/cross_region/kms_arn/{region}" + f"cross_region/kms_arn/{region}" ) kms_key_arns.append(kms_key_arn) s3_bucket = parameter_store.fetch_parameter( - f"/cross_region/s3_regional_bucket/{region}" + f"cross_region/s3_regional_bucket/{region}" ) s3_buckets.append(s3_bucket) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py index 36e953bfb..91059b6d9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_or_update_rule.py @@ -76,7 +76,7 @@ def lambda_handler(event, _): try: parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) default_scm_codecommit_account_id = parameter_store.fetch_parameter( - "/adf/scm/default-scm-codecommit-account-id", + "scm/default_scm_codecommit_account_id", ) except ParameterNotFoundError: default_scm_codecommit_account_id = deployment_account_id diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py index dc5b8c6f1..ed4bba604 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/create_repository.py @@ -45,7 +45,7 @@ def lambda_handler(event, _): parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) auto_create_repositories = parameter_store.fetch_parameter( - "auto_create_repositories" + "scm/auto_create_repositories" ) LOGGER.debug("Auto create repositories is: %s", auto_create_repositories) if auto_create_repositories != "enabled": diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py index 9bb0ce485..d9dab9926 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/generate_pipeline_inputs.py @@ -33,7 +33,7 @@ def store_regional_parameter_config( These are only used to track pipelines. """ parameter_store.put_parameter( - f"/deployment/{deployment_map_source}/{pipeline.name}/regions", + f"deployment/{deployment_map_source}/{pipeline.name}/regions", str(pipeline.get_all_regions()), ) @@ -59,10 +59,10 @@ def fetch_required_ssm_params(pipeline_input, regions): parameter_store = ParameterStore(region, boto3) output[region] = { "s3": parameter_store.fetch_parameter( - f"/cross_region/s3_regional_bucket/{region}", + f"cross_region/s3_regional_bucket/{region}", ), "kms": parameter_store.fetch_parameter( - f"/cross_region/kms_arn/{region}", + f"cross_region/kms_arn/{region}", ), } if region == DEPLOYMENT_ACCOUNT_REGION: @@ -70,10 +70,10 @@ def fetch_required_ssm_params(pipeline_input, regions): "deployment_account_bucket" ) output["default_scm_branch"] = parameter_store.fetch_parameter( - "default_scm_branch", + "scm/default_scm_branch", ) output["default_scm_codecommit_account_id"] = parameter_store.fetch_parameter( - "/adf/scm/default-scm-codecommit-account-id", + "scm/default_scm_codecommit_account_id", ) codestar_connection_path = ( pipeline_input diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py index e83a95817..b13fea98d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/identify_out_of_date_pipelines.py @@ -21,8 +21,8 @@ S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"] ADF_PIPELINE_PREFIX = os.environ["ADF_PIPELINE_PREFIX"] DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] -DEPLOYMENT_PREFIX = "/deployment/" -S3_BACKED_DEPLOYMENT_PREFIX = f"{DEPLOYMENT_PREFIX}S3/" +DEPLOYMENT_PREFIX = "deployment" +S3_BACKED_DEPLOYMENT_PREFIX = f"{DEPLOYMENT_PREFIX}/S3/" def download_deployment_maps(s3_resource, prefix, local): diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/templates/events.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/templates/events.yml index a4f534ca2..b4f2f18f2 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/templates/events.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/templates/events.yml @@ -5,7 +5,8 @@ Parameters: DeploymentAccountId: Type: "AWS::SSM::Parameter::Value" Description: Deployment Account ID - Default: deployment_account_id + Default: /adf/deployment_account_id + Resources: EventRole: Type: AWS::IAM::Role @@ -27,6 +28,7 @@ Resources: - Effect: Allow Action: events:PutEvents Resource: '*' + EventRule: Type: AWS::Events::Rule Properties: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py index 71ea847d4..113ce3b34 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/slack.py @@ -191,7 +191,7 @@ def lambda_handler(event, _): region_name=os.environ["AWS_REGION"], ) channel = parameter_store.fetch_parameter( - name=f'/notification_endpoint/{pipeline["name"]}', + name=f'notification_endpoint/{pipeline["name"]}', with_decryption=False, ) # All slack url's must be stored in /adf/slack/channel_name since ADF only diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml index 4f3e9d650..0bf5f1d68 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/pipeline_management.yml @@ -195,10 +195,8 @@ Resources: - Effect: Allow Action: - "ssm:GetParameter" - - "ssm:GetParameters" - - "ssm:GetParametersByPath" Resource: - - "*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/scm/auto_create_repositories" GeneratePipelineInputsLambdaRole: Type: "AWS::IAM::Role" @@ -226,13 +224,16 @@ Resources: - !Sub "arn:${AWS::Partition}:iam::${RootAccountId}:role/${CrossAccountAccessRole}-readonly" - Effect: Allow Action: - - "ssm:DeleteParameter" - "ssm:GetParameter" - "ssm:GetParameters" - "ssm:GetParametersByPath" + Resource: + - !Sub "arn:${AWS::Partition}:ssm:*:${AWS::AccountId}:parameter/adf/*" + - Effect: Allow + Action: - "ssm:PutParameter" Resource: - - "*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/deployment/*" StoreDefinitionLambdaRole: Type: "AWS::IAM::Role" @@ -285,10 +286,9 @@ Resources: - !Sub "${ADFPipelineBucket.Arn}" - Effect: Allow Action: - - "ssm:DeleteParameter" - "ssm:GetParametersByPath" Resource: - - "*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/deployment/*" StateMachineExecutionRole: Type: "AWS::IAM::Role" @@ -380,7 +380,7 @@ Resources: Action: - "ssm:DeleteParameter" Resource: - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/deployment/*" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/deployment/*" PipelineManagementStateMachine: Type: "AWS::StepFunctions::StateMachine" @@ -615,7 +615,7 @@ Resources: "DeletePipelineParams": { "Type": "Task", "Parameters": { - "Name.$": "States.Format('/deployment/S3/{}/regions', $.pipeline_name)" + "Name.$": "States.Format('/adf/deployment/S3/{}/regions', $.pipeline_name)" }, "Catch": [ { @@ -657,7 +657,7 @@ Resources: Value: "./adf-build/:./adf-build/python/" - Name: ACCOUNT_ID Value: !Ref AWS::AccountId - - Name: MASTER_ACCOUNT_ID + - Name: MANAGEMENT_ACCOUNT_ID Value: !Ref RootAccountId - Name: S3_BUCKET_NAME Value: !Ref PipelineBucket diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml index af76f4c62..3b7bb0e77 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml @@ -7,7 +7,7 @@ Description: ADF CloudFormation Template (Regional) for Deployment Account Parameters: OrganizationId: Type: "AWS::SSM::Parameter::Value" - Default: organization_id + Default: /adf/organization_id ADFTerraformExtension: Type: "AWS::SSM::Parameter::Value" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml index 338342109..44499dd09 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml @@ -8,7 +8,7 @@ Parameters: DeploymentAccountId: Type: "AWS::SSM::Parameter::Value" Description: Deployment Account ID - Default: deployment_account_id + Default: /adf/deployment_account_id # OrgStage can be set in the respective adfconfig file using the # path config.org.stage @@ -51,12 +51,7 @@ Resources: # These below actions are examples, change these to your requirements.. - "apigateway:*" - "cloudformation:*" # You will need CloudFormation actions in order to work with CloudFormation - - "ecr:*" - - "ecs:*" - - "ec2:*" - - "iam:*" - "logs:*" - - "s3:*" - "codedeploy:*" - "autoscaling:*" - "cloudwatch:*" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml index a323a94a8..bac3f6ea5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml @@ -10,17 +10,17 @@ Parameters: KMSArn: Type: "AWS::SSM::Parameter::Value" Description: ARN of the KMS CMK created in the Deployment account - Default: kms_arn + Default: /adf/kms_arn DeploymentAccountId: Type: "AWS::SSM::Parameter::Value" Description: Deployment Account ID - Default: deployment_account_id + Default: /adf/deployment_account_id DeploymentAccountBucketName: Type: "AWS::SSM::Parameter::Value" Description: Deployment Bucket Name - Default: bucket_name + Default: /adf/bucket_name Resources: CodeCommitRole: @@ -335,9 +335,9 @@ Resources: - "ssm:GetParameters" - "ssm:GetParameter" Resource: - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/bucket_name" - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/deployment_account_id" - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/kms_arn" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/bucket_name" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/deployment_account_id" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/kms_arn" - Effect: Allow Sid: "IAM" Action: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py index f5c028575..99d3e33a5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py @@ -149,12 +149,21 @@ def _store_cross_region_config(self): in Parameter Store on the management account in deployment account main region. """ + deployment_account_id = self.parameters_client.fetch_parameter( + 'deployment_account_id', + ) + self.client_deployment_region = ParameterStore( self.deployment_account_region, boto3 ) self.client_deployment_region.put_parameter("adf_version", ADF_VERSION) self.client_deployment_region.put_parameter( - "cross_account_access_role", self.cross_account_access_role + "deployment_account_id", + deployment_account_id, + ) + self.client_deployment_region.put_parameter( + "cross_account_access_role", + self.cross_account_access_role, ) def _store_config(self): @@ -178,6 +187,6 @@ def _store_config(self): for extension, attributes in self.extensions.items(): for attribute in attributes: self.parameters_client.put_parameter( - f"/adf/extensions/{extension}/{attribute}", + f"extensions/{extension}/{attribute}", str(attributes[attribute]), ) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml index cd78d0ceb..426e6a473 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml @@ -9,11 +9,12 @@ Parameters: DeploymentAccountId: Type: "AWS::SSM::Parameter::Value" Description: Deployment Account ID - Default: deployment_account_id + Default: /adf/deployment_account_id + CrossAccountAccessRole: Type: "AWS::SSM::Parameter::Value" Description: The role used to allow cross account access - Default: cross_account_access_role + Default: /adf/cross_account_access_role Resources: OrganizationsReadOnlyRole: @@ -74,7 +75,7 @@ Resources: # Only required if you intend to bootstrap the management account. Type: AWS::IAM::Policy Properties: - PolicyName: "adf-master-account-bootstrap-policy" + PolicyName: "adf-management-account-bootstrap-policy" PolicyDocument: Version: "2012-10-17" Statement: @@ -113,11 +114,15 @@ Resources: - iam:CreateAccountAlias - iam:DeleteAccountAlias - iam:ListAccountAliases + Resource: + - "*" + - Effect: Allow + Action: - ssm:PutParameter - ssm:GetParameters - ssm:GetParameter Resource: - - "*" + - !Sub "arn:${AWS::Partition}:ssm:*:${AWS::AccountId}:parameter/adf/*" - Effect: Allow Action: - iam:CreateRole diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index 69841139d..c4ce3777f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -33,7 +33,7 @@ S3_BUCKET_NAME = os.environ["S3_BUCKET"] REGION_DEFAULT = os.environ["AWS_REGION"] PARTITION = get_partition(REGION_DEFAULT) -ACCOUNT_ID = os.environ["MASTER_ACCOUNT_ID"] +ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] ADF_VERSION = os.environ["ADF_VERSION"] ADF_LOG_LEVEL = os.environ["ADF_LOG_LEVEL"] DEPLOYMENT_ACCOUNT_S3_BUCKET_NAME = os.environ["DEPLOYMENT_ACCOUNT_BUCKET"] @@ -55,7 +55,7 @@ ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN = os.environ.get( "ACCOUNT_BOOTSTRAPPING_STATE_MACHINE_ARN" ) -ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'master' +ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'main' ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET = False ADF_DEFAULT_ORG_STAGE = "none" LOGGER = configure_logger(__name__) @@ -100,11 +100,11 @@ def update_deployment_account_output_parameters( ) for key, value in outputs.items(): deployment_account_parameter_store.put_parameter( - f"/cross_region/{key}/{region}", + f"cross_region/{key}/{region}", value ) parameter_store.put_parameter( - f"/cross_region/{key}/{region}", + f"cross_region/{key}/{region}", value ) @@ -120,7 +120,7 @@ def prepare_deployment_account(sts, deployment_account_id, config): deployment_account_role = sts.assume_cross_account_role( f'arn:{PARTITION}:iam::{deployment_account_id}:role/' f'{config.cross_account_access_role}', - 'master' + 'management' ) for region in sorted(list( set([config.deployment_account_region] + config.target_regions))): @@ -129,25 +129,48 @@ def prepare_deployment_account(sts, deployment_account_id, config): deployment_account_role ) deployment_account_parameter_store.put_parameter( - 'organization_id', os.environ["ORGANIZATION_ID"] + 'adf_version', + ADF_VERSION, + ) + deployment_account_parameter_store.put_parameter( + 'adf_log_level', + ADF_LOG_LEVEL, + ) + deployment_account_parameter_store.put_parameter( + 'cross_account_access_role', + config.cross_account_access_role, + ) + deployment_account_parameter_store.put_parameter( + 'deployment_account_bucket', + DEPLOYMENT_ACCOUNT_S3_BUCKET_NAME, + ) + deployment_account_parameter_store.put_parameter( + 'deployment_account_id', + deployment_account_id, + ) + deployment_account_parameter_store.put_parameter( + 'management_account_id', + ACCOUNT_ID, + ) + deployment_account_parameter_store.put_parameter( + 'organization_id', + os.environ["ORGANIZATION_ID"], ) _store_extension_parameters(deployment_account_parameter_store, config) + # In main deployment region only: deployment_account_parameter_store = ParameterStore( config.deployment_account_region, deployment_account_role ) + auto_create_repositories = config.config.get( + 'scm', {}).get('auto-create-repositories') + if auto_create_repositories is not None: + deployment_account_parameter_store.put_parameter( + 'scm/auto_create_repositories', str(auto_create_repositories) + ) deployment_account_parameter_store.put_parameter( - 'adf_version', ADF_VERSION - ) - deployment_account_parameter_store.put_parameter( - 'adf_log_level', ADF_LOG_LEVEL - ) - deployment_account_parameter_store.put_parameter( - 'deployment_account_bucket', DEPLOYMENT_ACCOUNT_S3_BUCKET_NAME - ) - deployment_account_parameter_store.put_parameter( - 'default_scm_branch', + 'scm/default_scm_branch', ( config.config .get('scm', {}) @@ -155,7 +178,7 @@ def prepare_deployment_account(sts, deployment_account_id, config): ) ) deployment_account_parameter_store.put_parameter( - '/adf/scm/default-scm-codecommit-account-id', + 'scm/default_scm_codecommit_account_id', ( config.config .get('scm', {}) @@ -163,25 +186,19 @@ def prepare_deployment_account(sts, deployment_account_id, config): ) ) deployment_account_parameter_store.put_parameter( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', config.config.get('deployment-maps', {}).get( 'allow-empty-target', str(ADF_DEFAULT_DEPLOYMENT_MAPS_ALLOW_EMPTY_TARGET), ) ) deployment_account_parameter_store.put_parameter( - '/adf/org/stage', + 'org/stage', config.config.get('org', {}).get( 'stage', ADF_DEFAULT_ORG_STAGE, ) ) - auto_create_repositories = config.config.get( - 'scm', {}).get('auto-create-repositories') - if auto_create_repositories is not None: - deployment_account_parameter_store.put_parameter( - 'auto_create_repositories', str(auto_create_repositories) - ) if '@' not in config.notification_endpoint: config.notification_channel = config.notification_endpoint config.notification_endpoint = ( @@ -189,7 +206,6 @@ def prepare_deployment_account(sts, deployment_account_id, config): f"{deployment_account_id}:function:SendSlackNotification" ) for item in ( - 'cross_account_access_role', 'notification_type', 'notification_endpoint', 'notification_channel' @@ -197,13 +213,12 @@ def prepare_deployment_account(sts, deployment_account_id, config): if getattr(config, item) is not None: deployment_account_parameter_store.put_parameter( ( - '/notification_endpoint/main' + 'notification_endpoint/main' if item == 'notification_channel' else item ), str(getattr(config, item)) ) - _store_extension_parameters(deployment_account_parameter_store, config) return deployment_account_role @@ -215,7 +230,7 @@ def _store_extension_parameters(parameter_store, config): for extension, attributes in config.extensions.items(): for attribute in attributes: parameter_store.put_parameter( - f"/adf/extensions/{extension}/{attribute}", + f"extensions/{extension}/{attribute}", str(attributes[attribute]), ) @@ -223,11 +238,12 @@ def _store_extension_parameters(parameter_store, config): # pylint: disable=too-many-locals def worker_thread( account_id, + deployment_account_id, sts, config, s3, cache, - updated_kms_bucket_dict + updated_kms_bucket_dict, ): """ The Worker thread function that is created for each account @@ -262,6 +278,10 @@ def worker_thread( # Ensuring the kms_arn and bucket_name on the target account is # up-to-date parameter_store = ParameterStore(region, role) + parameter_store.put_parameter( + 'deployment_account_id', + deployment_account_id, + ) parameter_store.put_parameter( 'kms_arn', updated_kms_bucket_dict[region]['kms'], @@ -273,7 +293,7 @@ def worker_thread( # Ensuring the stage parameter on the target account is up-to-date parameter_store.put_parameter( - '/adf/org/stage', + 'org/stage', config.config.get('org', {}).get( 'stage', ADF_DEFAULT_ORG_STAGE, @@ -338,7 +358,8 @@ def await_sfn_executions(sfn_client): "Account Management State Machine encountered a failed, " "timed out, or aborted execution. Please look into this problem " "before retrying the bootstrap pipeline. You can navigate to: " - "https://%s.console.aws.amazon.com/states/home?region=%s#/statemachines/view/%s", + "https://%s.console.aws.amazon.com/states/home" + "?region=%s#/statemachines/view/%s", REGION_DEFAULT, REGION_DEFAULT, ACCOUNT_MANAGEMENT_STATE_MACHINE_ARN, @@ -515,6 +536,7 @@ def main(): # pylint: disable=R0915 for account_id in non_deployment_account_ids: thread = PropagatingThread(target=worker_thread, args=( account_id, + deployment_account_id, sts, config, s3, diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py index 483eda0e6..55e4dd167 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/organization_policy.py @@ -141,7 +141,7 @@ def apply( _policies = OrganizationPolicy._find_all(policy) try: current_stored_policy = ast.literal_eval( - parameter_store.fetch_parameter(policy) + parameter_store.fetch_parameter(policy.replace('-', '_')) ) for stored_policy in current_stored_policy: path = ( @@ -236,4 +236,9 @@ def apply( _type, ) organizations.attach_policy(policy_id, organization_mapping[path]) - parameter_store.put_parameter(policy, str(_policies)) + parameter_store.put_parameter( + # Make the key consistently use underscores instead of dashes. + # So tagging-policy gets changed into tagging_policy. + policy.replace('-', '_'), + str(_policies), + ) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py index 3df0b2f2a..530a7ef0c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codepipeline.py @@ -24,7 +24,7 @@ ADF_STACK_PREFIX = os.environ.get("ADF_STACK_PREFIX", "") ADF_PIPELINE_PREFIX = os.environ.get("ADF_PIPELINE_PREFIX", "") ADF_DEFAULT_BUILD_TIMEOUT = 20 -ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'master' +ADF_DEFAULT_SCM_FALLBACK_BRANCH = 'main' ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] @@ -76,7 +76,7 @@ def __init__(self, **kwargs): ADF_DEFAULT_SCM_FALLBACK_BRANCH, ) self.default_scm_codecommit_account_id = self.map_params.get( - "/adf/scm/default_scm_codecommit_account_id", + "scm/default_scm_codecommit_account_id", ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID, ) self.configuration = self._generate_configuration() @@ -759,7 +759,7 @@ def __init__( ADF_DEFAULT_SCM_FALLBACK_BRANCH, ) self.default_scm_codecommit_account_id = map_params.get( - "/adf/scm/default_scm_codecommit_account_id", + "scm/default_scm_codecommit_account_id", ADF_DEFAULT_SCM_CODECOMMIT_ACCOUNT_ID, ) self.cfn = _codepipeline.CfnPipeline( diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py index 23a10e14f..51493a2ad 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_github.py @@ -53,7 +53,7 @@ def create_webhook_when_required(scope, pipeline, map_params): .get('properties', {}) .get('branch') ) - or 'master' + or 'main' ) _codepipeline.CfnWebhook( scope, diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/clean_pipelines.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/clean_pipelines.py index 90503c990..22b0e8ca0 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/clean_pipelines.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/clean_pipelines.py @@ -23,7 +23,7 @@ LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] -MASTER_ACCOUNT_ID = os.environ["MASTER_ACCOUNT_ID"] +MANAGEMENT_ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] ORGANIZATION_ID = os.environ["ORGANIZATION_ID"] ADF_PIPELINE_PREFIX = os.environ["ADF_PIPELINE_PREFIX"] SHARED_MODULES_BUCKET = os.environ["SHARED_MODULES_BUCKET"] @@ -37,7 +37,7 @@ def clean(parameter_store, deployment_map): Deployment Pipelines that are no longer in the Deployment Map """ current_pipeline_parameters = parameter_store.fetch_parameters_by_path( - "/deployment/" + "deployment/" ) parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/execute_pipeline_stacks.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/execute_pipeline_stacks.py index 8e6ae7315..6f55e9f76 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/execute_pipeline_stacks.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/execute_pipeline_stacks.py @@ -23,7 +23,7 @@ LOGGER = configure_logger(__name__) DEPLOYMENT_ACCOUNT_REGION = os.environ["AWS_REGION"] DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] -MASTER_ACCOUNT_ID = os.environ["MASTER_ACCOUNT_ID"] +MANAGEMENT_ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] ORGANIZATION_ID = os.environ["ORGANIZATION_ID"] S3_BUCKET_NAME = os.environ["S3_BUCKET_NAME"] ADF_PIPELINE_PREFIX = os.environ["ADF_PIPELINE_PREFIX"] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py index 0500a6828..e9d858845 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/generate_params.py @@ -461,7 +461,7 @@ def main() -> None: """ parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) definition_bucket_name = parameter_store.fetch_parameter( - "/adf/pipeline_definition_bucket", + "pipeline_definition_bucket", ) definition_s3 = S3(DEPLOYMENT_ACCOUNT_REGION, definition_bucket_name) parameters = Parameters( diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh index 2f53d7150..c325be39e 100755 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh @@ -34,7 +34,7 @@ fi # Get list of regions supported by this application echo "Determine which regions need to be prepared" -app_regions=`aws ssm get-parameters --names /deployment/$ADF_DEPLOYMENT_MAP_SOURCE/$ADF_PROJECT_NAME/regions --with-decryption --output=text --query='Parameters[0].Value'` +app_regions=`aws ssm get-parameters --names /adf/deployment/$ADF_DEPLOYMENT_MAP_SOURCE/$ADF_PROJECT_NAME/regions --with-decryption --output=text --query='Parameters[0].Value'` # Convert json list to bash list (space delimited regions) regions="`echo $app_regions | sed -e 's/\[\([^]]*\)\]/\1/g' | sed 's/,/ /g' | sed "s/'//g"`" @@ -42,7 +42,7 @@ for region in $regions do if [ $CONTAINS_TRANSFORM ]; then echo "Packaging templates for region $region" - ssm_bucket_name="/cross_region/s3_regional_bucket/$region" + ssm_bucket_name="/adf/cross_region/s3_regional_bucket/$region" bucket=`aws ssm get-parameters --names $ssm_bucket_name --with-decryption --output=text --query='Parameters[0].Value'` sam package --s3-bucket $bucket --output-template-file $CODEBUILD_SRC_DIR/template_$region.yml --region $region else diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index efcc64dcd..d9635ab73 100755 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -7,7 +7,7 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store - S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) + S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/adf/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit cp -R "${CURRENT}"/tf/. "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 91fab866f..5d6fd2376 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -24,7 +24,7 @@ PARTITION = get_partition(REGION_DEFAULT) sts = boto3.client('sts') ssm = boto3.client('ssm') -response = ssm.get_parameter(Name='cross_account_access_role') +response = ssm.get_parameter(Name='/adf/cross_account_access_role') CROSS_ACCOUNT_ACCESS_ROLE = response['Parameter']['Value'] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/deployment_map.py index fd60a1b88..aee2077a8 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/deployment_map.py @@ -62,7 +62,7 @@ def update_deployment_parameters(self, pipeline): ) if pipeline.notification_endpoint: self.parameter_store.put_parameter( - f"/notification_endpoint/{pipeline.name}", + f"notification_endpoint/{pipeline.name}", str(pipeline.notification_endpoint) ) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py index 1f823b89e..f3d6907fc 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/organizations.py @@ -304,7 +304,7 @@ def get_accounts(self, protected_ou_ids=None, include_root=True): def get_organization_info(self): response = self.client.describe_organization() return { - "organization_master_account_id": ( + "organization_management_account_id": ( response .get("Organization") .get("MasterAccountId") diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py index 983827b23..690656dbf 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/parameter_store.py @@ -13,6 +13,7 @@ LOGGER = configure_logger(__name__) PARAMETER_DESCRIPTION = 'DO NOT EDIT - Used by The AWS Deployment Framework' +PARAMETER_PREFIX = '/adf' SSM_CONFIG = Config( retries={ "max_attempts": 10, @@ -36,13 +37,18 @@ def put_parameter(self, name, value, tier='Standard'): LOGGER.debug( 'No need to update parameter %s with value %s since they ' 'are the same', - name, + ParameterStore._build_param_name(name), value, ) except (ParameterNotFoundError, AssertionError): - LOGGER.debug('Putting SSM Parameter %s with value %s', name, value) + param_name = ParameterStore._build_param_name(name) + LOGGER.debug( + 'Putting SSM Parameter %s with value %s', + param_name, + value, + ) self.client.put_parameter( - Name=name, + Name=param_name, Description=PARAMETER_DESCRIPTION, Value=value, Type='String', @@ -51,44 +57,60 @@ def put_parameter(self, name, value, tier='Standard'): ) def delete_parameter(self, name): + param_name = ParameterStore._build_param_name(name) try: - LOGGER.debug('Deleting Parameter %s', name) + LOGGER.debug('Deleting Parameter %s', param_name) self.client.delete_parameter( - Name=name + Name=param_name, ) except self.client.exceptions.ParameterNotFound: LOGGER.debug( 'Attempted to delete Parameter %s but it was not found', - name, + param_name, ) def fetch_parameters_by_path(self, path): """Gets a Parameter(s) by Path from Parameter Store (Recursively) """ + param_path = ParameterStore._build_param_name(path) try: - LOGGER.debug('Fetching Parameters from path %s', path) + LOGGER.debug( + 'Fetching Parameters from path %s', + param_path, + ) return paginator( self.client.get_parameters_by_path, - Path=path, + Path=param_path, Recursive=True, WithDecryption=False ) except self.client.exceptions.ParameterNotFound as error: raise ParameterNotFoundError( - f'Parameter Path {path} Not Found', + f'Parameter Path {param_path} Not Found', ) from error - def fetch_parameter(self, name, with_decryption=False): + @staticmethod + def _build_param_name(name, adf_only=True): + slash_name = name if name.startswith('/') else f"/{name}" + add_prefix = ( + adf_only + and not slash_name.startswith(PARAMETER_PREFIX) + ) + param_prefix = PARAMETER_PREFIX if add_prefix else '' + return f"{param_prefix}{slash_name}" + + def fetch_parameter(self, name, with_decryption=False, adf_only=True): """Gets a Parameter from Parameter Store (Returns the Value) """ + param_name = ParameterStore._build_param_name(name, adf_only) try: - LOGGER.debug('Fetching Parameter %s', name) + LOGGER.debug('Fetching Parameter %s', param_name) response = self.client.get_parameter( - Name=name, + Name=param_name, WithDecryption=with_decryption ) return response['Parameter']['Value'] except self.client.exceptions.ParameterNotFound as error: raise ParameterNotFoundError( - f'Parameter {name} Not Found', + f'Parameter {param_name} Not Found', ) from error diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py index 8ad23324b..303fc2bda 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/target.py @@ -151,7 +151,7 @@ def __init__( # Set adf_deployment_maps_allow_empty_target as bool parameter_store = ParameterStore(DEPLOYMENT_ACCOUNT_REGION, boto3) adf_deployment_maps_allow_empty_target_bool = parameter_store.fetch_parameter( - "/adf/deployment-maps/allow-empty-target" + "deployment_maps/allow_empty_target" ).lower().capitalize() == "True" self.adf_deployment_maps_allow_empty_target = adf_deployment_maps_allow_empty_target_bool diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_organizations.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_organizations.py index c72a11a3b..dd7a38c2c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_organizations.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/stubs/stub_organizations.py @@ -11,7 +11,7 @@ 'Arn': 'string', 'FeatureSet': 'ALL', 'MasterAccountArn': 'string', - 'MasterAccountId': 'some_master_account_id', + 'MasterAccountId': 'some_management_account_id', 'MasterAccountEmail': 'string', 'AvailablePolicyTypes': [ { diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py index 5b5c5d4cc..050102377 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_organizations.py @@ -327,7 +327,7 @@ def test_get_organization_info(cls): ) assert cls.get_organization_info() == { "organization_id": "some_org_id", - "organization_master_account_id": "some_master_account_id", + "organization_management_account_id": "some_management_account_id", "feature_set": "ALL", } diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_parameter_store.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_parameter_store.py index 563d658e3..f8e3d277a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_parameter_store.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_parameter_store.py @@ -5,7 +5,7 @@ import os import boto3 -from pytest import fixture +from pytest import fixture, mark from stubs import stub_parameter_store from mock import Mock @@ -21,6 +21,39 @@ def cls(): return cls +@mark.parametrize( + "input_name, output_path", + [ + ('/adf/test', '/adf/test'), + ('adf/test', '/adf/test'), + ('/test', '/test'), + ('test', '/test'), + ('/other/test', '/other/test'), + ('other/test', '/other/test'), + ], +) +def test_build_param_name_not_adf_only(input_name, output_path): + assert ParameterStore._build_param_name( + input_name, + adf_only=False, + ) == output_path + + +@mark.parametrize( + "input_name, output_path", + [ + ('/adf/test', '/adf/test'), + ('adf/test', '/adf/test'), + ('/test', '/adf/test'), + ('test', '/adf/test'), + ('/other/test', '/adf/other/test'), + ('other/test', '/adf/other/test'), + ], +) +def test_build_param_name_adf_only(input_name, output_path): + assert ParameterStore._build_param_name(input_name) == output_path + + def test_fetch_parameter(cls): cls.client = Mock() cls.client.get_parameter.return_value = stub_parameter_store.get_parameter diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_target.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_target.py index bf3d7f5f0..c23c8c1e7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_target.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/tests/test_target.py @@ -52,7 +52,7 @@ def test_fetch_accounts_for_target_ou_path(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -76,7 +76,7 @@ def test_fetch_accounts_for_target_account_id(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -99,7 +99,7 @@ def test_fetch_accounts_for_target_ou_id(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -122,7 +122,7 @@ def test_fetch_accounts_for_approval(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -145,7 +145,7 @@ def test_fetch_account_error(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -167,7 +167,7 @@ def test_fetch_account_error_invalid_account_id(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -189,7 +189,7 @@ def test_target_structure_respects_wave(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -287,7 +287,7 @@ def test_target_structure_respects_multi_region(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -334,7 +334,7 @@ def test_target_structure_respects_multi_action_single_region(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -378,7 +378,7 @@ def test_target_structure_respects_multi_action_multi_region(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -422,7 +422,7 @@ def test_target_structure_respects_change_set_approval_single_region(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] @@ -467,7 +467,7 @@ def test_target_wave_structure_respects_exclude_config(): with patch.object(ParameterStore, 'fetch_parameter') as mock: expected_calls = [ call( - '/adf/deployment-maps/allow-empty-target', + 'deployment_maps/allow_empty_target', 'False', ), ] diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_param_store.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_param_store.py index 432c93bb6..22e966fb1 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_param_store.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_param_store.py @@ -57,7 +57,7 @@ def resolve(self, lookup_str: str, random_filename: str) -> str: return self.cache.get(cache_key) client = ParameterStore(region, boto3) try: - param_value = client.fetch_parameter(param_path) + param_value = client.fetch_parameter(param_path, adf_only=False) if param_value: self.cache.add(f'{region}/{param_path}', param_value) return param_value diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_upload.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_upload.py index e94e23d8b..86dcba5f6 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_upload.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/resolver_upload.py @@ -68,7 +68,7 @@ def resolve(self, lookup_str: str, random_filename: str) -> str: lookup_str, ) bucket_name = self.parameter_store.fetch_parameter( - f'/cross_region/s3_regional_bucket/{region}' + f'cross_region/s3_regional_bucket/{region}' ) s3_client = S3(region, bucket_name) resolved_location = s3_client.put_object( diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/templates/events.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/templates/events.yml index c4cc84f0a..7032af664 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/templates/events.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/templates/events.yml @@ -5,7 +5,7 @@ Parameters: DeploymentAccountId: Type: "AWS::SSM::Parameter::Value" Description: Deployment Account ID - Default: deployment_account_id + Default: /adf/deployment_account_id Resources: EventRole: @@ -28,6 +28,7 @@ Resources: - Effect: Allow Action: events:PutEvents Resource: "*" + EventRule: Type: AWS::Events::Rule Properties: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py index 278cd3236..18c33e1c4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_main.py @@ -65,11 +65,11 @@ def test_update_deployment_account_output_parameters(cls, sts): with patch.object(ParameterStore, 'put_parameter') as mock: expected_calls = [ call( - '/cross_region/kms_arn/eu-central-1', + 'cross_region/kms_arn/eu-central-1', 'some_kms_arn', ), call( - '/cross_region/s3_regional_bucket/eu-central-1', + 'cross_region/s3_regional_bucket/eu-central-1', 'some_s3_bucket', ), ] @@ -133,27 +133,36 @@ def test_prepare_deployment_account_defaults(param_store_cls, cls, sts): ) for param_store in parameter_store_list: assert param_store.put_parameter.call_count == ( - 13 if param_store == deploy_param_store else 2 + 14 if param_store == deploy_param_store else 8 ) param_store.put_parameter.assert_has_calls( [ + call('adf_version', '1.0.0'), + call('adf_log_level', 'CRITICAL'), + call('cross_account_access_role', 'some_role'), + call( + 'deployment_account_bucket', + 'some_deployment_account_bucket', + ), + call('deployment_account_id', deployment_account_id), + call('management_account_id', '123'), call('organization_id', 'o-123456789'), - call('/adf/extensions/terraform/enabled', 'False'), + call('extensions/terraform/enabled', 'False'), ], any_order=False, ) deploy_param_store.put_parameter.assert_has_calls( [ - call('adf_version', '1.0.0'), - call('adf_log_level', 'CRITICAL'), - call('deployment_account_bucket', 'some_deployment_account_bucket'), - call('default_scm_branch', 'master'), - call('/adf/org/stage', 'none'), - call('cross_account_access_role', 'some_role'), + call('scm/default_scm_branch', 'main'), + call( + 'scm/default_scm_codecommit_account_id', + deployment_account_id, + ), + call('deployment_maps/allow_empty_target', 'False'), + call('org/stage', 'none'), call('notification_type', 'email'), call('notification_endpoint', 'john@example.com'), - call('/adf/extensions/terraform/enabled', 'False'), - call('/adf/deployment-maps/allow-empty-target', 'False'), + call('extensions/terraform/enabled', 'False'), ], any_order=True, ) @@ -225,33 +234,41 @@ def test_prepare_deployment_account_specific_config(param_store_cls, cls, sts): ) for param_store in parameter_store_list: assert param_store.put_parameter.call_count == ( - 15 if param_store == deploy_param_store else 2 + 16 if param_store == deploy_param_store else 8 ) param_store.put_parameter.assert_has_calls( [ + call('adf_version', '1.0.0'), + call('adf_log_level', 'CRITICAL'), + call('cross_account_access_role', 'some_role'), + call( + 'deployment_account_bucket', + 'some_deployment_account_bucket', + ), + call('deployment_account_id', deployment_account_id), + call('management_account_id', '123'), call('organization_id', 'o-123456789'), - call('/adf/extensions/terraform/enabled', 'True'), + call('extensions/terraform/enabled', 'True'), ], any_order=False, ) deploy_param_store.put_parameter.assert_has_calls( [ - call('adf_version', '1.0.0'), - call('adf_log_level', 'CRITICAL'), - call('deployment_account_bucket', 'some_deployment_account_bucket'), - call('default_scm_branch', 'main'), - call('/adf/org/stage', 'test-stage'), - call('auto_create_repositories', 'disabled'), - call('cross_account_access_role', 'some_role'), + call('scm/auto_create_repositories', 'disabled'), + call('scm/default_scm_branch', 'main'), + call( + 'scm/default_scm_codecommit_account_id', + deployment_account_id, + ), + call('deployment_maps/allow_empty_target', 'False'), + call('org/stage', 'test-stage'), call('notification_type', 'slack'), call( 'notification_endpoint', "arn:aws:lambda:eu-central-1:" f"{deployment_account_id}:function:SendSlackNotification", ), - call('/notification_endpoint/main', 'slack-channel'), - call('/adf/extensions/terraform/enabled', 'True'), - call('/adf/deployment-maps/allow-empty-target', 'False'), + call('notification_endpoint/main', 'slack-channel'), ], - any_order=True, + any_order=False, ) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml index e84ff8ed4..62b177abd 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml @@ -25,9 +25,9 @@ config: scm: # Source control management auto-create-repositories: enabled # ^ If true and using CodeCommit as source, the repository will be automatically created - default-scm-branch: master + default-scm-branch: main # ^ The default branch is used when the pipeline does not specify a specific branch. - # If this parameter is not specified, it defaults to the "master" branch. + # If this parameter is not specified, it defaults to the "main" branch. org: stage: prod # ^ This value will be set as an SSM Parameter named /adf/org/stage diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/tox.ini b/src/lambda_codebase/initial_commit/bootstrap_repository/tox.ini index ff7f2e7bb..c6ab5d9f3 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/tox.ini +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/tox.ini @@ -19,7 +19,7 @@ setenv= S3_BUCKET=some_bucket S3_BUCKET_NAME=some_bucket DEPLOYMENT_ACCOUNT_BUCKET=some_deployment_account_bucket - MASTER_ACCOUNT_ID=123 + MANAGEMENT_ACCOUNT_ID=123 ADF_VERSION=1.0.0 ADF_LOG_LEVEL=CRITICAL ADF_PROJECT_NAME=whatever diff --git a/src/lambda_codebase/initial_commit/initial_commit.py b/src/lambda_codebase/initial_commit/initial_commit.py index 8a2b55c13..9cc2ad320 100644 --- a/src/lambda_codebase/initial_commit/initial_commit.py +++ b/src/lambda_codebase/initial_commit/initial_commit.py @@ -33,6 +33,9 @@ "bootstrap_repository/adf-bootstrap/example-global-iam.yml": ( "adf-bootstrap/global-iam.yml" ), + "bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml": ( + "adf-bootstrap/deployment/global-iam.yml" + ), "adf.yml.j2": "adf-accounts/adf.yml", "adfconfig.yml.j2": "adfconfig.yml", } diff --git a/src/lambda_codebase/wait_until_complete.py b/src/lambda_codebase/wait_until_complete.py index 37444e951..2587f9643 100644 --- a/src/lambda_codebase/wait_until_complete.py +++ b/src/lambda_codebase/wait_until_complete.py @@ -41,15 +41,16 @@ def update_deployment_account_output_parameters( regional_parameter_store.put_parameter( "organization_id", os.environ['ORGANIZATION_ID'] ) - # Regions needs to store its kms arn and s3 bucket in master and regional + # Regions needs to store its KMS ARN and S3 bucket. + # It is also stored in the main deployment region. for key, value in cloudformation.get_stack_regional_outputs().items(): LOGGER.info('Updating %s on deployment account in %s', key, region) deployment_account_parameter_store.put_parameter( - f"/cross_region/{key}/{region}", + f"cross_region/{key}/{region}", value ) regional_parameter_store.put_parameter( - f"/cross_region/{key}/{region}", + f"cross_region/{key}/{region}", value ) @@ -65,7 +66,7 @@ def lambda_handler(event, _): role = sts.assume_cross_account_role( f'arn:{partition}:iam::{account_id}:role/{cross_account_access_role}', - 'master' + 'management' ) s3 = S3(REGION_DEFAULT, S3_BUCKET) diff --git a/src/template.yml b/src/template.yml index 5f69d9a3d..fba734adf 100644 --- a/src/template.yml +++ b/src/template.yml @@ -320,7 +320,7 @@ Resources: - !Ref ADFSharedPythonLambdaLayerVersion Environment: Variables: - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -379,7 +379,7 @@ Resources: Environment: Variables: AWS_PARTITION: !Ref AWS::Partition - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -423,7 +423,7 @@ Resources: - !Ref ADFSharedPythonLambdaLayerVersion Environment: Variables: - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -458,7 +458,7 @@ Resources: - Effect: Allow Action: ssm:GetParameter Resource: - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/target_regions" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/target_regions" AccountRegionConfigFunction: Type: 'AWS::Serverless::Function' @@ -471,7 +471,7 @@ Resources: - !Ref ADFSharedPythonLambdaLayerVersion Environment: Variables: - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -491,7 +491,7 @@ Resources: - !Ref ADFSharedPythonLambdaLayerVersion Environment: Variables: - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -539,7 +539,7 @@ Resources: Environment: Variables: AWS_PARTITION: !Ref AWS::Partition - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -574,7 +574,7 @@ Resources: Environment: Variables: AWS_PARTITION: !Ref AWS::Partition - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -608,7 +608,7 @@ Resources: - !Ref ADFSharedPythonLambdaLayerVersion Environment: Variables: - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -652,7 +652,7 @@ Resources: - !Ref ADFSharedPythonLambdaLayerVersion Environment: Variables: - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ['Metadata', 'ADF', 'Version'] ADF_LOG_LEVEL: !Ref LogLevel @@ -976,7 +976,7 @@ Resources: - arm64 CompatibleRuntimes: - python3.12 - Description: "Shared Lambda Layer between master and deployment account" + Description: "Shared Lambda Layer between management and deployment account" LayerName: adf_shared_layer Metadata: BuildMethod: python3.12 @@ -1002,6 +1002,7 @@ Resources: - !Ref GetAccountRegionsFunctionRole - !Ref LambdaRole - !Ref RegisterAccountForSupportFunctionRole + - !Ref AccountHandlerFunctionRole LambdaRole: Type: "AWS::IAM::Role" @@ -1039,9 +1040,18 @@ Resources: - "iam:PutRolePolicy" - "organizations:DescribeOrganization" - "organizations:DescribeAccount" - - "ssm:*" - "states:StartExecution" Resource: "*" + - Effect: Allow + Action: + - "ssm:DeleteParameter" + - "ssm:DeleteParameters" + - "ssm:GetParameter" + - "ssm:GetParameters" + - "ssm:GetParametersByPath" + - "ssm:PutParameter" + Resource: + - !Sub "arn:${AWS::Partition}:ssm:*:${AWS::AccountId}:parameter/adf/*" - Effect: "Allow" Action: "s3:ListBucket" Resource: !GetAtt BootstrapTemplatesBucket.Arn @@ -1063,7 +1073,7 @@ Resources: Variables: S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket TERMINATION_PROTECTION: false - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel @@ -1084,7 +1094,7 @@ Resources: S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket TERMINATION_PROTECTION: false DEPLOYMENT_ACCOUNT_BUCKET: !GetAtt SharedModulesBucketName.Value - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel @@ -1105,7 +1115,7 @@ Resources: S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket TERMINATION_PROTECTION: false DEPLOYMENT_ACCOUNT_BUCKET: !GetAtt SharedModulesBucketName.Value - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel @@ -1126,7 +1136,7 @@ Resources: Variables: S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket TERMINATION_PROTECTION: false - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: RoleStackDeploymentFunction @@ -1145,7 +1155,7 @@ Resources: Variables: S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket TERMINATION_PROTECTION: false - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: MovedToRootActionFunction @@ -1165,7 +1175,7 @@ Resources: Variables: S3_BUCKET_NAME: !Ref BootstrapTemplatesBucket TERMINATION_PROTECTION: false - MASTER_ACCOUNT_ID: !Ref AWS::AccountId + MANAGEMENT_ACCOUNT_ID: !Ref AWS::AccountId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] ADF_LOG_LEVEL: !Ref LogLevel FunctionName: UpdateResourcePoliciesFunction @@ -1234,7 +1244,7 @@ Resources: CodeBuildPolicy: Type: "AWS::IAM::ManagedPolicy" Properties: - Description: "Policy to allow codebuild to perform actions" + Description: "Policy to allow CodeBuild to perform actions" PolicyDocument: Version: "2012-10-17" Statement: @@ -1265,15 +1275,17 @@ Resources: - "organizations:MoveAccount" - "organizations:DescribeCreateAccountStatus" - "organizations:TagResource" - - "ssm:GetParameter" - - "ssm:GetParameters" - - "ssm:PutParameter" - - "states:Describe*" - - "states:StartExecution" - "sts:GetCallerIdentity" - - "sts:assumeRole" + - "sts:AssumeRole" - "cloudformation:ValidateTemplate" Resource: "*" + - Effect: Allow + Action: + - "ssm:GetParameter" + - "ssm:GetParameters" + - "ssm:PutParameter" + Resource: + - !Sub "arn:${AWS::Partition}:ssm:*:${AWS::AccountId}:parameter/adf/*" - Effect: "Allow" Action: - "states:ListExecutions" @@ -1366,7 +1378,7 @@ Resources: Value: !Ref BootstrapTemplatesBucket - Name: ACCOUNT_BUCKET Value: !Ref ADFAccountBucket - - Name: MASTER_ACCOUNT_ID + - Name: MANAGEMENT_ACCOUNT_ID Value: !Ref AWS::AccountId - Name: DEPLOYMENT_ACCOUNT_BUCKET Value: !GetAtt SharedModulesBucketName.Value @@ -1847,7 +1859,7 @@ Resources: Version: !FindInMap ["Metadata", "ADF", "Version"] RepositoryArn: !GetAtt CodeCommitRepository.Arn DirectoryName: bootstrap_repository - ExistingAccountId: !Ref DeploymentAccountId + ExistingAccountId: !GetAtt DeploymentAccount.AccountId DeploymentAccountRegion: !Ref DeploymentAccountMainRegion DeploymentAccountFullName: !Ref DeploymentAccountName DeploymentAccountEmailAddress: !Ref DeploymentAccountEmailAddress @@ -1905,7 +1917,7 @@ Resources: Type: AWS::SSM::Parameter Properties: Description: DO NOT EDIT - Used by The AWS Deployment Framework - Name: shared_modules_bucket + Name: /adf/shared_modules_bucket Type: String Value: !GetAtt SharedModulesBucket.BucketName @@ -1913,7 +1925,7 @@ Resources: Type: AWS::SSM::Parameter Properties: Description: DO NOT EDIT - Used by The AWS Deployment Framework - Name: adf_log_level + Name: /adf/adf_log_level Type: String Value: !Ref LogLevel @@ -1944,8 +1956,8 @@ Resources: Resource: # Hardcoded name (instead of ${SharedModulesBucketName}) to avoid a circular # dependency. Converting this to an inline policy can break the circle - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/shared_modules_bucket" - - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/deployment_account_region" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/shared_modules_bucket" + - !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/deployment_account_region" - Effect: "Allow" Action: "lambda:GetLayerVersion" Resource: !Ref ADFSharedPythonLambdaLayerVersion @@ -2016,6 +2028,44 @@ Resources: AccountEmailAddress: !Ref DeploymentAccountEmailAddress CrossAccountAccessRoleName: !Ref CrossAccountAccessRoleName ExistingAccountId: !Ref DeploymentAccountId + TriggerOnUpdateOfADF: !FindInMap ["Metadata", "ADF", "Version"] + + AccountHandlerFunctionRole: + Type: "AWS::IAM::Role" + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Principal: + Service: + - lambda.amazonaws.com + Action: "sts:AssumeRole" + Path: "/aws-deployment-framework/" + Policies: + - PolicyName: "adf-account-management-access" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "organizations:CreateAccount" + - "organizations:DescribeCreateAccountStatus" + - "organizations:ListAccountsForParent" + - "organizations:ListOrganizationalUnitsForParent" + - "organizations:ListRoots" + Resource: "*" + - Effect: Allow + Action: + - "ssm:GetParameter" + - "ssm:PutParameter" + Resource: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/adf/deployment_account_id" + - Effect: Allow + Action: + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + Resource: "*" AccountHandler: Type: AWS::Serverless::Function @@ -2023,18 +2073,10 @@ Resources: Handler: handler.lambda_handler CodeUri: lambda_codebase/account Description: "ADF Lambda Function - Create Account" - Policies: - - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "organizations:CreateAccount" - - "organizations:DescribeCreateAccountStatus" - Resource: "*" - - Effect: Allow - Action: ssm:GetParameter - Resource: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/deployment_account_id" + Role: !GetAtt AccountHandlerFunctionRole.Arn FunctionName: AccountHandler + Layers: + - !Ref ADFSharedPythonLambdaLayerVersion Metadata: BuildMethod: python3.12 diff --git a/tox.ini b/tox.ini index b467c5737..5c274c325 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@ setenv= S3_BUCKET=some_bucket S3_BUCKET_NAME=some_bucket DEPLOYMENT_ACCOUNT_BUCKET=some_deployment_account_bucket - MASTER_ACCOUNT_ID=123 + MANAGEMENT_ACCOUNT_ID=123 ADF_VERSION=1.0.0 ADF_LOG_LEVEL=CRITICAL ADF_PROJECT_NAME=whatever