From 5f625acb292200819ad814bee3d2c68bdcf2f516 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 10:48:38 +0200 Subject: [PATCH 001/179] added .gitignore --- .gitignore | 143 +++++++++++--- .../shared/helpers/terraform/adf_terraform.sh | 175 ++++++++++++++++++ .../shared/helpers/terraform/get_accounts.py | 133 +++++++++++++ .../helpers/terraform/install_terraform.sh | 6 +- .../shared/helpers/terraform/paginator.py | 14 ++ 5 files changed, 440 insertions(+), 31 deletions(-) create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py create mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py diff --git a/.gitignore b/.gitignore index 9fca97731..bbefe6c21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,38 +1,47 @@ -.vscode -.idea -.pyc -.zip -.DS_Store -cleanup.py -config -config.sec -config.bak -policy.json -pipeline.yml -packaged.yaml -template-sam.yml -template-deploy.yml -template-sam-out.yml -master-deploy.yml -.pytest_cache -shared_layer.zip -.aws-sam -pipeline.json -template-sam.yml -deploy.sh +# Created by https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=macos,terraform,python,visualstudiocode + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### # Byte-compiled / optimized / DLL files -__pycache__ +__pycache__/ *.py[cod] *$py.class -outfile + # C extensions *.so # Distribution / packaging -package.yaml -aws_deployment_framework.egg-info .Python +build/ develop-eggs/ dist/ downloads/ @@ -44,6 +53,7 @@ parts/ sdist/ var/ wheels/ +share/python-wheels/ *.egg-info/ .installed.cfg *.egg @@ -69,8 +79,10 @@ htmlcov/ nosetests.xml coverage.xml *.cover +*.py,cover .hypothesis/ .pytest_cache/ +cover/ # Translations *.mo @@ -80,6 +92,7 @@ coverage.xml *.log local_settings.py db.sqlite3 +db.sqlite3-journal # Flask stuff: instance/ @@ -92,6 +105,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -102,10 +116,23 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ -# celery beat schedule file +# Celery stuff celerybeat-schedule +celerybeat.pid # SageMath parsed files *.sage.py @@ -136,3 +163,63 @@ dmypy.json # Pyre type checker .pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +# +*.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +# Ignore CLI configuration files +.terraformrc +terraform.rc + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode + diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh new file mode 100644 index 000000000..798d5fd11 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -0,0 +1,175 @@ +#!/usr/bin/env bash +PATH=$PATH:$(pwd) +export PATH +CURRENT=$(pwd) +terraform --version +echo "Terraform stage: $TF_STAGE" + +tfinit(){ + # retrieve regional S3 bucket name from parameter store + S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name /cross_region/s3_regional_bucket/"$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) + mkdir -p "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + cd "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" || exit + cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + # if account related variables exist copy the folder in the work directory + if [ -d "$CURRENT/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then + cp -R "$CURRENT"/tfvars/"$TF_VAR_TARGET_ACCOUNT_ID"/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + fi + cp -R "$CURRENT"/tfvars/global.auto.tfvars "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + terraform init \ + -backend-config "bucket=$S3_BUCKET_REGION_NAME" \ + -backend-config "region=$AWS_REGION" \ + -backend-config "key=$ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" \ + -backend-config "dynamodb_table=adf-tflocktable" + + echo "Bucket: $S3_BUCKET_REGION_NAME" + echo "Region: $AWS_REGION" + echo "Key: $ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" + echo "DynamoDB table: adf-tflocktable" +} +tfplan(){ + bash "$CURRENT"/adf-build/helpers/sts.sh "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" + terraform plan -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" +} +tfapply(){ + terraform apply "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" +} + +# if REGIONS is not defined as pipeline parameters use default region +if [[ -z "$REGIONS" ]] +then + REGIONS=$AWS_DEFAULT_REGION +fi +echo "List of target regions: $REGIONS" +for REGION in $(echo $REGIONS | sed "s/,/ /g") +do + AWS_REGION=$REGION + export TF_VAR_TARGET_REGION=$REGION + # if TARGET_ACCOUNTS and TARGET_OUS are not defined apply to all accounts + if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]] + then + echo "Apply to all accounts" + for ACCOUNT_ID in $(jq '.[].AccountId' "$CURRENT"/accounts.json | sed 's/"//g' ) + do + export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID + echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" + if [[ "$TF_STAGE" = "plan" ]] + then + tfinit + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfplan + if [[ $? -ne 0 ]] + then + exit 1 + fi + elif [[ "$TF_STAGE" = "apply" ]] + then + tfinit + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfplan + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfapply + if [[ $? -ne 0 ]] + then + exit 1 + fi + else + echo "Invalid terraform command" + fi + done + fi + + if ! [[ -z "$TARGET_ACCOUNTS" ]] + then + # apply only on a subset of accounts (TARGET_ACCOUNTS) + echo "List of target account: $TARGET_ACCOUNTS" + for ACCOUNT_ID in $(echo $TARGET_ACCOUNTS | sed "s/,/ /g") + do + export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID + echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" + if [[ "$TF_STAGE" = "plan" ]] + then + tfinit + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfplan + if [[ $? -ne 0 ]] + then + exit 1 + fi + elif [[ "$TF_STAGE" = "apply" ]] + then + tfinit + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfplan + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfapply + if [[ $? -ne 0 ]] + then + exit 1 + fi + else + echo "Invalid terraform command" + fi + done + fi + + if ! [[ -z "$TARGET_OUS" ]] + then + echo "List target OUs: $TARGET_OUS" + for ACCOUNT_ID in $(jq '.[].AccountId' "$CURRENT"/accounts_from_ous.json | sed 's/"//g' ) + do + export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID + echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" + if [[ "$TF_STAGE" = "plan" ]] + then + tfinit + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfplan + if [[ $? -ne 0 ]] + then + exit 1 + fi + elif [[ "$TF_STAGE" = "apply" ]] + then + tfinit + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfplan + if [[ $? -ne 0 ]] + then + exit 1 + fi + tfapply + if [[ $? -ne 0 ]] + then + exit 1 + fi + else + echo "Invalid terraform command" + fi + done + fi +done diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py new file mode 100644 index 000000000..725256098 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -0,0 +1,133 @@ +import boto3 +import json +import os +from paginator import paginator + + +sts = boto3.client('sts') + +master_acc_id = os.environ["MASTER_ACCOUNT_ID"] +# master_acc_id = "568100899893" +if("TARGET_OUS" in os.environ): + ou_path = os.environ["TARGET_OUS"] +# ou_path="/core,/sandbox" +# root_id="r-okhg" + + +def list_organizational_units_for_parent(parent_ou): + organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/OrganizationAccountAccessRole-readonly', 'getaccountID') + # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') + organizational_units = [ + ou + for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) + for ou in org_units['OrganizationalUnits'] + ] + return organizational_units + +def get_accounts(): + # Return an array of objects like this: [{'AccountId':'xxx','Email':''}] + account_details = [] # [{'AccountId':'123','Email':''},{'AccountId':'456','Email':''}] + + print("Master Account ID: " + master_acc_id) + # Assume a role into the Org Master role to get account ID's and emails + organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/OrganizationAccountAccessRole-readonly', 'getaccountID') + # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') + for account in paginator(organizations.list_accounts): + if account['Status'] == 'ACTIVE': + account_details.append({ + 'AccountId': account['Id'], + 'Email': account['Email'] + }) + return account_details + +def get_accounts_from_ous(): + parent_ou_id = None + account_list = [] + organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/OrganizationAccountAccessRole-readonly', 'getaccountID') + # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') + # Read organization root id + root_ids = [] + for id in paginator(organizations.list_roots): + root_ids.append({ + 'AccountId': id['Id'] + }) + root_id = root_ids[0]['AccountId'] + print("Target OUs") + for path in ou_path.split(','): + print(path) + # Set initial OU to start looking for given ou_path + if parent_ou_id is None: + parent_ou_id = root_id + + # Parse ou_path and find the ID + ou_hierarchy = path.strip('/').split('/') + hierarchy_index = 0 + if(path.strip() == '/'): + account_list.extend(get_account_recursive(organizations, parent_ou_id, '/')) + else: + while hierarchy_index < len(ou_hierarchy): + org_units = list_organizational_units_for_parent(parent_ou_id) + for ou in org_units: + if ou['Name'] == ou_hierarchy[hierarchy_index]: + parent_ou_id = ou['Id'] + hierarchy_index += 1 + break + else: + raise ValueError(f'Could not find ou with name {ou_hierarchy} in OU list {org_units}.') + + account_list.extend(get_account_recursive(organizations, parent_ou_id, '/')) + parent_ou_id=None + print("Account list: ", end = '') + for i in account_list: + print(i['AccountId'] + " ", end = '') + print() + print("Number of target accounts: " + str(len(account_list))) + return account_list + + +def get_boto3_client(service, role, session_name): + role = sts.assume_role( + RoleArn=role, + RoleSessionName=session_name, + DurationSeconds=900 + ) + session = boto3.Session( + aws_access_key_id=role['Credentials']['AccessKeyId'], + aws_secret_access_key=role['Credentials']['SecretAccessKey'], + aws_session_token=role['Credentials']['SessionToken'] + ) + return session.client(service) + +def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> list: + account_list = [] + # Get OUs + paginator = org_client.get_paginator('list_children') + pages = paginator.paginate( + ParentId=ou_id, + ChildType='ORGANIZATIONAL_UNIT' + ) + for page in pages: + for child in page['Children']: + account_list.extend(get_account_recursive(org_client, child['Id'], path+ou_id+'/')) + + # Get Accounts + pages = paginator.paginate( + ParentId=ou_id, + ChildType='ACCOUNT' + ) + for page in pages: + for child in page['Children']: + account_list.append({ + 'AccountId': child['Id'] + }) + return account_list + + +accounts = get_accounts() +with open('accounts.json', 'w') as outfile: + json.dump(accounts, outfile) + +if("TARGET_OUS" in os.environ): + accounts_from_ous = get_accounts_from_ous() + with open('accounts_from_ous.json', 'w') as outfile: + json.dump(accounts_from_ous, outfile) \ No newline at end of file diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/install_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/install_terraform.sh index d4f1c64d8..2715eec71 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/install_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/install_terraform.sh @@ -2,9 +2,9 @@ set -e apt-get install --assume-yes jq -terraform_url=$(curl https://releases.hashicorp.com/index.json | jq '{terraform}' | egrep "linux.*amd64" | sort --version-sort -r | head -1 | awk -F[\"] '{print $4}') -echo "Downloading $terraform_url." -curl -o terraform.zip $terraform_url +TERRAFORM_URL="https://releases.hashicorp.com/terraform/$TERRAFORM_VERSION/terraform_${TERRAFORM_VERSION}_linux_amd64.zip" +echo "Downloading $TERRAFORM_URL." +curl -o terraform.zip $TERRAFORM_URL unzip terraform.zip export PATH=$PATH:$(pwd) terraform --version diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py new file mode 100644 index 000000000..18e8cd552 --- /dev/null +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py @@ -0,0 +1,14 @@ +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +Paginator used with certain boto3 calls +when pagination is required +""" + +def paginator(method, **kwargs): + client = method.__self__ + iterator = client.get_paginator(method.__name__) + for page in iterator.paginate(**kwargs).result_key_iters(): + for result in page: + yield result \ No newline at end of file From ff378bc0c1f0083e6efaad3903d51b6756d4089e Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 10:55:27 +0200 Subject: [PATCH 002/179] added adf terraform role to global.yml --- .../adf-bootstrap/deployment/global.yml | 323 +++++++++++------- .../adf-bootstrap/global.yml | 42 ++- 2 files changed, 236 insertions(+), 129 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index dd6b761dd..c6058e91c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -6,22 +6,22 @@ Transform: "AWS::Serverless-2016-10-31" Description: ADF CloudFormation Template (Global) for Deployment Account Parameters: ADFVersion: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: adf_version ADFLogLevel: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: adf_log_level MasterAccountId: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: master_account_id SharedModulesBucket: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: deployment_account_bucket OrganizationId: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: organization_id CrossAccountAccessRole: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: cross_account_access_role Image: Description: The Image you wish to use for CodeBuild (defaults to ubuntu). @@ -32,14 +32,14 @@ Parameters: Type: String Default: "BUILD_GENERAL1_LARGE" # For threading with large amounts of pipelines this is the most effective default AllowedValues: - - "BUILD_GENERAL1_SMALL" #3 GB memory, 2 vCPU + - "BUILD_GENERAL1_SMALL" #3 GB memory, 2 vCPU - "BUILD_GENERAL1_MEDIUM" #7 GB memory, 4 vCPU - - "BUILD_GENERAL1_LARGE" #15 GB memory, 8 vCPU + - "BUILD_GENERAL1_LARGE" #15 GB memory, 8 vCPU NotificationEndpoint: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: notification_endpoint NotificationType: - Type : "AWS::SSM::Parameter::Value" + Type: "AWS::SSM::Parameter::Value" Default: notification_type PipelinePrefix: Description: The Prefix that will be attached to pipeline stacks and names for ADF. @@ -360,7 +360,7 @@ Resources: - "cloudformation:UpdateStack" - "cloudformation:UpdateTerminationProtection" Resource: - - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${PipelinePrefix}*/*" + - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${PipelinePrefix}*/*" - Effect: Allow Action: - "cloudformation:ValidateTemplate" @@ -624,12 +624,12 @@ Resources: RepositoryName: "aws-deployment-framework-pipelines" RepositoryDescription: !Sub "CodeCommit Repo for all pipelines in ${AWS::AccountId}" Triggers: - - Name: Email - DestinationArn: !Ref PipelineSNSTopic - Branches: - - master - Events: - - all + - Name: Email + DestinationArn: !Ref PipelineSNSTopic + Branches: + - master + Events: + - all CodeBuildProject: Type: AWS::CodeBuild::Project Properties: @@ -718,16 +718,16 @@ Resources: Actions: - Name: CreateOrUpdate ActionTypeId: - Category: Build - Owner: AWS - Version: "1" - Provider: CodeBuild + Category: Build + Owner: AWS + Version: "1" + Provider: CodeBuild OutputArtifacts: - Name: "aws-deployment-pipelines-build" InputArtifacts: - Name: "Source" Configuration: - ProjectName: !Ref CodeBuildProject + ProjectName: !Ref CodeBuildProject RunOrder: 1 PipelineSNSTopic: Type: AWS::SNS::Topic @@ -769,16 +769,16 @@ Resources: Id: !Sub "${AWS::StackName}" Version: "2012-10-17" Statement: - - Effect: Allow - Principal: - Service: - - codecommit.amazonaws.com - - events.amazonaws.com - - states.amazonaws.com - Action: sns:Publish - Resource: "*" + - Effect: Allow + Principal: + Service: + - codecommit.amazonaws.com + - events.amazonaws.com + - states.amazonaws.com + Action: sns:Publish + Resource: "*" Topics: - - !Ref PipelineSNSTopic + - !Ref PipelineSNSTopic CodePipelineRole: Type: AWS::IAM::Role Properties: @@ -899,15 +899,15 @@ Resources: - Effect: Allow Sid: "PassRole" Action: - - 'iam:PassRole' - Resource: '*' + - "iam:PassRole" + Resource: "*" Condition: - StringEqualsIfExists: - 'iam:PassedToService': - - cloudformation.amazonaws.com - - elasticbeanstalk.amazonaws.com - - ec2.amazonaws.com - - ecs-tasks.amazonaws.com + StringEqualsIfExists: + "iam:PassedToService": + - cloudformation.amazonaws.com + - elasticbeanstalk.amazonaws.com + - ec2.amazonaws.com + - ecs-tasks.amazonaws.com - Effect: Allow Sid: "AllowCodeStarConnections" Action: @@ -945,7 +945,7 @@ Resources: Properties: CodeUri: lambda_codebase/ Layers: - - !Ref LambdaLayerVersion + - !Ref LambdaLayerVersion Description: "ADF Lambda Function - Send Slack Notification" FunctionName: SendSlackNotification Handler: slack.lambda_handler @@ -961,7 +961,7 @@ Resources: Properties: CodeUri: lambda_codebase/ Layers: - - !Ref LambdaLayerVersion + - !Ref LambdaLayerVersion Description: "ADF Lambda Function - EnableCrossAccountAccess" MemorySize: 1024 Environment: @@ -979,7 +979,7 @@ Resources: Properties: CodeUri: lambda_codebase/ Layers: - - !Ref LambdaLayerVersion + - !Ref LambdaLayerVersion Description: "ADF Lambda Function - CheckPipelineStatus" Environment: Variables: @@ -1097,76 +1097,76 @@ Resources: Properties: StateMachineName: "EnableCrossAccountAccess" DefinitionString: !Sub |- - { - "Comment": "Enable Cross Account Access from Deployment Account", - "StartAt": "DetermineEvent", - "States": { - "DetermineEvent": { - "Type": "Choice", - "Choices": [{ - "Variable": "$.update_only", - "NumericEquals": 1, - "Next": "UpdateDeploymentPipelines" - }, - { - "Not": { - "Variable": "$.error", - "NumericEquals": 0 - }, - "Next": "NotifyFailure" - } - ], - "Default": "EnableCrossAccountAccess" - }, - "EnableCrossAccountAccess": { - "Type": "Task", - "Resource": "${EnableCrossAccountAccess.Arn}", - "Next": "UpdateDeploymentPipelines", - "TimeoutSeconds": 900 - }, - "UpdateDeploymentPipelines": { - "Type": "Task", - "Resource": "${CheckPipelineStatus.Arn}", - "TimeoutSeconds": 60, - "Next": "NeedToNotifySuccess?" - }, - "NeedToNotifySuccess?": { - "Type": "Choice", - "Choices": [{ + { + "Comment": "Enable Cross Account Access from Deployment Account", + "StartAt": "DetermineEvent", + "States": { + "DetermineEvent": { + "Type": "Choice", + "Choices": [{ "Variable": "$.update_only", "NumericEquals": 1, - "Next": "Success" - }], - "Default": "NotifySuccess" - }, - "Success": { - "Type": "Succeed" - }, - "Failure": { - "Type": "Fail" - }, - "NotifySuccess": { - "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", - "Parameters": { - "TopicArn": "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", - "Message.$": "$.message", - "Subject": "Success - AWS Deployment Framework Bootstrap" + "Next": "UpdateDeploymentPipelines" }, + { + "Not": { + "Variable": "$.error", + "NumericEquals": 0 + }, + "Next": "NotifyFailure" + } + ], + "Default": "EnableCrossAccountAccess" + }, + "EnableCrossAccountAccess": { + "Type": "Task", + "Resource": "${EnableCrossAccountAccess.Arn}", + "Next": "UpdateDeploymentPipelines", + "TimeoutSeconds": 900 + }, + "UpdateDeploymentPipelines": { + "Type": "Task", + "Resource": "${CheckPipelineStatus.Arn}", + "TimeoutSeconds": 60, + "Next": "NeedToNotifySuccess?" + }, + "NeedToNotifySuccess?": { + "Type": "Choice", + "Choices": [{ + "Variable": "$.update_only", + "NumericEquals": 1, "Next": "Success" + }], + "Default": "NotifySuccess" + }, + "Success": { + "Type": "Succeed" + }, + "Failure": { + "Type": "Fail" + }, + "NotifySuccess": { + "Type": "Task", + "Resource": "arn:aws:states:::sns:publish", + "Parameters": { + "TopicArn": "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", + "Message.$": "$.message", + "Subject": "Success - AWS Deployment Framework Bootstrap" }, - "NotifyFailure": { - "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", - "Parameters": { - "TopicArn": "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", - "Message.$": "$.error", - "Subject": "Failure - AWS Deployment Framework Bootstrap" - }, - "Next": "Failure" - } + "Next": "Success" + }, + "NotifyFailure": { + "Type": "Task", + "Resource": "arn:aws:states:::sns:publish", + "Parameters": { + "TopicArn": "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", + "Message.$": "$.error", + "Subject": "Failure - AWS Deployment Framework Bootstrap" + }, + "Next": "Failure" } } + } RoleArn: !GetAtt StatesExecutionRole.Arn InitialCommit: Type: Custom::InitialCommit @@ -1205,25 +1205,36 @@ Resources: Type: "String" Value: !GetAtt KMSKey.Arn PipelineCloudWatchEventRole: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - events.amazonaws.com - Action: sts:AssumeRole - Path: / - Policies: - - PolicyName: adf-pipelines-execute-cwe - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: codepipeline:StartPipelineExecution - Resource: !Join [ '', [ 'arn:aws:codepipeline:', !Ref 'AWS::Region', ':', !Ref 'AWS::AccountId', ':', !Ref CodePipeline ] ] + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - events.amazonaws.com + Action: sts:AssumeRole + Path: / + Policies: + - PolicyName: adf-pipelines-execute-cwe + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: codepipeline:StartPipelineExecution + Resource: + !Join [ + "", + [ + "arn:aws:codepipeline:", + !Ref "AWS::Region", + ":", + !Ref "AWS::AccountId", + ":", + !Ref CodePipeline, + ], + ] PipelineCloudWatchEventRule: Type: AWS::Events::Rule Properties: @@ -1231,9 +1242,19 @@ Resources: source: - aws.codecommit detail-type: - - 'CodeCommit Repository State Change' + - "CodeCommit Repository State Change" resources: - - !Join [ '', [ 'arn:aws:codecommit:', !Ref 'AWS::Region', ':', !Ref 'AWS::AccountId', ':', !GetAtt CodeCommitRepository.Name ] ] + - !Join [ + "", + [ + "arn:aws:codecommit:", + !Ref "AWS::Region", + ":", + !Ref "AWS::AccountId", + ":", + !GetAtt CodeCommitRepository.Name, + ], + ] detail: event: - referenceCreated @@ -1244,9 +1265,57 @@ Resources: - master Targets: - Arn: - !Join [ '', [ 'arn:aws:codepipeline:', !Ref 'AWS::Region', ':', !Ref 'AWS::AccountId', ':', !Ref CodePipeline ] ] + !Join [ + "", + [ + "arn:aws:codepipeline:", + !Ref "AWS::Region", + ":", + !Ref "AWS::AccountId", + ":", + !Ref CodePipeline, + ], + ] RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline + ADFTerraformRole: + # ADF role to run terraform pipelines + Type: AWS::IAM::Role + Properties: + RoleName: "adf-terraform-role" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Sid: "AssumeRole" + Principal: + AWS: + # This would allow all codebuild projects to be able to assume this role + - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role + # - !Sub arn:aws:iam::${DeploymentAccountId}:role/my-custom-codebuild-role + # The above role would be created on the deployment account for the purpose deploying this custom resource via codebuild + Action: + - sts:AssumeRole + Path: / + ADFTerraformPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: "adf-terraform-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "ec2:*" + - "s3:*" + - "ssm:*" + - "sso:*" + - "identitystore:*" + - "organizations:*" + Resource: + - "*" + Roles: + - !Ref ADFTerraformRole Outputs: ADFVersionNumber: Value: !Ref ADFVersion diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml index 62d55ec71..a584fe83f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml @@ -183,8 +183,8 @@ Resources: Action: - sts:AssumeRole Condition: - ArnEquals: - 'aws:SourceArn': !Sub 'arn:aws:codepipeline:${AWS::Region}:${DeploymentAccountId}:*' + ArnEquals: + "aws:SourceArn": !Sub "arn:aws:codepipeline:${AWS::Region}:${DeploymentAccountId}:*" Path: / AdfAutomationRole: # This role is used by CodeBuild on the Deployment Account when creating new Codepipeline Pipelines. @@ -317,3 +317,41 @@ Resources: - "*" Roles: - !Ref ReadOnlyAutomationRole + ADFTerraformRole: + # ADF role to run terraform pipelines + Type: AWS::IAM::Role + Properties: + RoleName: "adf-terraform-role" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Sid: "AssumeRole" + Principal: + AWS: + # This would allow all codebuild projects to be able to assume this role + - !Sub arn:aws:iam::${DeploymentAccountId}:role/adf-codebuild-role + # - !Sub arn:aws:iam::${DeploymentAccountId}:role/my-custom-codebuild-role + # The above role would be created on the deployment account for the purpose deploying this custom resource via codebuild + Action: + - sts:AssumeRole + Path: / + ADFTerraformPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: "adf-terraform-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "ec2:*" + - "s3:*" + - "ssm:*" + - "sso:*" + - "identitystore:*" + - "organizations:*" + Resource: + - "*" + Roles: + - !Ref ADFTerraformRole From 0aebbf8cd1d0091038ad5fcdacad66e598d42ab8 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 10:59:14 +0200 Subject: [PATCH 003/179] Added DynamoDB permission to adf-codebuild-policy in deployment/global.yml --- .../adf-bootstrap/deployment/global.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index c6058e91c..8401b29b7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -207,6 +207,15 @@ Resources: PolicyDocument: Version: "2012-10-17" Statement: + - Effect: Allow + Sid: "DynamoDB" + Action: + - dynamodb:PutItem + - dynamodb:GetItem + - dynamodb:DeleteItem + - dynamodb:DescribeTable + Resource: + - !Sub "arn:aws:dynamodb:*:${AWS::AccountId}:table/adf-tflocktable*" - Effect: Allow Sid: "S3" Action: From c5121a6236e22edb82b76d6da2b02fc0a60ddb30 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 11:37:07 +0200 Subject: [PATCH 004/179] added adf-terraform-role to Organization Policy adf-build/global.yml --- .../bootstrap_repository/adf-build/global.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml index 76a0cb9b5..221fbbbc9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml @@ -90,8 +90,8 @@ Resources: Action: - cloudformation:ValidateTemplate - ssm:PutParameter - - ssm:GetParameters - - ssm:GetParameter + - ssm:GetParameters + - ssm:GetParameter Resource: - "*" - Effect: Allow @@ -112,6 +112,6 @@ Resources: - !Sub "arn:aws:iam::${AWS::AccountId}:role/adf-automation-role" - !Sub "arn:aws:iam::${AWS::AccountId}:role/adf-readonly-automation-role" - !Sub "arn:aws:iam::${AWS::AccountId}:role/${CrossAccountAccessRole}" + - !Sub "arn:aws:iam::${AWS::AccountId}:role/adf-terraform-role" Roles: - !Ref OrganizationsRole - From fb49f24fd9e85fb8a087be16180b8dd209ad9e95 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 11:40:50 +0200 Subject: [PATCH 005/179] added organizations:ListChildren to OrganizationsReadOnlyPolicy --- .../initial_commit/bootstrap_repository/adf-build/global.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml index 221fbbbc9..e5ddd36e4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/global.yml @@ -43,6 +43,7 @@ Resources: - organizations:DescribeAccount - organizations:ListOrganizationalUnitsForParent - organizations:ListRoots + - organizations:ListChildren - tag:GetResources Resource: "*" Roles: From 6a9b86827639bc2eab91cbd48b8bb68955829942 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 11:47:36 +0200 Subject: [PATCH 006/179] added DynamoDB table for terraform lock to adf-bootstrap/deployment/global.yml and regional.yml --- .../adf-bootstrap/deployment/regional.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml index d28909919..e998655f7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml @@ -1,11 +1,11 @@ # // Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. # // SPDX-License-Identifier: Apache-2.0 -AWSTemplateFormatVersion: '2010-09-09' +AWSTemplateFormatVersion: "2010-09-09" Description: ADF CloudFormation Template (Regional) for Deployment Account Parameters: OrganizationId: - Type : 'AWS::SSM::Parameter::Value' + Type: "AWS::SSM::Parameter::Value" Default: organization_id Resources: DeploymentFrameworkRegionalS3Bucket: @@ -89,6 +89,19 @@ Resources: Condition: StringEquals: aws:PrincipalOrgID: !Ref OrganizationId + TerraformLockTable: + Type: "AWS::DynamoDB::Table" + Properties: + AttributeDefinitions: + - AttributeName: LockID + AttributeType: S + KeySchema: + - AttributeName: LockID + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: 5 + WriteCapacityUnits: 5 + TableName: adf-tflocktable Outputs: DeploymentFrameworkRegionalS3Bucket: Description: The S3 Bucket used for cross region codepipeline deployments From 9cb0d8390b3cdbc378904d14a5c9d2e24cb434ac Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 11:48:07 +0200 Subject: [PATCH 007/179] added DynamoDB table for terraform lock to adf-bootstrap/deployment/global.yml --- .../adf-bootstrap/deployment/global.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 8401b29b7..5b7942fe7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -1287,6 +1287,19 @@ Resources: ] RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline + TerraformLockTable: + Type: "AWS::DynamoDB::Table" + Properties: + AttributeDefinitions: + - AttributeName: LockID + AttributeType: S + KeySchema: + - AttributeName: LockID + KeyType: HASH + ProvisionedThroughput: + ReadCapacityUnits: 5 + WriteCapacityUnits: 5 + TableName: adf-tflocktable ADFTerraformRole: # ADF role to run terraform pipelines Type: AWS::IAM::Role From 3ceabf4e94fea539b26622a3e67b1147ce905d2b Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 13:57:04 +0200 Subject: [PATCH 008/179] added sample-terraform repository --- samples/sample-terraform/.gitignore | 226 ++++++++++++++++++ samples/sample-terraform/README.md | 90 +++++-- samples/sample-terraform/buildspec.yml | 10 +- samples/sample-terraform/main.tf | 16 -- samples/sample-terraform/my_test_spec.yml | 26 -- samples/sample-terraform/params/global.yml | 2 + .../scripts/terraform/install_terraform.sh | 10 - samples/sample-terraform/terraform.tfvars | 1 - samples/sample-terraform/{ => tf}/backend.tf | 0 samples/sample-terraform/tf/main.tf | 14 ++ samples/sample-terraform/tf/s3.tf | 4 + .../sample-terraform/{ => tf}/variables.tf | 3 +- samples/sample-terraform/tf_apply.yml | 23 ++ samples/sample-terraform/tf_plan.yml | 22 ++ samples/sample-terraform/tf_sec.yml | 12 + 15 files changed, 388 insertions(+), 71 deletions(-) create mode 100644 samples/sample-terraform/.gitignore delete mode 100644 samples/sample-terraform/main.tf delete mode 100644 samples/sample-terraform/my_test_spec.yml create mode 100644 samples/sample-terraform/params/global.yml delete mode 100644 samples/sample-terraform/scripts/terraform/install_terraform.sh delete mode 100644 samples/sample-terraform/terraform.tfvars rename samples/sample-terraform/{ => tf}/backend.tf (100%) create mode 100644 samples/sample-terraform/tf/main.tf create mode 100644 samples/sample-terraform/tf/s3.tf rename samples/sample-terraform/{ => tf}/variables.tf (69%) create mode 100644 samples/sample-terraform/tf_apply.yml create mode 100644 samples/sample-terraform/tf_plan.yml create mode 100644 samples/sample-terraform/tf_sec.yml diff --git a/samples/sample-terraform/.gitignore b/samples/sample-terraform/.gitignore new file mode 100644 index 000000000..572727729 --- /dev/null +++ b/samples/sample-terraform/.gitignore @@ -0,0 +1,226 @@ + +# Created by https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=macos,terraform,python,visualstudiocode + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +pytestdebug.log + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ +doc/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +# .env +.env/ +.venv/ +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ +pythonenv* + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# operating system-related files +# file properties cache/storage on macOS +*.DS_Store +# thumbnail cache on Windows +Thumbs.db + +# profiling data +.prof + + +### Terraform ### +# Local .terraform directories +**/.terraform/* + +# .tfstate files +*.tfstate +*.tfstate.* + +# Crash log files +crash.log + +# Ignore any .tfvars files that are generated automatically for each Terraform run. Most +# .tfvars files are managed as part of configuration and so should be included in +# version control. +# +# example.tfvars + +# Ignore override files as they are usually used to override resources locally and so +# are not checked in +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Include override files you do wish to add to version control using negated pattern +# !example_override.tf + +# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan +# example: *tfplan* + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode + diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 095c794d5..4fe2357a1 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -1,19 +1,79 @@ -## Sample Terraform +# Terraform template -### Deployment Map example +## Overview + +This repository contains a module that manage the deployment of terraform code to multiple accounts and regions. +The module consists of three build stages defined in the following file: + +- `buildspec.yml`: install the version of terraform specified in the pipeline configuration +- `tf_scan.yml`: (optional) returns any vulnerabilities in terraform code according with terrascan utilitiy +- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan +- `tf_apply.yml`: run a terraform apply after the manual step approval + +## Parameters + +- TERRAFORM_VERSION: the terraform version used to deploy the resource +- TARGET_ACCOUNTS: comma separated list of target accounts +- TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) +- REGIONS: comma separated list of target region + +### Deployment procedure + +1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: ```yaml - - name: my-terraform-example - default_providers: - source: - provider: codecommit - properties: - account_id: 1111111111111 - deploy: - provider: codebuild - properties: - image: "STANDARD_4_0" - targets: - - properties: - spec_filename: my_test_spec.yml +- name: sample-terraform + default_providers: + source: + provider: codecommit + properties: + account_id: 111111111111 # source account id + build: + provider: codebuild + properties: + environment_variables: + TERRAFORM_VERSION: "0.14.10" # terraform version + deploy: + provider: codebuild + properties: + image: "STANDARD_5_0" + environment_variables: + TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts + TARGET_OUS: /core/infrastructure,/sandbox # target OUs + MASTER_ACCOUNT_ID: 333333333333 # master account + REGIONS: eu-west-1 # target regions + params: + restart_execution_on_update: true + targets: + - name: terraform-scan # optional + properties: + spec_filename: tf_scan.yml # terraform scan + - name: terraform-plan + properties: + spec_filename: tf_plan.yml # terraform plan + - approval # manual approval + - name: terraform-apply + properties: + spec_filename: tf_apply.yml # terraform apply ``` + +2. Add the project name in params/global.yml file +3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. +4. Add variable definition to tf\variables.tf file and variable values to tfvars/global.auto.tfvars + + - Local variables (per account) can be configured using the following naming convention + + ``` + tfvars <-- This folder contains the structure to define terraform variables + │ + └───global.auto.tfvars <-- this file contains global variables applied to all the target accounts + │ + └───111111111111 <-- this folders contains variable files related to account 111111111111 + │ └──────│ local.auto.tfvars <-- this file contains variables related to account 111111111111 + │ + └───222222222222 <-- this folders contains variable files related to account 222222222222 + └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 + ``` + +5. Push to sample-terraform ADF repository +6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. diff --git a/samples/sample-terraform/buildspec.yml b/samples/sample-terraform/buildspec.yml index 5b61b5fa0..245c84f9a 100644 --- a/samples/sample-terraform/buildspec.yml +++ b/samples/sample-terraform/buildspec.yml @@ -3,7 +3,13 @@ version: 0.2 phases: install: commands: + - aws s3 cp s3://$S3_BUCKET_NAME/adf-build/ adf-build/ --recursive --quiet - export PATH=$PATH:$(pwd) - - bash scripts/terraform/install_terraform.sh + - bash adf-build/helpers/terraform/install_terraform.sh + - pip install -r adf-build/requirements.txt -q + build: + commands: + - python adf-build/generate_params.py + artifacts: - files: '**/*' \ No newline at end of file + files: "**/*" diff --git a/samples/sample-terraform/main.tf b/samples/sample-terraform/main.tf deleted file mode 100644 index 2cd9d4b34..000000000 --- a/samples/sample-terraform/main.tf +++ /dev/null @@ -1,16 +0,0 @@ -provider "aws" { - version = "~> 2.0" - assume_role { - role_arn = "arn:aws:iam::${var.TARGET_ACCOUNT_ID}:role/${var.TARGET_ACCOUNT_ROLE}" - } -} - -resource "aws_s3_bucket" "b" { - bucket = "${var.my_bucket_name}" - acl = "private" - - tags = { - Name = "My bucket" - Environment = "Dev" - } -} \ No newline at end of file diff --git a/samples/sample-terraform/my_test_spec.yml b/samples/sample-terraform/my_test_spec.yml deleted file mode 100644 index c27464bf2..000000000 --- a/samples/sample-terraform/my_test_spec.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 0.2 - -env: - variables: - AWS_DEFAULT_REGION: eu-central-1 # The region you plan on deploying resources into - TF_VAR_TARGET_ACCOUNT_ID: 111111111111 # The AWS Account you plan on targeting - TF_VAR_TARGET_ACCOUNT_ROLE: adf-custom-deploy-role # The IAM Role terraform will assume to deploy resources - TF_IN_AUTOMATION: true - TF_CLI_ARGS: "-no-color" - -phases: - install: - runtime-versions: - python: 3.8 - build: - commands: - - export PATH=$PATH:$(pwd) - - | - terraform init \ - -backend-config "bucket=$S3_BUCKET_NAME" \ - -backend-config "region=$AWS_DEFAULT_REGION" \ - -backend-config "key=$ADF_PROJECT_NAME/$TF_VAR_TARGET_ACCOUNT_ID" - - terraform validate - - bash adf-build/helpers/sts.sh $TF_VAR_TARGET_ACCOUNT_ID $TF_VAR_TARGET_ACCOUNT_ROLE - - terraform plan -out $ADF_PROJECT_NAME - - terraform apply $ADF_PROJECT_NAME diff --git a/samples/sample-terraform/params/global.yml b/samples/sample-terraform/params/global.yml new file mode 100644 index 000000000..1aab20ee4 --- /dev/null +++ b/samples/sample-terraform/params/global.yml @@ -0,0 +1,2 @@ +Parameters: + ProjectName: "sample-terraform" # $ADF_PROJECT_NAME diff --git a/samples/sample-terraform/scripts/terraform/install_terraform.sh b/samples/sample-terraform/scripts/terraform/install_terraform.sh deleted file mode 100644 index d4f1c64d8..000000000 --- a/samples/sample-terraform/scripts/terraform/install_terraform.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -e - -apt-get install --assume-yes jq -terraform_url=$(curl https://releases.hashicorp.com/index.json | jq '{terraform}' | egrep "linux.*amd64" | sort --version-sort -r | head -1 | awk -F[\"] '{print $4}') -echo "Downloading $terraform_url." -curl -o terraform.zip $terraform_url -unzip terraform.zip -export PATH=$PATH:$(pwd) -terraform --version diff --git a/samples/sample-terraform/terraform.tfvars b/samples/sample-terraform/terraform.tfvars deleted file mode 100644 index 39fa70520..000000000 --- a/samples/sample-terraform/terraform.tfvars +++ /dev/null @@ -1 +0,0 @@ -my_bucket_name = "some-random-unique-bucket-name" \ No newline at end of file diff --git a/samples/sample-terraform/backend.tf b/samples/sample-terraform/tf/backend.tf similarity index 100% rename from samples/sample-terraform/backend.tf rename to samples/sample-terraform/tf/backend.tf diff --git a/samples/sample-terraform/tf/main.tf b/samples/sample-terraform/tf/main.tf new file mode 100644 index 000000000..0c1cb8a42 --- /dev/null +++ b/samples/sample-terraform/tf/main.tf @@ -0,0 +1,14 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 3.0" + } + } + required_version = ">= 0.13.0" +} +provider "aws" { + assume_role { + role_arn = "arn:aws:iam::${var.TARGET_ACCOUNT_ID}:role/${var.TARGET_ACCOUNT_ROLE}" + } +} diff --git a/samples/sample-terraform/tf/s3.tf b/samples/sample-terraform/tf/s3.tf new file mode 100644 index 000000000..8894dfddb --- /dev/null +++ b/samples/sample-terraform/tf/s3.tf @@ -0,0 +1,4 @@ +resource "aws_s3_bucket" "b" { + bucket = "my-tf-test-bucket-${var.TARGET_REGION}-${var.TARGET_ACCOUNT_ID}" + acl = "private" +} diff --git a/samples/sample-terraform/variables.tf b/samples/sample-terraform/tf/variables.tf similarity index 69% rename from samples/sample-terraform/variables.tf rename to samples/sample-terraform/tf/variables.tf index dfe4dff2a..6ceecff26 100644 --- a/samples/sample-terraform/variables.tf +++ b/samples/sample-terraform/tf/variables.tf @@ -1,3 +1,4 @@ variable "TARGET_ACCOUNT_ID" {} variable "TARGET_ACCOUNT_ROLE" {} -variable my_bucket_name {} +variable "TARGET_REGION" {} + diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml new file mode 100644 index 000000000..35d2769a0 --- /dev/null +++ b/samples/sample-terraform/tf_apply.yml @@ -0,0 +1,23 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: eu-west-1 # The region you plan on deploying resources into + TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources + TF_IN_AUTOMATION: true + TF_CLI_ARGS: "-no-color" + TF_STAGE: "apply" + +phases: + install: + runtime-versions: + python: 3.8 + build: + commands: + #- python scripts/get_accounts.py + - python adf-build/helpers/terraform/get_accounts.py + #- bash if test "$(scripts/terraform/adf_terraform.sh $TF_STAGE)" false; then exit 1; fi + - bash adf-build/helpers/terraform/adf_terraform.sh $TF_STAGE + +artifacts: + files: "**/*" diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml new file mode 100644 index 000000000..4956a40cd --- /dev/null +++ b/samples/sample-terraform/tf_plan.yml @@ -0,0 +1,22 @@ +version: 0.2 + +env: + variables: + AWS_DEFAULT_REGION: eu-west-1 # The region you plan on deploying resources into + TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources + TF_IN_AUTOMATION: true + TF_STAGE: "plan" + TF_CLI_ARGS: "-no-color" + +phases: + install: + runtime-versions: + python: 3.8 + # commands: + # - pip install -r scripts/requirements.txt -q + build: + commands: + #- python scripts/get_accounts.py + - python adf-build/helpers/terraform/get_accounts.py + #- bash if test "$(scripts/terraform/adf_terraform.sh $TF_STAGE)" false; then exit 1; fi + - bash adf-build/helpers/terraform/adf_terraform.sh $TF_STAGE diff --git a/samples/sample-terraform/tf_sec.yml b/samples/sample-terraform/tf_sec.yml new file mode 100644 index 000000000..14ec65720 --- /dev/null +++ b/samples/sample-terraform/tf_sec.yml @@ -0,0 +1,12 @@ +version: 0.2 + +phases: + install: + commands: + - curl -L "$(curl -s https://api.github.com/repos/accurics/terrascan/releases/latest | grep -o -E "https://.+?_Linux_x86_64.tar.gz")" > terrascan.tar.gz + - tar -xf terrascan.tar.gz terrascan && rm terrascan.tar.gz + - install terrascan /usr/local/bin && rm terrascan + build: + commands: + - cd tf + - terrascan scan -o yaml --skip-rules="AC_AWS_078" From 8be3e71b41c45889df65c9aa9c51544296a7f2ea Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 13:58:59 +0200 Subject: [PATCH 009/179] Added tfvars examples --- .gitignore | 5 +++-- samples/sample-terraform/tfvars/global.auto.tfvars | 0 2 files changed, 3 insertions(+), 2 deletions(-) create mode 100644 samples/sample-terraform/tfvars/global.auto.tfvars diff --git a/.gitignore b/.gitignore index bbefe6c21..4e2ecf934 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,8 @@ .LSOverride # Icon must end with two \r -Icon +Icon + # Thumbnails ._* @@ -186,7 +187,7 @@ crash.log # control as they are data points which are potentially sensitive and subject # to change depending on the environment. # -*.tfvars +#*.tfvars # Ignore override files as they are usually used to override resources locally and so # are not checked in diff --git a/samples/sample-terraform/tfvars/global.auto.tfvars b/samples/sample-terraform/tfvars/global.auto.tfvars new file mode 100644 index 000000000..e69de29bb From bc8d00c76a9643aed10481f2f28b4d2f5e320825 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 14:34:42 +0200 Subject: [PATCH 010/179] clean comments --- samples/sample-terraform/tf_apply.yml | 4 +--- samples/sample-terraform/tf_plan.yml | 6 +----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml index 35d2769a0..f46b49a97 100644 --- a/samples/sample-terraform/tf_apply.yml +++ b/samples/sample-terraform/tf_apply.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - AWS_DEFAULT_REGION: eu-west-1 # The region you plan on deploying resources into + AWS_DEFAULT_REGION: eu-west-1 # ADF main region TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources TF_IN_AUTOMATION: true TF_CLI_ARGS: "-no-color" @@ -14,9 +14,7 @@ phases: python: 3.8 build: commands: - #- python scripts/get_accounts.py - python adf-build/helpers/terraform/get_accounts.py - #- bash if test "$(scripts/terraform/adf_terraform.sh $TF_STAGE)" false; then exit 1; fi - bash adf-build/helpers/terraform/adf_terraform.sh $TF_STAGE artifacts: diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml index 4956a40cd..83b5aa8c6 100644 --- a/samples/sample-terraform/tf_plan.yml +++ b/samples/sample-terraform/tf_plan.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - AWS_DEFAULT_REGION: eu-west-1 # The region you plan on deploying resources into + AWS_DEFAULT_REGION: eu-west-1 # ADF main region TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources TF_IN_AUTOMATION: true TF_STAGE: "plan" @@ -12,11 +12,7 @@ phases: install: runtime-versions: python: 3.8 - # commands: - # - pip install -r scripts/requirements.txt -q build: commands: - #- python scripts/get_accounts.py - python adf-build/helpers/terraform/get_accounts.py - #- bash if test "$(scripts/terraform/adf_terraform.sh $TF_STAGE)" false; then exit 1; fi - bash adf-build/helpers/terraform/adf_terraform.sh $TF_STAGE From 3711c3d166ff7775ca90559f8e9ee239d4b16b4d Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 14:35:06 +0200 Subject: [PATCH 011/179] pull request template --- .github/PULL_REQUEST_TEMPLATE.md | 97 +++++++++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 6bdaa999f..41d622678 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,99 @@ -*Issue #, if available:* +_Issue #, if available:_ -*Description of changes:* +_Description of changes:_ +This PR enables ADF to run terraform pipelines multi-accounts/OUs and multi-regions and manages in a structured way terraform state file and lock. + +Terraform module includes: + +- `adf_terraform.sh`: this script + - runs terraform plan and apply + - centralize the terraform state file in the S3 bucket of the deployment account + - e.g. main region adf-global-base-deployment-pipelinebucketxyz/ProjectName/accountID.tfstate + - + - centralize lock of the state file in regional DynamoDB tables in deployment account +- `sample-terraform`: this folder contains an example of terraform pipelines definition and repository +- `adf_terraform_role`: additional role added during the bootstrap of an account to deploy terraform resources +- `adf_locktable`: regional DynamoDB to manage the lock of the state file + +An overview of the terraform template components and functionalities + +### Overview + +This repository contains a module that manage the deployment of terraform code to multiple accounts and regions. +The module consists of three build stages defined in the following file: + +- `buildspec.yml`: install the version of terraform specified in the pipeline configuration +- `tf_scan.yml`: (optional) returns any vulnerabilities in terraform code according with terrascan utilitiy +- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan +- `tf_apply.yml`: run a terraform apply after the manual step approval + +### Parameters + +- TERRAFORM_VERSION: the terraform version used to deploy the resource +- TARGET_ACCOUNTS: comma separated list of target accounts +- TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) +- REGIONS: comma separated list of target region + +### Deployment procedure + +1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: + +```yaml +- name: sample-terraform + default_providers: + source: + provider: codecommit + properties: + account_id: 111111111111 # source account id + build: + provider: codebuild + properties: + environment_variables: + TERRAFORM_VERSION: "0.14.10" # terraform version + deploy: + provider: codebuild + properties: + image: "STANDARD_5_0" + environment_variables: + TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts + TARGET_OUS: /core/infrastructure,/sandbox # target OUs + MASTER_ACCOUNT_ID: 333333333333 # master account + REGIONS: eu-west-1 # target regions + params: + restart_execution_on_update: true + targets: + - name: terraform-scan # optional + properties: + spec_filename: tf_scan.yml # terraform scan + - name: terraform-plan + properties: + spec_filename: tf_plan.yml # terraform plan + - approval # manual approval + - name: terraform-apply + properties: + spec_filename: tf_apply.yml # terraform apply +``` + +2. Add the project name in params/global.yml file +3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. +4. Add variable definition to tf\variables.tf file and variable values to tfvars/global.auto.tfvars + + - Local variables (per account) can be configured using the following naming convention + + ``` + tfvars <-- This folder contains the structure to define terraform variables + │ + └───global.auto.tfvars <-- this file contains global variables applied to all the target accounts + │ + └───111111111111 <-- this folders contains variable files related to account 111111111111 + │ └──────│ local.auto.tfvars <-- this file contains variables related to account 111111111111 + │ + └───222222222222 <-- this folders contains variable files related to account 222222222222 + └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 + ``` + +5. Push to sample-terraform ADF repository +6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. From ecc13aa5e96b35712fbc1c24e5601fe166cf5e0b Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 29 Oct 2021 14:43:42 +0200 Subject: [PATCH 012/179] Define PULL REQUEST --- .github/PULL_REQUEST_TEMPLATE.md | 18 ++++++++++++++++-- samples/sample-terraform/README.md | 13 +++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 41d622678..909b65891 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -6,15 +6,16 @@ This PR enables ADF to run terraform pipelines multi-accounts/OUs and multi-regi Terraform module includes: -- `adf_terraform.sh`: this script +- `adf_terraform.sh`: this script: - runs terraform plan and apply - centralize the terraform state file in the S3 bucket of the deployment account - e.g. main region adf-global-base-deployment-pipelinebucketxyz/ProjectName/accountID.tfstate - - + - e.g. secondary region adf-regional-base-deploy-deploymentframeworkregio-jsm/ProjectName/accountID.tfstate - centralize lock of the state file in regional DynamoDB tables in deployment account - `sample-terraform`: this folder contains an example of terraform pipelines definition and repository - `adf_terraform_role`: additional role added during the bootstrap of an account to deploy terraform resources - `adf_locktable`: regional DynamoDB to manage the lock of the state file +- minor changes to adf core roles to enable terraform pipelines An overview of the terraform template components and functionalities @@ -96,4 +97,17 @@ The module consists of three build stages defined in the following file: 5. Push to sample-terraform ADF repository 6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. +Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created +e.g. Project name: sample-tf-module +Target accounts: 111111111111, 222222222222 +Target regions: eu-west-1 (main ADF region), us-east-1 +The following state files are created + +- 111111111111 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/111111111111.tfstate +- 111111111111 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/111111111111.tfstate +- 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate +- 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate + +A DynamoDB table manage the lock of the state file. It is deployed in every ADF regions named adf_locktable + By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 4fe2357a1..1e57e5e86 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -77,3 +77,16 @@ The module consists of three build stages defined in the following file: 5. Push to sample-terraform ADF repository 6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. + +Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created +e.g. Project name: sample-tf-module +Target accounts: 111111111111, 222222222222 +Target regions: eu-west-1 (main ADF region), us-east-1 +The following state files are created + +- 111111111111 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/111111111111.tfstate +- 111111111111 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/111111111111.tfstate +- 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate +- 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate + +A DynamoDB table manage the lock of the state file. It is deployed in every ADF regions named adf_locktable From c2657ddc8653ae846712a18942b3c32b329971cd Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 5 Nov 2021 22:53:08 +0100 Subject: [PATCH 013/179] clean PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 108 ------------------------------- 1 file changed, 108 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 909b65891..db6ceeddd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -2,112 +2,4 @@ _Issue #, if available:_ _Description of changes:_ -This PR enables ADF to run terraform pipelines multi-accounts/OUs and multi-regions and manages in a structured way terraform state file and lock. - -Terraform module includes: - -- `adf_terraform.sh`: this script: - - runs terraform plan and apply - - centralize the terraform state file in the S3 bucket of the deployment account - - e.g. main region adf-global-base-deployment-pipelinebucketxyz/ProjectName/accountID.tfstate - - e.g. secondary region adf-regional-base-deploy-deploymentframeworkregio-jsm/ProjectName/accountID.tfstate - - centralize lock of the state file in regional DynamoDB tables in deployment account -- `sample-terraform`: this folder contains an example of terraform pipelines definition and repository -- `adf_terraform_role`: additional role added during the bootstrap of an account to deploy terraform resources -- `adf_locktable`: regional DynamoDB to manage the lock of the state file -- minor changes to adf core roles to enable terraform pipelines - -An overview of the terraform template components and functionalities - -### Overview - -This repository contains a module that manage the deployment of terraform code to multiple accounts and regions. -The module consists of three build stages defined in the following file: - -- `buildspec.yml`: install the version of terraform specified in the pipeline configuration -- `tf_scan.yml`: (optional) returns any vulnerabilities in terraform code according with terrascan utilitiy -- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan -- `tf_apply.yml`: run a terraform apply after the manual step approval - -### Parameters - -- TERRAFORM_VERSION: the terraform version used to deploy the resource -- TARGET_ACCOUNTS: comma separated list of target accounts -- TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) -- REGIONS: comma separated list of target region - -### Deployment procedure - -1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: - -```yaml -- name: sample-terraform - default_providers: - source: - provider: codecommit - properties: - account_id: 111111111111 # source account id - build: - provider: codebuild - properties: - environment_variables: - TERRAFORM_VERSION: "0.14.10" # terraform version - deploy: - provider: codebuild - properties: - image: "STANDARD_5_0" - environment_variables: - TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts - TARGET_OUS: /core/infrastructure,/sandbox # target OUs - MASTER_ACCOUNT_ID: 333333333333 # master account - REGIONS: eu-west-1 # target regions - params: - restart_execution_on_update: true - targets: - - name: terraform-scan # optional - properties: - spec_filename: tf_scan.yml # terraform scan - - name: terraform-plan - properties: - spec_filename: tf_plan.yml # terraform plan - - approval # manual approval - - name: terraform-apply - properties: - spec_filename: tf_apply.yml # terraform apply -``` - -2. Add the project name in params/global.yml file -3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. -4. Add variable definition to tf\variables.tf file and variable values to tfvars/global.auto.tfvars - - - Local variables (per account) can be configured using the following naming convention - - ``` - tfvars <-- This folder contains the structure to define terraform variables - │ - └───global.auto.tfvars <-- this file contains global variables applied to all the target accounts - │ - └───111111111111 <-- this folders contains variable files related to account 111111111111 - │ └──────│ local.auto.tfvars <-- this file contains variables related to account 111111111111 - │ - └───222222222222 <-- this folders contains variable files related to account 222222222222 - └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 - ``` - -5. Push to sample-terraform ADF repository -6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. - -Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created -e.g. Project name: sample-tf-module -Target accounts: 111111111111, 222222222222 -Target regions: eu-west-1 (main ADF region), us-east-1 -The following state files are created - -- 111111111111 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/111111111111.tfstate -- 111111111111 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/111111111111.tfstate -- 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate -- 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate - -A DynamoDB table manage the lock of the state file. It is deployed in every ADF regions named adf_locktable - By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. From 4ce58cc658a11300a9e1a5a59568ef5c411eec6a Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:55:22 +0100 Subject: [PATCH 014/179] Update samples/sample-terraform/README.md Co-authored-by: Stewart Wallace --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 1e57e5e86..bf245dff6 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -15,7 +15,7 @@ The module consists of three build stages defined in the following file: - TERRAFORM_VERSION: the terraform version used to deploy the resource - TARGET_ACCOUNTS: comma separated list of target accounts - TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) -- REGIONS: comma separated list of target region +- REGIONS: comma separated list of target regions ### Deployment procedure From ab489949aa223e1eec50e36a4adcc04de93ae04c Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:56:38 +0100 Subject: [PATCH 015/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 725256098..4185e1e7c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -7,7 +7,6 @@ sts = boto3.client('sts') master_acc_id = os.environ["MASTER_ACCOUNT_ID"] -# master_acc_id = "568100899893" if("TARGET_OUS" in os.environ): ou_path = os.environ["TARGET_OUS"] # ou_path="/core,/sandbox" From d562c389a7095800d7648a674ec1868616ca4f6a Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:57:10 +0100 Subject: [PATCH 016/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py clean comments Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 4185e1e7c..ab9b8489a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -9,8 +9,6 @@ master_acc_id = os.environ["MASTER_ACCOUNT_ID"] if("TARGET_OUS" in os.environ): ou_path = os.environ["TARGET_OUS"] -# ou_path="/core,/sandbox" -# root_id="r-okhg" def list_organizational_units_for_parent(parent_ou): From dec1f81f4faac5487258855c07285266c758ffa5 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:58:32 +0100 Subject: [PATCH 017/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index ab9b8489a..7e00c76a7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -12,7 +12,7 @@ def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') organizational_units = [ ou From 04c16db8e8ce45e6cc84bd720ccea1bd119cbfbd Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:58:50 +0100 Subject: [PATCH 018/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py clean comments Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 7e00c76a7..ca1c241ee 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -13,7 +13,6 @@ def list_organizational_units_for_parent(parent_ou): organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') - # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) From a2eb1250cf5d64651c8c92d0e6a6d1bda1f879c7 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:59:03 +0100 Subject: [PATCH 019/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index ca1c241ee..40aa0b10b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -26,7 +26,7 @@ def get_accounts(): print("Master Account ID: " + master_acc_id) # Assume a role into the Org Master role to get account ID's and emails - organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') for account in paginator(organizations.list_accounts): if account['Status'] == 'ACTIVE': From 20720565701b34c745cc09a8ee951499a3b20cc0 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:59:23 +0100 Subject: [PATCH 020/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 40aa0b10b..f9036875c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -39,7 +39,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') # Read organization root id root_ids = [] From 2278028fa615bd8cbf75601fa9e6bead414d298b Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 22:59:35 +0100 Subject: [PATCH 021/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py remove comments Co-authored-by: Stewart Wallace --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index f9036875c..d1899fb9b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -40,7 +40,6 @@ def get_accounts_from_ous(): parent_ou_id = None account_list = [] organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') - # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') # Read organization root id root_ids = [] for id in paginator(organizations.list_roots): From c6d660dc13993c8fa08fb20175e7d8737046ac75 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 5 Nov 2021 23:01:51 +0100 Subject: [PATCH 022/179] removed .gitignore from sample-terraform --- samples/sample-terraform/.gitignore | 226 ---------------------------- 1 file changed, 226 deletions(-) delete mode 100644 samples/sample-terraform/.gitignore diff --git a/samples/sample-terraform/.gitignore b/samples/sample-terraform/.gitignore deleted file mode 100644 index 572727729..000000000 --- a/samples/sample-terraform/.gitignore +++ /dev/null @@ -1,226 +0,0 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode -# Edit at https://www.toptal.com/developers/gitignore?templates=macos,terraform,python,visualstudiocode - -### macOS ### -# General -.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### Python ### -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -pytestdebug.log - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ -doc/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -#poetry.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -# .env -.env/ -.venv/ -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ -pythonenv* - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# operating system-related files -# file properties cache/storage on macOS -*.DS_Store -# thumbnail cache on Windows -Thumbs.db - -# profiling data -.prof - - -### Terraform ### -# Local .terraform directories -**/.terraform/* - -# .tfstate files -*.tfstate -*.tfstate.* - -# Crash log files -crash.log - -# Ignore any .tfvars files that are generated automatically for each Terraform run. Most -# .tfvars files are managed as part of configuration and so should be included in -# version control. -# -# example.tfvars - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - -### VisualStudioCode ### -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -### VisualStudioCode Patch ### -# Ignore all local history of files -.history -.ionide - -# End of https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode - From b18a0f51ec3d7a9453678d3fd11dc1c575199cf8 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 23:02:58 +0100 Subject: [PATCH 023/179] Update samples/sample-terraform/README.md add details to documenation Co-authored-by: Simon --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index bf245dff6..e1b3c0e7c 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -6,7 +6,7 @@ This repository contains a module that manage the deployment of terraform code t The module consists of three build stages defined in the following file: - `buildspec.yml`: install the version of terraform specified in the pipeline configuration -- `tf_scan.yml`: (optional) returns any vulnerabilities in terraform code according with terrascan utilitiy +- `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. - `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan - `tf_apply.yml`: run a terraform apply after the manual step approval From 5f0ed7f12bd7bc90ff91d26d138366f99a0b0613 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 5 Nov 2021 23:07:35 +0100 Subject: [PATCH 024/179] renamed tf_sec to tf_scan in sample-terraform --- samples/sample-terraform/README.md | 2 +- samples/sample-terraform/{tf_sec.yml => tf_scan.yml} | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename samples/sample-terraform/{tf_sec.yml => tf_scan.yml} (86%) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index e1b3c0e7c..4d0e02d14 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -3,7 +3,7 @@ ## Overview This repository contains a module that manage the deployment of terraform code to multiple accounts and regions. -The module consists of three build stages defined in the following file: +The module consists of four build stages defined in the following file: - `buildspec.yml`: install the version of terraform specified in the pipeline configuration - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. diff --git a/samples/sample-terraform/tf_sec.yml b/samples/sample-terraform/tf_scan.yml similarity index 86% rename from samples/sample-terraform/tf_sec.yml rename to samples/sample-terraform/tf_scan.yml index 14ec65720..02f75f9a9 100644 --- a/samples/sample-terraform/tf_sec.yml +++ b/samples/sample-terraform/tf_scan.yml @@ -9,4 +9,4 @@ phases: build: commands: - cd tf - - terrascan scan -o yaml --skip-rules="AC_AWS_078" + - terrascan scan -o yaml From 89524835cd8ece58bcc60cafb68e1b0120fa3e8f Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 5 Nov 2021 23:15:39 +0100 Subject: [PATCH 025/179] added more details to README.md regarding REGIONS parameters --- samples/sample-terraform/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 4d0e02d14..a7d621904 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -8,14 +8,16 @@ The module consists of four build stages defined in the following file: - `buildspec.yml`: install the version of terraform specified in the pipeline configuration - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. - `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan -- `tf_apply.yml`: run a terraform apply after the manual step approval +- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply + +An optional approval step could be added between plan and apply as shown in the pipeline definition below. ## Parameters - TERRAFORM_VERSION: the terraform version used to deploy the resource - TARGET_ACCOUNTS: comma separated list of target accounts - TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) -- REGIONS: comma separated list of target regions +- REGIONS: comma separated list of target regions. If this parameter is empty, the main ADF region is used. ### Deployment procedure @@ -41,7 +43,7 @@ The module consists of four build stages defined in the following file: TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts TARGET_OUS: /core/infrastructure,/sandbox # target OUs MASTER_ACCOUNT_ID: 333333333333 # master account - REGIONS: eu-west-1 # target regions + REGIONS: eu-west-1 # target regions. Add a comma separated list to define multiple regions e.g. eu-west-1,us-east-1 params: restart_execution_on_update: true targets: From 948e1478f06895bb89e970a9fcabf191321bd92b Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 5 Nov 2021 23:16:29 +0100 Subject: [PATCH 026/179] removed restart_execution_on_update: true --- samples/sample-terraform/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index a7d621904..a427c67c7 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -44,8 +44,6 @@ An optional approval step could be added between plan and apply as shown in the TARGET_OUS: /core/infrastructure,/sandbox # target OUs MASTER_ACCOUNT_ID: 333333333333 # master account REGIONS: eu-west-1 # target regions. Add a comma separated list to define multiple regions e.g. eu-west-1,us-east-1 - params: - restart_execution_on_update: true targets: - name: terraform-scan # optional properties: From 5d18eeb3aec2580db39839294849ea38358510a2 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 23:20:33 +0100 Subject: [PATCH 027/179] Update samples/sample-terraform/README.md Rename parameter from MASTER_ACCOUNT_ID to MANAGEMENT_ACCOUNT_ID Co-authored-by: Simon --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index a427c67c7..3035298ca 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -42,7 +42,7 @@ An optional approval step could be added between plan and apply as shown in the environment_variables: TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts TARGET_OUS: /core/infrastructure,/sandbox # target OUs - MASTER_ACCOUNT_ID: 333333333333 # master account + MANAGEMENT_ACCOUNT_ID: 333333333333 # management account / billing account REGIONS: eu-west-1 # target regions. Add a comma separated list to define multiple regions e.g. eu-west-1,us-east-1 targets: - name: terraform-scan # optional From 23c8d5570d61316db58b812f7922fd185eb72224 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 23:21:05 +0100 Subject: [PATCH 028/179] Update samples/sample-terraform/README.md Co-authored-by: Simon --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 3035298ca..6263c2592 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -75,7 +75,7 @@ An optional approval step could be added between plan and apply as shown in the └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 ``` -5. Push to sample-terraform ADF repository +5. Push to your sample-terraform ADF repository 6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created From 88449f79eed24c33b92e3532a822bdcffd0c7c45 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 23:21:53 +0100 Subject: [PATCH 029/179] Update samples/sample-terraform/README.md Co-authored-by: Simon --- samples/sample-terraform/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 6263c2592..a2c20117a 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -82,7 +82,8 @@ Terraform state files are stored in the regional S3 buckets in the deployment ac e.g. Project name: sample-tf-module Target accounts: 111111111111, 222222222222 Target regions: eu-west-1 (main ADF region), us-east-1 -The following state files are created + +The following state files are created: - 111111111111 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/111111111111.tfstate - 111111111111 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/111111111111.tfstate From d173a2be8c9e44976a57cd2c0f56d6bdbdf78ac4 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 5 Nov 2021 23:23:30 +0100 Subject: [PATCH 030/179] Update samples/sample-terraform/README.md Co-authored-by: Simon --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index a2c20117a..b7cc86934 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -90,4 +90,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table manage the lock of the state file. It is deployed in every ADF regions named adf_locktable +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named adf_locktable. From d234a363feea181b4686eea19c5d0f07ecb6db13 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:04:35 +0100 Subject: [PATCH 031/179] added latest terraform version in sample-terraform example --- samples/sample-terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index b7cc86934..8344558d4 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -34,7 +34,7 @@ An optional approval step could be added between plan and apply as shown in the provider: codebuild properties: environment_variables: - TERRAFORM_VERSION: "0.14.10" # terraform version + TERRAFORM_VERSION: "1.0.10" # terraform version deploy: provider: codebuild properties: @@ -42,7 +42,7 @@ An optional approval step could be added between plan and apply as shown in the environment_variables: TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts TARGET_OUS: /core/infrastructure,/sandbox # target OUs - MANAGEMENT_ACCOUNT_ID: 333333333333 # management account / billing account + MANAGEMENT_ACCOUNT_ID: 333333333333 # management account / billing account REGIONS: eu-west-1 # target regions. Add a comma separated list to define multiple regions e.g. eu-west-1,us-east-1 targets: - name: terraform-scan # optional From 88f3e480d9f21f0a6f27709c90f574b298140fe7 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:06:06 +0100 Subject: [PATCH 032/179] added terraform version support in documetation --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 8344558d4..0794b4f01 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -34,7 +34,7 @@ An optional approval step could be added between plan and apply as shown in the provider: codebuild properties: environment_variables: - TERRAFORM_VERSION: "1.0.10" # terraform version + TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0 deploy: provider: codebuild properties: From cd71d422d13ff4beb48ed47fc9aa62908c256c3d Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:11:06 +0100 Subject: [PATCH 033/179] fixed README.md format --- samples/sample-terraform/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 0794b4f01..63c98e056 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -34,7 +34,7 @@ An optional approval step could be added between plan and apply as shown in the provider: codebuild properties: environment_variables: - TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0 + TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0. deploy: provider: codebuild properties: @@ -78,10 +78,13 @@ An optional approval step could be added between plan and apply as shown in the 5. Push to your sample-terraform ADF repository 6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. -Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created -e.g. Project name: sample-tf-module -Target accounts: 111111111111, 222222222222 -Target regions: eu-west-1 (main ADF region), us-east-1 +Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. + +e.g. + +- Project name: sample-tf-module +- Target accounts: 111111111111, 222222222222 +- Target regions: eu-west-1 (main ADF region), us-east-1 The following state files are created: From 0ffe83cf65fcc6abb73a6aa6360230b130eb9571 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:18:07 +0100 Subject: [PATCH 034/179] added tf pipeline documentation in user guide --- docs/user-guide.md | 411 ++++++++++++++++++++++++++++----------------- 1 file changed, 260 insertions(+), 151 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 97337c452..a45067b14 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1,26 +1,35 @@ # User Guide -- [Deployment Map](#deployment-map) - - [Providers](#providers) - - [Targets Syntax](#targets-syntax) - - [Params](#params) - - [Repositories](#repositories) - - [Completion Triggers](#completion-triggers) - - [Additional Deployment Maps](#additional-deployment-maps) - - [Removing Pipelines](#removing-pipelines) -- [Deploying via Pipelines](#deploying-via-pipelines) - - [BuildSpec](#buildspec) - - [Parameters and Tagging](#cloudformation-parameters-and-tagging) - - [Serverless Transforms](#serverless-transforms) - - [Parameter Injection](#parameter-injection) - - [Nested Stacks](#nested-cloudformation-stacks) - - [Deploying Serverless Applications with SAM](#deploying-serverless-applications-with-sam) - - [Using Anchors and Alias](#using-anchors-and-alias) - - [One to many Relationships](#one-to-many-relationships) +- [User Guide](#user-guide) + - [Deployment Map](#deployment-map) + - [Targeting via Tags](#targeting-via-tags) + - [Important Notes](#important-notes) + - [Zero-prefixed AWS Account Ids](#zero-prefixed-aws-account-ids) + - [Providers](#providers) + - [Targets Syntax](#targets-syntax) + - [Params](#params) + - [Completion Triggers](#completion-triggers) + - [Additional Deployment Maps](#additional-deployment-maps) + - [Repositories](#repositories) + - [Removing Pipelines](#removing-pipelines) + - [Deploying via Pipelines](#deploying-via-pipelines) + - [BuildSpec](#buildspec) + - [Custom Build Images](#custom-build-images) + - [CloudFormation Parameters and Tagging](#cloudformation-parameters-and-tagging) + - [Serverless Transforms](#serverless-transforms) + - [Parameter Injection](#parameter-injection) + - [Retrieving parameter values](#retrieving-parameter-values) + - [Importing output values](#importing-output-values) + - [Uploading assets](#uploading-assets) + - [Nested CloudFormation Stacks](#nested-cloudformation-stacks) + - [Deploying Serverless Applications with SAM](#deploying-serverless-applications-with-sam) + - [Using Anchors and Alias](#using-anchors-and-alias) + - [One to many relationships](#one-to-many-relationships) + - [Terraform pipeline](#terraform-pipeline) ## Deployment Map -The `deployment_map.yml` file *(or [files](#additional-deployment-maps))* lives in the repository named `aws-deployment-framework-pipelines` on the Deployment Account. These files are the general pipeline definitions that are responsible for mapping the specific pipelines to their deployment targets along with their respective parameters. The [AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/home.html) will synthesize during the CodeBuild step within the `aws-deployment-framework-pipelines` pipeline. Prior to the CDK creating these pipeline templates, a input generation step will run to parse the deployment_map.yml files, it will then assume a readonly role on the master account in the Organization that will have access to resolve the accounts in the AWS Organizations OU's specified in the mapping file. It will return the account name and ID for each of the accounts and pass those values into the input files that will go on to be main CDK applications inputs. +The `deployment_map.yml` file _(or [files](#additional-deployment-maps))_ lives in the repository named `aws-deployment-framework-pipelines` on the Deployment Account. These files are the general pipeline definitions that are responsible for mapping the specific pipelines to their deployment targets along with their respective parameters. The [AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/home.html) will synthesize during the CodeBuild step within the `aws-deployment-framework-pipelines` pipeline. Prior to the CDK creating these pipeline templates, a input generation step will run to parse the deployment_map.yml files, it will then assume a readonly role on the master account in the Organization that will have access to resolve the accounts in the AWS Organizations OU's specified in the mapping file. It will return the account name and ID for each of the accounts and pass those values into the input files that will go on to be main CDK applications inputs. The deployment map file defines the pipelines along with their inputs, providers to use and their configuration. It also defines the targets of the @@ -44,7 +53,7 @@ pipelines: properties: account_id: 111112233332 # The AWS Account where the source code will be in a CodeCommit Repository params: - notification_endpoint: janes_team@doe.com # Optional + notification_endpoint: janes_team@doe.com # Optional tags: foo: bar # Pipelines support tagging targets: @@ -63,17 +72,17 @@ pipelines: oauth_token_path: /adf/github_token # The path in AWS Secrets Manager that holds the GitHub Oauth token, ADF only has access to /adf/ prefix in Secrets Manager json_field: token # The field (key) name of the json object stored in AWS Secrets Manager that holds the Oauth token params: - notification_endpoint: joes_team@company.nl + notification_endpoint: joes_team@company.nl targets: - path: /banking/testing name: fancy-name #Optional way to pass a name for this stage in the pipeline ``` -In the above example we are creating two pipelines with AWS CodePipeline. The first one will deploy from a repository named **iam** that lives in the account **123456789101**. This CodeCommit Repository will automatically be created by default in the 123456789101 AWS Account if it does not exist. The automatic repository creation occurs if you enable `'auto-create-repositories'` (which is enabled by default). The `iam` pipeline will use AWS CodeCommit as its source and deploy in 3 steps. The first stage of the deployment will occur against all AWS Accounts that are in the `/security` Organization unit and be targeted to the `eu-west-1` region. After that, there is a manual approval phase which is denoted by the keyword `approval`. The next step will be targeted to the accounts within the `/banking/testing` OU *(in your default deployment account region)* region. By providing a simple path without a region definition it will default to the region chosen as the deployment account region in your [adfconfig](./admin-guide/adfconfig.yml). Any failure during the pipeline will cause it to halt. +In the above example we are creating two pipelines with AWS CodePipeline. The first one will deploy from a repository named **iam** that lives in the account **123456789101**. This CodeCommit Repository will automatically be created by default in the 123456789101 AWS Account if it does not exist. The automatic repository creation occurs if you enable `'auto-create-repositories'` (which is enabled by default). The `iam` pipeline will use AWS CodeCommit as its source and deploy in 3 steps. The first stage of the deployment will occur against all AWS Accounts that are in the `/security` Organization unit and be targeted to the `eu-west-1` region. After that, there is a manual approval phase which is denoted by the keyword `approval`. The next step will be targeted to the accounts within the `/banking/testing` OU _(in your default deployment account region)_ region. By providing a simple path without a region definition it will default to the region chosen as the deployment account region in your [adfconfig](./admin-guide/adfconfig.yml). Any failure during the pipeline will cause it to halt. -The second pipeline (*vpc*) example deploys to an OU path `/banking/testing`. You can choose between an absolute path in your AWS Organization, AWS Account ID or an array of OUs or IDs. This pipeline also uses Github as a source rather than AWS CodeCommit. When generating the pipeline, ADF expects [GitHub Token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) to be placed in AWS Secrets Manager in a path prefixed with `/adf/`. +The second pipeline (_vpc_) example deploys to an OU path `/banking/testing`. You can choose between an absolute path in your AWS Organization, AWS Account ID or an array of OUs or IDs. This pipeline also uses Github as a source rather than AWS CodeCommit. When generating the pipeline, ADF expects [GitHub Token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) to be placed in AWS Secrets Manager in a path prefixed with `/adf/`. -By default, the above pipelines will be created to deploy CloudFormation using a change in two actions *(Create then Execute)*. +By default, the above pipelines will be created to deploy CloudFormation using a change in two actions _(Create then Execute)_. #### Targeting via Tags @@ -82,13 +91,12 @@ Tags on AWS Accounts can also be used to define stages within a pipeline. For ex We do that with the following syntax: ```yaml - - name: vpc-for-foo-team - default_providers: - ... - targets: - - tags: # Using tags to define the stage rather than a path or account id - cost-center: foo-team - name: foo-team # You can optionally use the name key to give this stage some meaningful name +- name: vpc-for-foo-team + default_providers: ... + targets: + - tags: # Using tags to define the stage rather than a path or account id + cost-center: foo-team + name: foo-team # You can optionally use the name key to give this stage some meaningful name ``` Adding or Removing Tags to an AWS Account in AWS Organizations will automatically trigger a run of the bootstrap pipeline which will in turn execute the pipeline generation pipeline in the deployment account. @@ -137,11 +145,11 @@ pipelines: The pipeline `sample-ec2-java-app-codedeploy` has a `default_providers` key that defines the high-level structure of the pipeline. -It explicitly defines the source *(requirement for all pipelines)* and also +It explicitly defines the source _(requirement for all pipelines)_ and also defines what type of build will occur along with any associated parameters. In this example, we're explicitly saying we want to use AWS CodeBuild -*(which is also the default)* and also to use a specific Docker Image for build +_(which is also the default)_ and also to use a specific Docker Image for build stage. The default deployment provider for this pipeline is configured to be `codedeploy` in this example. This means that any of the targets of the pipeline will use AWS CodeDeploy as their default @@ -149,7 +157,7 @@ pipeline will use AWS CodeDeploy as their default In the targets section itself we have the opportunity to override the provider itself or pass in any additional properties to that provider. In this example -we are passing in `application_name` and *deployment_group_name* as properties +we are passing in `application_name` and _deployment_group_name_ as properties to CodeDeploy for this specific stage. The `properties` can either be defined by changing the `default_providers` configuration or get updated at the stage level. Stage level config overrides default provider config. @@ -162,14 +170,14 @@ For detailed information on providers and their supported properties, see the ### Targets Syntax -The Deployment Map has a shorthand syntax along with a more detailed version when you need extra configuration for the *targets* key as detailed below: +The Deployment Map has a shorthand syntax along with a more detailed version when you need extra configuration for the _targets_ key as detailed below: **Shorthand:** ```yaml targets: - 9999999999 # Single Account, Deployment Account Region - - /my_ou/production # Group of Accounts, Deployment Account Region + - /my_ou/production # Group of Accounts, Deployment Account Region ``` **Detailed:** @@ -195,60 +203,62 @@ Pipelines also have parameters that don't relate to a specific stage but rather The following are the available pipeline parameters: -- *notification_endpoint* *(String)* defaults to none. +- _notification_endpoint_ _(String)_ defaults to none. + > Can either be a valid email address or a string that represents the name of a Slack Channel. In order to integrate ADF with Slack see [Integrating with Slack](./admin-guide.md) in the admin guide. By Default, Notifications will be sent when pipelines Start, Complete or Fail. -- *schedule* *(String)* defaults to none. +- _schedule_ _(String)_ defaults to none. + > If the Pipeline should execute on a specific Schedule. Schedules are defined by using a Rate or an Expression. See [here](https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#RateExpressions) for more information on how to define Rate or an Expression. -- *restart_execution_on_update* *(Boolean)* default: `False`. +- _restart_execution_on_update_ _(Boolean)_ default: `False`. > If the Pipeline should start a new execution if its structure is updated. Pipelines can often update their structure if targets of the pipeline are Organizational Unit paths. This setting allows pipelines to automatically run once an AWS Account has been moved in or out of a targeted OU. ### Completion Triggers -Pipelines can also trigger other pipelines upon completion. To do this, use the *completion_trigger* key on the pipeline definition. For example: +Pipelines can also trigger other pipelines upon completion. To do this, use the _completion_trigger_ key on the pipeline definition. For example: ```yaml - - name: ami-builder - default_providers: - source: - provider: codecommit - properties: - account_id: 222222222222 - build: - provider: codebuild - role: packer - size: medium - params: - schedule: rate(7 days) - completion_trigger: # What should happen when this pipeline completes - pipelines: - - my-web-app-pipeline # Start this pipeline - - - name: my-web-app-pipeline - default_providers: - source: - provider: github - properties: - repository: my-web-app - owner: cool_coder - oauth_token_path: /adf/github_token - json_field: token - targets: - - path: /banking/testing - name: web-app-testing +- name: ami-builder + default_providers: + source: + provider: codecommit + properties: + account_id: 222222222222 + build: + provider: codebuild + role: packer + size: medium + params: + schedule: rate(7 days) + completion_trigger: # What should happen when this pipeline completes + pipelines: + - my-web-app-pipeline # Start this pipeline + +- name: my-web-app-pipeline + default_providers: + source: + provider: github + properties: + repository: my-web-app + owner: cool_coder + oauth_token_path: /adf/github_token + json_field: token + targets: + - path: /banking/testing + name: web-app-testing ``` -In the above example, the *ami-builder* pipeline runs every 7 days based on its schedule. When it completes, it executes the *my-web-app-pipeline* pipeline as defined in its *completion_trigger* property. +In the above example, the _ami-builder_ pipeline runs every 7 days based on its schedule. When it completes, it executes the _my-web-app-pipeline_ pipeline as defined in its _completion_trigger_ property. ### Additional Deployment Maps You can also create additional deployment map files. -These can live in a folder in the pipelines repository called *deployment_maps*. +These can live in a folder in the pipelines repository called _deployment_maps_. These are entirely optional, but can help split up complex environments with many pipelines. For example, you might have a map used for infrastructure type pipelines and one used for deploying applications. -These additional deployment map files can have any name, as long as they end with *.yml*. +These additional deployment map files can have any name, as long as they end with _.yml_. Taking it a step further, you can create a map per service. So you can organize these deployment map files inside your preferred directory structure @@ -272,7 +282,7 @@ Source entities for pipelines can consist of AWS CodeCommit Repositories, Amazon ### Removing Pipelines -If you decide you no longer require a specific pipeline you can remove it from the deployment_map.yml file and commit those changes back to the *aws-deployment-framework-pipelines* repository *(on the deployment account)* in order for it to be cleaned up. The resources that were created as outputs from this pipeline will **not** be removed by this process. +If you decide you no longer require a specific pipeline you can remove it from the deployment*map.yml file and commit those changes back to the \_aws-deployment-framework-pipelines* repository _(on the deployment account)_ in order for it to be cleaned up. The resources that were created as outputs from this pipeline will **not** be removed by this process. ## Deploying via Pipelines @@ -292,24 +302,24 @@ phases: - pip install -r adf-build/requirements.txt -q # Install Requirements via requirements.txt - python adf-build/generate_params.py # Generate Parameter files dynamically artifacts: - files: '**/*' # Package up all outputs and pass them along to next stage + files: "**/*" # Package up all outputs and pass them along to next stage ``` -In the example we have three steps to our install phase in our build, the remaining phases and steps you add are up to you. In the above steps we simply bring in the shared modules we will need to run our main function in *generate_params.py*. The $S3_BUCKET_NAME variable is available in AWS CodeBuild as we pass this in from our initial creation of the that defines the CodeBuild Project. You do not need to change this. +In the example we have three steps to our install phase in our build, the remaining phases and steps you add are up to you. In the above steps we simply bring in the shared modules we will need to run our main function in _generate_params.py_. The $S3_BUCKET_NAME variable is available in AWS CodeBuild as we pass this in from our initial creation of the that defines the CodeBuild Project. You do not need to change this. -Other packages such as [cfn-lint](https://github.com/awslabs/cfn-python-lint) can be installed in order to validate that our CloudFormation templates are up to standard and do not contain any obvious errors. If you wish to add in any extra packages you can add them to the *requirements.txt* in the `bootstrap_repository` which is brought down into AWS CodeBuild and installed. Otherwise you can add them into any pipelines specific buildspec.yml. +Other packages such as [cfn-lint](https://github.com/awslabs/cfn-python-lint) can be installed in order to validate that our CloudFormation templates are up to standard and do not contain any obvious errors. If you wish to add in any extra packages you can add them to the _requirements.txt_ in the `bootstrap_repository` which is brought down into AWS CodeBuild and installed. Otherwise you can add them into any pipelines specific buildspec.yml. -If you wish to hide away the steps that can occur in AWS CodeBuild, you can move the *buildspec.yml* content itself into the pipeline by using the *spec_inline* property in your map files. By doing this, you can remove the option to have a buildspec.yml in the source repository at all. This is a potential way to enforce certain build steps for certain pipeline types. +If you wish to hide away the steps that can occur in AWS CodeBuild, you can move the _buildspec.yml_ content itself into the pipeline by using the _spec_inline_ property in your map files. By doing this, you can remove the option to have a buildspec.yml in the source repository at all. This is a potential way to enforce certain build steps for certain pipeline types. #### Custom Build Images + You can use [custom build](https://aws.amazon.com/blogs/devops/extending-aws-codebuild-with-custom-build-environments/) environments in AWS CodeBuild. This can be defined in the your deployment map files like so: ```yaml pipelines: - name: example-custom-image default_providers: - source: - ... + source: ... build: provider: codebuild image: @@ -321,20 +331,20 @@ pipelines: ### CloudFormation Parameters and Tagging -When you define CloudFormation templates as artifacts to push through a pipeline you might want to have a set of parameters associated with the templates. You can utilize the `params` folder in your repository to add in parameters as you see fit. To avoid having to create a parameter file for each of the stacks you wish to deploy to, you can create a parameter file called `global.yml` *(or .json)* any parameters defined in this file will be merged into the parameters for any specific account parameter file at build time. For example you might have a single parameter for a template called `CostCenter` the value of this will be the same across every deployment of your application however you might have another parameter called `InstanceType` that you want to be different per account. Using this example we can create a `global.yml` file that contains the following content: +When you define CloudFormation templates as artifacts to push through a pipeline you might want to have a set of parameters associated with the templates. You can utilize the `params` folder in your repository to add in parameters as you see fit. To avoid having to create a parameter file for each of the stacks you wish to deploy to, you can create a parameter file called `global.yml` _(or .json)_ any parameters defined in this file will be merged into the parameters for any specific account parameter file at build time. For example you might have a single parameter for a template called `CostCenter` the value of this will be the same across every deployment of your application however you might have another parameter called `InstanceType` that you want to be different per account. Using this example we can create a `global.yml` file that contains the following content: ```yaml Parameters: - CostCenter: department-abc + CostCenter: department-abc ``` -This can be represented in *json* in the same way if desired. +This can be represented in _json_ in the same way if desired. ```json { - "Parameters": { - "CostCenter": "department-abc" - } + "Parameters": { + "CostCenter": "department-abc" + } } ``` @@ -342,20 +352,20 @@ Then we can have a more specific parameter for another account, this file should ```yaml Parameters: - InstanceType: m5.large + InstanceType: m5.large ``` When the stack is executed it will be executed with the following parameters: ```yaml Parameters: - InstanceType: m5.large - CostCenter: department-abc + InstanceType: m5.large + CostCenter: department-abc ``` -This aggregation of parameters works for a few different levels, where the most specific level takes precedence. In the example above, if *CostCenter* is defined in both `global.yml` and `account.yml` *("account" here represents the name of the account)* then the value in the `account.yml` file will take precedence. +This aggregation of parameters works for a few different levels, where the most specific level takes precedence. In the example above, if _CostCenter_ is defined in both `global.yml` and `account.yml` _("account" here represents the name of the account)_ then the value in the `account.yml` file will take precedence. -The different types of parameter files and their order of precedence *(in the tree below, the lowest level has the highest precedence)* can be used to simplify how parameters are specified. For example, a parameter such as `Environment` might be the same for all accounts under a certain OU, so placing it under a single `ou.yml` params file means you don't need to populate it for each account under that OU. +The different types of parameter files and their order of precedence _(in the tree below, the lowest level has the highest precedence)_ can be used to simplify how parameters are specified. For example, a parameter such as `Environment` might be the same for all accounts under a certain OU, so placing it under a single `ou.yml` params file means you don't need to populate it for each account under that OU. **Note:** When using OU parameter files, the OU must be specified in the deployment map as a target. If only the account number is in the deployment map the corresponding OU parameter file will not be referenced. @@ -377,53 +387,53 @@ This concept also works for applying **Tags** to the resources within your stack ```yml Parameters: - CostCenter: '123' - Environment: testing + CostCenter: "123" + Environment: testing Tags: - TagKey: TagValue - MyKey: MyValue + TagKey: TagValue + MyKey: MyValue ``` -Again this example in *json* would look like: +Again this example in _json_ would look like: ```json { - "Parameters": { - "CostCenter": "123", - "Environment": "testing" - }, - "Tags": { - "TagKey": "TagValue", - "MyKey": "MyValue" - } + "Parameters": { + "CostCenter": "123", + "Environment": "testing" + }, + "Tags": { + "TagKey": "TagValue", + "MyKey": "MyValue" + } } ``` This means that all resources that support tags within your CloudFormation stack will be tagged as defined above. -It is important to keep in mind that each Deployment Provider *(Code Deploy, CloudFormation, Service Catalog etc)* have their [own parameter structure](https://docs.aws.amazon.com/codepipeline/latest/userguide/reference-pipeline-structure.html) and configuration files. For example, Service catalog allows you to pass a configuration file as such: +It is important to keep in mind that each Deployment Provider _(Code Deploy, CloudFormation, Service Catalog etc)_ have their [own parameter structure](https://docs.aws.amazon.com/codepipeline/latest/userguide/reference-pipeline-structure.html) and configuration files. For example, Service catalog allows you to pass a configuration file as such: ```json { - "SchemaVersion": "1.1", - "ProductVersionName": "test", - "ProductVersionDescription": "My awesome product", - "ProductType": "CLOUD_FORMATION_TEMPLATE", - "Properties": { - "TemplateFilePath": "/template.yml" - } + "SchemaVersion": "1.1", + "ProductVersionName": "test", + "ProductVersionDescription": "My awesome product", + "ProductType": "CLOUD_FORMATION_TEMPLATE", + "Properties": { + "TemplateFilePath": "/template.yml" + } } ``` -You can create the above parameter files if you are deploying products to your Service Catalog's in the same fashion as with CloudFormation *(global.yml etc)*. +You can create the above parameter files if you are deploying products to your Service Catalog's in the same fashion as with CloudFormation _(global.yml etc)_. For more examples of parameters and their usage see the `samples` folder in the root of the repository. -*Note:* Currently only Strings type values are supported as parameters to CloudFormation templates when deploying via AWS CodePipeline. +_Note:_ Currently only Strings type values are supported as parameters to CloudFormation templates when deploying via AWS CodePipeline. ### Serverless Transforms -If the template that is being deployed contains a transform, such as a Serverless Transform it needs to be packaged and uploaded to S3 in every region where it will be deployed. This can be achieved by setting the `CONTAINS_TRANSFORM` environment variable to *True* in your pipeline definition with a deployment map file. Once the environment variable has been set, within your *buildspec.yml* file you will need to use the *package_transform.sh* helper script (`bash adf-build/helpers/package_transform.sh`). This script will package your template to each region and transparently generate a region specific template for the pipeline deploy stages. +If the template that is being deployed contains a transform, such as a Serverless Transform it needs to be packaged and uploaded to S3 in every region where it will be deployed. This can be achieved by setting the `CONTAINS_TRANSFORM` environment variable to _True_ in your pipeline definition with a deployment map file. Once the environment variable has been set, within your _buildspec.yml_ file you will need to use the _package_transform.sh_ helper script (`bash adf-build/helpers/package_transform.sh`). This script will package your template to each region and transparently generate a region specific template for the pipeline deploy stages. ```yaml pipelines: @@ -444,7 +454,7 @@ pipelines: ### Parameter Injection -Parameter injection solves problems that occur with Cross Account parameter access. This concept allows the resolution of values directly from SSM Parameter Store within the Deployment account into Parameter files *(eg global.json, account-name.json)* and also importing of output values from CloudFormation stacks across accounts and regions. +Parameter injection solves problems that occur with Cross Account parameter access. This concept allows the resolution of values directly from SSM Parameter Store within the Deployment account into Parameter files _(eg global.json, account-name.json)_ and also importing of output values from CloudFormation stacks across accounts and regions. #### Retrieving parameter values @@ -452,16 +462,16 @@ If you wish to resolve values from Parameter Store on the Deployment Account dir ```yaml Parameters: - Environment: development - InstanceType: m5.large - SomeValueFromSSM: resolve:/my/path/to/value + Environment: development + InstanceType: m5.large + SomeValueFromSSM: resolve:/my/path/to/value ``` -When you use the special keyword **"resolve:"**, the value in the specified path will be fetched from Parameter Store on the deployment account during the CodeBuild Containers execution and populated into the parameter file for each account you have defined. If you plan on using any sensitive data, ensure you are using the [NoEcho](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html) property to ensure it is kept out of the console and logs. Resolving parameters across regions is also possible using the notation of *resolve:region:/my/path/to/value*. This allows you to fetch values from the deployment account in other regions other than the main deployment region. +When you use the special keyword **"resolve:"**, the value in the specified path will be fetched from Parameter Store on the deployment account during the CodeBuild Containers execution and populated into the parameter file for each account you have defined. If you plan on using any sensitive data, ensure you are using the [NoEcho](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html) property to ensure it is kept out of the console and logs. Resolving parameters across regions is also possible using the notation of _resolve:region:/my/path/to/value_. This allows you to fetch values from the deployment account in other regions other than the main deployment region. To highlight an example of how Parameter Injection can work well, think of the following scenario: You have some value that you wish to rotate on a monthly basis. You have some automation in place that updates the value of a Parameter store parameter on a schedule. Each time this pipeline runs it will check for that value and update the resources accordingly, effectively detaching the parameters from the pipeline itself. -There is also the concept of optionally resolving or importing values. This can be achieved by ending the import or resolve function with a **?**. For example, if you want to resolve a value from Parameter Store that might or might not yet exist you can use an optional resolve *(eg resolve:/my/path/to/myMagicKey?)*. If the key *myMagicKey* does not exist in Parameter Store then an empty string will be returned as the value. +There is also the concept of optionally resolving or importing values. This can be achieved by ending the import or resolve function with a **?**. For example, if you want to resolve a value from Parameter Store that might or might not yet exist you can use an optional resolve _(eg resolve:/my/path/to/myMagicKey?)_. If the key _myMagicKey_ does not exist in Parameter Store then an empty string will be returned as the value. #### Importing output values @@ -469,24 +479,24 @@ Parameter injection is also useful for importing output values from CloudFormati ```yaml Parameters: - BucketInLoggingAccount: 'import:123456789101:eu-west-1:stack_name:output_key' + BucketInLoggingAccount: "import:123456789101:eu-west-1:stack_name:output_key" ``` -In the above example *123456789101* is the AWS Account Id in which we want to pull a value from, *eu-west-1* is the region, stack_name is the CloudFormation stack name and *output_key* is the output key name *(not export name)*. Again, this concept works with the optional style syntax *(eg, import:123456789101:eu-west-1:stack_name:output_key?)* if the key *output_key* does not exist at the point in time when this specific import is executed, it will return an empty string as the parameter value rather than an error since it is considered optional. +In the above example _123456789101_ is the AWS Account Id in which we want to pull a value from, _eu-west-1_ is the region, stack*name is the CloudFormation stack name and \_output_key* is the output key name _(not export name)_. Again, this concept works with the optional style syntax _(eg, import:123456789101:eu-west-1:stack_name:output_key?)_ if the key _output_key_ does not exist at the point in time when this specific import is executed, it will return an empty string as the parameter value rather than an error since it is considered optional. #### Uploading assets -Another built-in function is **upload**, You can use *upload* to perform an automated upload of a resource such as a template or file into Amazon S3 as part of the build process. -Once the upload is complete, the Amazon S3 URL for the object will be put in place of the *upload* string in the parameter file. +Another built-in function is **upload**, You can use _upload_ to perform an automated upload of a resource such as a template or file into Amazon S3 as part of the build process. +Once the upload is complete, the Amazon S3 URL for the object will be put in place of the _upload_ string in the parameter file. -For example, If you are deploying products that will be made available via Service Catalog to many teams throughout your organization *(see samples)* you will need to reference the AWS CloudFormation template URL of the product as part of the template that creates the product definition. The problem that the **upload** function is solving in this case is that the template URL of the product cannot exist at this point, since the file has not yet been uploaded to S3. +For example, If you are deploying products that will be made available via Service Catalog to many teams throughout your organization _(see samples)_ you will need to reference the AWS CloudFormation template URL of the product as part of the template that creates the product definition. The problem that the **upload** function is solving in this case is that the template URL of the product cannot exist at this point, since the file has not yet been uploaded to S3. ```yaml Parameters: - ProductYTemplateURL: 'upload:path:productY/template.yml' + ProductYTemplateURL: "upload:path:productY/template.yml" ``` -In the above example, we are calling the **upload** function on a file called `template.yml` that lives in the *productY* folder within our repository and then returning the path style URL from S3 (indicated by the word *path* in the string). The string *"upload:path:productY/template.yml"* will be replaced by the URL of the object in S3 once it has been uploaded. +In the above example, we are calling the **upload** function on a file called `template.yml` that lives in the _productY_ folder within our repository and then returning the path style URL from S3 (indicated by the word _path_ in the string). The string _"upload:path:productY/template.yml"_ will be replaced by the URL of the object in S3 once it has been uploaded. Syntax: @@ -500,25 +510,25 @@ upload:${region}:${style}:${local_path} There are five different styles that one could choose from. -* `path` style, as shown in the example above, will return the S3 path to the object as. +- `path` style, as shown in the example above, will return the S3 path to the object as. This is referred to as the classic [Path Style method](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). - * In case the bucket is stored in us-east-1, it will return: + - In case the bucket is stored in us-east-1, it will return: `https://s3.amazonaws.com/${bucket}/${key}` - * In case the bucket is stored in any other region, it will return: + - In case the bucket is stored in any other region, it will return: `https://s3-${region}.amazonaws.com/${bucket}/${key}` -* `virtual-hosted` style, will return the S3 location using the virtual hosted bucket domain. - * In case the bucket is stored in us-east-1, it will return: +- `virtual-hosted` style, will return the S3 location using the virtual hosted bucket domain. + - In case the bucket is stored in us-east-1, it will return: `https://${bucket}.s3.amazonaws.com/${key}` - * In case the bucket is stored in any other region, it will return: + - In case the bucket is stored in any other region, it will return: `https://${bucket}.s3-${region}.amazonaws.com/${key}` -* `s3-url` style, will return the S3 location using S3 URL with the `s3://` protocol. - As an example, this style is required for [CloudFormation AWS::Include transform](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/create-reusable-transform-function-snippets-and-add-to-your-template-with-aws-include-transform.html). - * It returns: `s3://${bucket}/${key}` -* `s3-uri` style, will return the S3 location using S3 URI without specifying a protocol. +- `s3-url` style, will return the S3 location using S3 URL with the `s3://` protocol. + As an example, this style is required for [CloudFormation AWS::Include transform](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/create-reusable-transform-function-snippets-and-add-to-your-template-with-aws-include-transform.html). + - It returns: `s3://${bucket}/${key}` +- `s3-uri` style, will return the S3 location using S3 URI without specifying a protocol. As an example, this style is required for [CodeBuild project source locations](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-codebuild-project-source.html#cfn-codebuild-project-source-location). - * It returns: `${bucket}/${key}` -* `s3-key-only` style, similar to `s3-uri` but it will only return the `key` value. - * It returns: `${key}` + - It returns: `${bucket}/${key}` +- `s3-key-only` style, similar to `s3-uri` but it will only return the `key` value. + - It returns: `${key}` The `region` is optional. This allows you to upload files to S3 Buckets within specific regions by @@ -530,11 +540,11 @@ the location where `adf-build/generate-params.py` scripts gets executed from. As shown in the example shared above, the file to upload would be the `productY/template.yml` file that is stored in the root of the repository. -The bucket being used to hold the uploaded object is the same Amazon S3 Bucket that holds deployment artifacts *(On the Deployment Account)* for the specific region which they are intended to be deployed to. Files that are uploaded using this functionality will receive a random name each time they are uploaded. +The bucket being used to hold the uploaded object is the same Amazon S3 Bucket that holds deployment artifacts _(On the Deployment Account)_ for the specific region which they are intended to be deployed to. Files that are uploaded using this functionality will receive a random name each time they are uploaded. ### Nested CloudFormation Stacks -AWS CloudFormation allows stacks to create other stacks via the [nested stacks](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html) feature. ADF supports a single entry template which defaults to `template.yml`, the stacks that you wish to nest will need to spawn from this template. Nested stacks allow users to pass a `TemplateURL` value that points directly to another CloudFormation template that is either in S3 or on the File System. If you reference a template on the file system you will need to use the `package_transform.sh` helper script during AWS CodeBuild execution *(during the build phase)* in your pipeline to package up the contents of your templates into finalized artifacts. +AWS CloudFormation allows stacks to create other stacks via the [nested stacks](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html) feature. ADF supports a single entry template which defaults to `template.yml`, the stacks that you wish to nest will need to spawn from this template. Nested stacks allow users to pass a `TemplateURL` value that points directly to another CloudFormation template that is either in S3 or on the File System. If you reference a template on the file system you will need to use the `package_transform.sh` helper script during AWS CodeBuild execution _(during the build phase)_ in your pipeline to package up the contents of your templates into finalized artifacts. This can be achieved with a `buildspec.yml` like so: @@ -551,25 +561,25 @@ phases: commands: - bash adf-build/helpers/package_transform.sh artifacts: - files: '**/*' + files: "**/*" ``` This allows us to specify nested stacks that are in the same repository as our main `template.yml` in our like so: ```yaml - MyStack: - Type: "AWS::CloudFormation::Stack" - Properties: - TemplateURL: another_template.yml # file path to the nested stack template +MyStack: + Type: "AWS::CloudFormation::Stack" + Properties: + TemplateURL: another_template.yml # file path to the nested stack template ``` -When the `package_transform.sh` command is executed, the file will be packaged up and uploaded to Amazon S3. Its *TemplateURL* key will be updated to point to the object in S3 and this will be a valid path when `template.yml` is executed in the deploy stages of your pipeline. +When the `package_transform.sh` command is executed, the file will be packaged up and uploaded to Amazon S3. Its _TemplateURL_ key will be updated to point to the object in S3 and this will be a valid path when `template.yml` is executed in the deploy stages of your pipeline. ### Deploying Serverless Applications with SAM -Serverless Applications can also be deployed via ADF *(see samples)*. The only extra step required to deploy a SAM template is that you execute `bash adf-build/helpers/package_transform.sh` from within your build stage like so: +Serverless Applications can also be deployed via ADF _(see samples)_. The only extra step required to deploy a SAM template is that you execute `bash adf-build/helpers/package_transform.sh` from within your build stage like so: -For example, deploying a NodeJS Serverless Application from AWS CodeBuild with the *aws/codebuild/standard:2.0* image can be done with a *buildspec.yml* that looks like the following [read more](https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#runtime-versions-buildspec-file): +For example, deploying a NodeJS Serverless Application from AWS CodeBuild with the _aws/codebuild/standard:2.0_ image can be done with a _buildspec.yml_ that looks like the following [read more](https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#runtime-versions-buildspec-file): ```yaml version: 0.2 @@ -588,12 +598,12 @@ phases: commands: - bash adf-build/helpers/package_transform.sh artifacts: - files: '**/*' + files: "**/*" ``` ### Using Anchors and Alias -You can take advantage of YAML Anchors and Alias' in the deployment map files. As you can see from the example below, The &generic_params and &generic_targets are anchors. They can be added to any mapping, sequence or scalar. Once you create an anchor, you can reference it anywhere within the map again with its alias *(eg *generic_params)* to reproduce their values, similar to variables. +You can take advantage of YAML Anchors and Alias' in the deployment map files. As you can see from the example below, The &generic_params and &generic_targets are anchors. They can be added to any mapping, sequence or scalar. Once you create an anchor, you can reference it anywhere within the map again with its alias *(eg *generic_params)\* to reproduce their values, similar to variables. ```yaml pipelines: @@ -659,4 +669,103 @@ pipelines: targets: *generic_targets ``` -By passing in the Repository name *(repository)* we are overriding the **name** property which normally is the name of our associated repository. This will tie both of these pipelines to the single *sample-vpc* repository on the *111111111111* AWS Account. +By passing in the Repository name _(repository)_ we are overriding the **name** property which normally is the name of our associated repository. This will tie both of these pipelines to the single _sample-vpc_ repository on the _111111111111_ AWS Account. + +### Terraform pipeline + +**Overview** + +ADF support the deployment of terraform code to multiple accounts and regions through terraform pipelines. +The module consists of four build stages defined in the following file: + +- `buildspec.yml`: install the version of terraform specified in the pipeline configuration +- `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. +- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan +- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply + +An optional approval step could be added between plan and apply as shown in the pipeline definition below. + +Refer to sample-terraform repository in samples for an example of terraform pipeline. + +**Parameters** + +- TERRAFORM_VERSION: the terraform version used to deploy the resource +- TARGET_ACCOUNTS: comma separated list of target accounts +- TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) +- REGIONS: comma separated list of target regions. If this parameter is empty, the main ADF region is used. + +**Deployment procedure** + +1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: + +```yaml +- name: sample-terraform + default_providers: + source: + provider: codecommit + properties: + account_id: 111111111111 # source account id + build: + provider: codebuild + properties: + environment_variables: + TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0. + deploy: + provider: codebuild + properties: + image: "STANDARD_5_0" + environment_variables: + TARGET_ACCOUNTS: 111111111111,222222222222 # target accounts + TARGET_OUS: /core/infrastructure,/sandbox # target OUs + MANAGEMENT_ACCOUNT_ID: 333333333333 # management account / billing account + REGIONS: eu-west-1 # target regions. Add a comma separated list to define multiple regions e.g. eu-west-1,us-east-1 + targets: + - name: terraform-scan # optional + properties: + spec_filename: tf_scan.yml # terraform scan + - name: terraform-plan + properties: + spec_filename: tf_plan.yml # terraform plan + - approval # manual approval + - name: terraform-apply + properties: + spec_filename: tf_apply.yml # terraform apply +``` + +2. Add the project name in params/global.yml file +3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. +4. Add variable definition to tf\variables.tf file and variable values to tfvars/global.auto.tfvars + + - Local variables (per account) can be configured using the following naming convention + + ``` + tfvars <-- This folder contains the structure to define terraform variables + │ + └───global.auto.tfvars <-- this file contains global variables applied to all the target accounts + │ + └───111111111111 <-- this folders contains variable files related to account 111111111111 + │ └──────│ local.auto.tfvars <-- this file contains variables related to account 111111111111 + │ + └───222222222222 <-- this folders contains variable files related to account 222222222222 + └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 + ``` + +5. Push to your sample-terraform ADF repository +6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. + +Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. + +e.g. + +- Project name: sample-tf-module +- Target accounts: 111111111111, 222222222222 +- Target regions: eu-west-1 (main ADF region), us-east-1 + +The following state files are created: + +- 111111111111 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/111111111111.tfstate +- 111111111111 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/111111111111.tfstate +- 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate +- 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate + +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named adf_locktable. From 7440ae328048ef2d22a055597fc04861eba1218a Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:19:53 +0100 Subject: [PATCH 035/179] added s3 public access block to terraform sample --- samples/sample-terraform/tf/s3.tf | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/samples/sample-terraform/tf/s3.tf b/samples/sample-terraform/tf/s3.tf index 8894dfddb..0beb9d026 100644 --- a/samples/sample-terraform/tf/s3.tf +++ b/samples/sample-terraform/tf/s3.tf @@ -1,4 +1,11 @@ -resource "aws_s3_bucket" "b" { +resource "aws_s3_bucket" "s3" { bucket = "my-tf-test-bucket-${var.TARGET_REGION}-${var.TARGET_ACCOUNT_ID}" acl = "private" } + +resource "aws_s3_bucket_public_access_block" "s3-public-block" { + bucket = aws_s3_bucket.s3.id + + block_public_acls = true + block_public_policy = true +} From bb524710c6038504e2cdf87baee74232f0b375bb Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:27:29 +0100 Subject: [PATCH 036/179] removed TF_STAGE as adf_terraform.sh input parameter --- samples/sample-terraform/tf_apply.yml | 2 +- samples/sample-terraform/tf_plan.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml index f46b49a97..9e93bba0b 100644 --- a/samples/sample-terraform/tf_apply.yml +++ b/samples/sample-terraform/tf_apply.yml @@ -15,7 +15,7 @@ phases: build: commands: - python adf-build/helpers/terraform/get_accounts.py - - bash adf-build/helpers/terraform/adf_terraform.sh $TF_STAGE + - bash adf-build/helpers/terraform/adf_terraform.sh artifacts: files: "**/*" diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml index 83b5aa8c6..fc0e6e452 100644 --- a/samples/sample-terraform/tf_plan.yml +++ b/samples/sample-terraform/tf_plan.yml @@ -15,4 +15,4 @@ phases: build: commands: - python adf-build/helpers/terraform/get_accounts.py - - bash adf-build/helpers/terraform/adf_terraform.sh $TF_STAGE + - bash adf-build/helpers/terraform/adf_terraform.sh From 0b720d6866a416db85fa1e67c8f212169ee81354 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 9 Nov 2021 23:37:05 +0100 Subject: [PATCH 037/179] removed AWS_DEFAULT_REGION from tf_apply and tf_plan --- samples/sample-terraform/tf_apply.yml | 1 - samples/sample-terraform/tf_plan.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml index 9e93bba0b..cccaca324 100644 --- a/samples/sample-terraform/tf_apply.yml +++ b/samples/sample-terraform/tf_apply.yml @@ -2,7 +2,6 @@ version: 0.2 env: variables: - AWS_DEFAULT_REGION: eu-west-1 # ADF main region TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources TF_IN_AUTOMATION: true TF_CLI_ARGS: "-no-color" diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml index fc0e6e452..a4acfecf2 100644 --- a/samples/sample-terraform/tf_plan.yml +++ b/samples/sample-terraform/tf_plan.yml @@ -2,7 +2,6 @@ version: 0.2 env: variables: - AWS_DEFAULT_REGION: eu-west-1 # ADF main region TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources TF_IN_AUTOMATION: true TF_STAGE: "plan" From 30f116f86099ae6c953576e17f54f7b4e46abe17 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 9 Nov 2021 23:43:51 +0100 Subject: [PATCH 038/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 798d5fd11..6f6e87c38 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -8,7 +8,7 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name /cross_region/s3_regional_bucket/"$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) - mkdir -p "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" cd "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" || exit cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" # if account related variables exist copy the folder in the work directory From 0f630b43c0c6de85b13f40b6c4c948b76fe7c883 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 9 Nov 2021 23:46:43 +0100 Subject: [PATCH 039/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index d1899fb9b..5ec4ba78b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -6,7 +6,7 @@ sts = boto3.client('sts') -master_acc_id = os.environ["MASTER_ACCOUNT_ID"] +management_acc_id = os.environ["MANAGEMENT_ACCOUNT_ID"] if("TARGET_OUS" in os.environ): ou_path = os.environ["TARGET_OUS"] From 99e5e9fbf506afeaa7710bef19b5b8f58a07fa35 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 9 Nov 2021 23:48:18 +0100 Subject: [PATCH 040/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 5ec4ba78b..a11107c3b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -27,7 +27,6 @@ def get_accounts(): print("Master Account ID: " + master_acc_id) # Assume a role into the Org Master role to get account ID's and emails organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') - # organizations = get_boto3_client('organizations', 'arn:aws:sts::' + master_acc_id + ':role/Admin', 'getaccountID') for account in paginator(organizations.list_accounts): if account['Status'] == 'ACTIVE': account_details.append({ From 1625ae040aeb01b61679250189a16d91e3fa780e Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 9 Nov 2021 23:48:51 +0100 Subject: [PATCH 041/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon --- .../adf-build/shared/helpers/terraform/get_accounts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index a11107c3b..e4bc458ca 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -42,9 +42,9 @@ def get_accounts_from_ous(): # Read organization root id root_ids = [] for id in paginator(organizations.list_roots): - root_ids.append({ - 'AccountId': id['Id'] - }) + root_ids.append({ + 'AccountId': id['Id'] + }) root_id = root_ids[0]['AccountId'] print("Target OUs") for path in ou_path.split(','): From e78223bd1459fb06d59e676e375abcfbecea2969 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 9 Nov 2021 23:49:46 +0100 Subject: [PATCH 042/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index e4bc458ca..641d8155b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -92,6 +92,7 @@ def get_boto3_client(service, role, session_name): ) return session.client(service) + def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> list: account_list = [] # Get OUs From 85ce85c7b7f98a07c2e26efcfcd0d6470b317ccf Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 09:52:35 +0100 Subject: [PATCH 043/179] revert to original PULL_REQUEST_TEMPLATE.md --- .github/PULL_REQUEST_TEMPLATE.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index db6ceeddd..6bdaa999f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,5 +1,6 @@ -_Issue #, if available:_ +*Issue #, if available:* + +*Description of changes:* -_Description of changes:_ By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice. From 793ab2c59ef3a1e2339005728d83a06fa96cf88c Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 09:56:07 +0100 Subject: [PATCH 044/179] clean .gitignore --- .gitignore | 130 +++++++++++------------------------------------------ 1 file changed, 27 insertions(+), 103 deletions(-) diff --git a/.gitignore b/.gitignore index 4e2ecf934..9d8273558 100644 --- a/.gitignore +++ b/.gitignore @@ -1,48 +1,38 @@ - -# Created by https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode -# Edit at https://www.toptal.com/developers/gitignore?templates=macos,terraform,python,visualstudiocode - -### macOS ### -# General +.vscode +.idea +.pyc +.zip .DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent +cleanup.py +config +config.sec +config.bak +policy.json +pipeline.yml +packaged.yaml +template-sam.yml +template-deploy.yml +template-sam-out.yml +master-deploy.yml +.pytest_cache +shared_layer.zip +.aws-sam +pipeline.json +template-sam.yml +deploy.sh -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### Python ### # Byte-compiled / optimized / DLL files -__pycache__/ +__pycache__ *.py[cod] *$py.class - +outfile # C extensions *.so # Distribution / packaging +package.yaml +aws_deployment_framework.egg-info .Python -build/ develop-eggs/ dist/ downloads/ @@ -54,7 +44,6 @@ parts/ sdist/ var/ wheels/ -share/python-wheels/ *.egg-info/ .installed.cfg *.egg @@ -80,10 +69,8 @@ htmlcov/ nosetests.xml coverage.xml *.cover -*.py,cover .hypothesis/ .pytest_cache/ -cover/ # Translations *.mo @@ -93,7 +80,6 @@ cover/ *.log local_settings.py db.sqlite3 -db.sqlite3-journal # Flask stuff: instance/ @@ -106,7 +92,6 @@ instance/ docs/_build/ # PyBuilder -.pybuilder/ target/ # Jupyter Notebook @@ -117,23 +102,10 @@ profile_default/ ipython_config.py # pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock +.python-version -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff +# celery beat schedule file celerybeat-schedule -celerybeat.pid # SageMath parsed files *.sage.py @@ -165,12 +137,6 @@ dmypy.json # Pyre type checker .pyre/ -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - ### Terraform ### # Local .terraform directories **/.terraform/* @@ -179,48 +145,6 @@ cython_debug/ *.tfstate *.tfstate.* -# Crash log files -crash.log - -# Exclude all .tfvars files, which are likely to contain sentitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject -# to change depending on the environment. -# -#*.tfvars - -# Ignore override files as they are usually used to override resources locally and so -# are not checked in -override.tf -override.tf.json -*_override.tf -*_override.tf.json - -# Include override files you do wish to add to version control using negated pattern -# !example_override.tf - -# Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan -# example: *tfplan* - # Ignore CLI configuration files .terraformrc terraform.rc - -### VisualStudioCode ### -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - -### VisualStudioCode Patch ### -# Ignore all local history of files -.history -.ionide - -# End of https://www.toptal.com/developers/gitignore/api/macos,terraform,python,visualstudiocode - From 3fa1b92ca5daea2c5acb55811125ac7dff5cece9 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 12:12:17 +0100 Subject: [PATCH 045/179] add logging module --- .../shared/helpers/terraform/get_accounts.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 641d8155b..4485f2d64 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -1,9 +1,15 @@ import boto3 import json import os +import logging from paginator import paginator +# Configure logging +logging.basicConfig(level=logging.INFO) +LOGGER = logging.getLogger(__name__) +LOGGER.setLevel(logging.INFO) + sts = boto3.client('sts') management_acc_id = os.environ["MANAGEMENT_ACCOUNT_ID"] @@ -12,7 +18,7 @@ def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{management_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) @@ -23,10 +29,12 @@ def list_organizational_units_for_parent(parent_ou): def get_accounts(): # Return an array of objects like this: [{'AccountId':'xxx','Email':''}] account_details = [] # [{'AccountId':'123','Email':''},{'AccountId':'456','Email':''}] - - print("Master Account ID: " + master_acc_id) - # Assume a role into the Org Master role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + LOGGER.info( + "Management Account ID: %s", + management_acc_id + ) + # Assume a role into the management accounts role to get account ID's and emails + organizations = get_boto3_client('organizations', f'arn:aws:sts::{management_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') for account in paginator(organizations.list_accounts): if account['Status'] == 'ACTIVE': account_details.append({ @@ -38,7 +46,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:aws:sts::{master_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{management_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') # Read organization root id root_ids = [] for id in paginator(organizations.list_roots): @@ -46,9 +54,9 @@ def get_accounts_from_ous(): 'AccountId': id['Id'] }) root_id = root_ids[0]['AccountId'] - print("Target OUs") + LOGGER.info("Target OUs") for path in ou_path.split(','): - print(path) + LOGGER.info(path) # Set initial OU to start looking for given ou_path if parent_ou_id is None: parent_ou_id = root_id @@ -71,11 +79,6 @@ def get_accounts_from_ous(): account_list.extend(get_account_recursive(organizations, parent_ou_id, '/')) parent_ou_id=None - print("Account list: ", end = '') - for i in account_list: - print(i['AccountId'] + " ", end = '') - print() - print("Number of target accounts: " + str(len(account_list))) return account_list From 99fa20dd77d0cd0424ea9bca2e45eea791c59299 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 13:50:24 +0100 Subject: [PATCH 046/179] added main --- .../shared/helpers/terraform/get_accounts.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 4485f2d64..68aea95fe 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -4,17 +4,26 @@ import logging from paginator import paginator - # Configure logging logging.basicConfig(level=logging.INFO) LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.INFO) -sts = boto3.client('sts') - management_acc_id = os.environ["MANAGEMENT_ACCOUNT_ID"] if("TARGET_OUS" in os.environ): ou_path = os.environ["TARGET_OUS"] +sts = boto3.client('sts') + +def main(): + + accounts = get_accounts() + with open('accounts.json', 'w') as outfile: + json.dump(accounts, outfile) + + if("TARGET_OUS" in os.environ): + accounts_from_ous = get_accounts_from_ous() + with open('accounts_from_ous.json', 'w') as outfile: + json.dump(accounts_from_ous, outfile) def list_organizational_units_for_parent(parent_ou): @@ -54,9 +63,7 @@ def get_accounts_from_ous(): 'AccountId': id['Id'] }) root_id = root_ids[0]['AccountId'] - LOGGER.info("Target OUs") for path in ou_path.split(','): - LOGGER.info(path) # Set initial OU to start looking for given ou_path if parent_ou_id is None: parent_ou_id = root_id @@ -120,12 +127,5 @@ def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> li }) return account_list - -accounts = get_accounts() -with open('accounts.json', 'w') as outfile: - json.dump(accounts, outfile) - -if("TARGET_OUS" in os.environ): - accounts_from_ous = get_accounts_from_ous() - with open('accounts_from_ous.json', 'w') as outfile: - json.dump(accounts_from_ous, outfile) \ No newline at end of file +if __name__ == "__main__": + main() \ No newline at end of file From bb58170337f5af47706a2b973f6d4c69977acca2 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 14:15:41 +0100 Subject: [PATCH 047/179] undo linter changes --- .../adf-bootstrap/deployment/global.yml | 228 ++++++------------ 1 file changed, 77 insertions(+), 151 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 5b7942fe7..cf627354d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -32,9 +32,9 @@ Parameters: Type: String Default: "BUILD_GENERAL1_LARGE" # For threading with large amounts of pipelines this is the most effective default AllowedValues: - - "BUILD_GENERAL1_SMALL" #3 GB memory, 2 vCPU - - "BUILD_GENERAL1_MEDIUM" #7 GB memory, 4 vCPU - - "BUILD_GENERAL1_LARGE" #15 GB memory, 8 vCPU + - "BUILD_GENERAL1_SMALL" # 3 GB memory, 2 vCPU + - "BUILD_GENERAL1_MEDIUM" # 7 GB memory, 4 vCPU + - "BUILD_GENERAL1_LARGE" # 15 GB memory, 8 vCPU NotificationEndpoint: Type: "AWS::SSM::Parameter::Value" Default: notification_endpoint @@ -78,7 +78,7 @@ Resources: - Sid: Allows admin of the key Effect: Allow Principal: - AWS: !Sub arn:aws:iam::${AWS::AccountId}:root + AWS: !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:root Action: - "kms:CancelKeyDeletion" - "kms:Create*" @@ -207,15 +207,6 @@ Resources: PolicyDocument: Version: "2012-10-17" Statement: - - Effect: Allow - Sid: "DynamoDB" - Action: - - dynamodb:PutItem - - dynamodb:GetItem - - dynamodb:DeleteItem - - dynamodb:DescribeTable - Resource: - - !Sub "arn:aws:dynamodb:*:${AWS::AccountId}:table/adf-tflocktable*" - Effect: Allow Sid: "S3" Action: @@ -224,10 +215,10 @@ Resources: - s3:List* - s3:PutObject Resource: - - !Sub arn:aws:s3:::${PipelineBucket} - - !Sub arn:aws:s3:::${PipelineBucket}/* - - !Sub arn:aws:s3:::${SharedModulesBucket} - - !Sub arn:aws:s3:::${SharedModulesBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket} + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${SharedModulesBucket} + - !Sub arn:${AWS::Partition}:s3:::${SharedModulesBucket}/* - Effect: Allow Sid: "KMS" Action: @@ -253,27 +244,27 @@ Resources: aws:PrincipalOrgID: !Ref OrganizationId Action: - "secretsmanager:Get*" - Resource: !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:/adf/*" # Only allow CodeBuild access to secrets that start with /adf/* + Resource: !Sub "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:/adf/*" # Only allow CodeBuild access to secrets that start with /adf/* - Effect: Allow Action: - "ssm:GetParameter" - "ssm:GetParameters" Resource: - - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/* - - !Sub arn:aws:ssm:${AWS::Region}::parameter/aws/service/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}:${AWS::AccountId}:parameter/* + - !Sub arn:${AWS::Partition}:ssm:${AWS::Region}::parameter/aws/service/* - Effect: Deny Action: - "sts:AssumeRole" Resource: - !GetAtt CloudFormationDeploymentRole.Arn - - !Sub arn:aws:iam::${MasterAccountId}:role/${CrossAccountAccessRole} + - !Sub arn:${AWS::Partition}:iam::${MasterAccountId}:role/${CrossAccountAccessRole} - Effect: Allow Action: - "logs:CreateLogGroup" - "logs:CreateLogStream" - "logs:PutLogEvents" Resource: - - !Sub arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/* + - !Sub arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/* - Effect: Allow Action: # If you plan on building docker images in CodeBuild you need these - "ecr:GetAuthorizationToken" @@ -314,10 +305,10 @@ Resources: - s3:List* - s3:PutObject Resource: - - !Sub arn:aws:s3:::${PipelineBucket} - - !Sub arn:aws:s3:::${PipelineBucket}/* - - !Sub arn:aws:s3:::${SharedModulesBucket} - - !Sub arn:aws:s3:::${SharedModulesBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket} + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${SharedModulesBucket} + - !Sub arn:${AWS::Partition}:s3:::${SharedModulesBucket}/* - Effect: Allow Sid: "KMS" Action: @@ -331,17 +322,17 @@ Resources: Action: - "sts:AssumeRole" Resource: - - !Sub "arn:aws:iam::${MasterAccountId}:role/${CrossAccountAccessRole}-readonly" - - arn:aws:iam::*:role/adf-automation-role + - !Sub "arn:${AWS::Partition}:iam::${MasterAccountId}:role/${CrossAccountAccessRole}-readonly" + - !Sub "arn:${AWS::Partition}:iam::*:role/adf-automation-role" - Effect: Allow Action: - "secretsmanager:Get*" - Resource: !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:/adf/*" # Only allow CodeBuild access to secrets that start with /adf/* + Resource: !Sub "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:/adf/*" # Only allow CodeBuild access to secrets that start with /adf/* - Effect: Allow Action: - "events:PutPermission" Resource: - - !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:event-bus/default + - !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:event-bus/default - Effect: Allow Action: - "events:PutRule" @@ -351,7 +342,7 @@ Resources: - "events:DeleteRule" - "events:DescribeRule" Resource: - - !Sub arn:aws:events:${AWS::Region}:${AWS::AccountId}:rule/${PipelinePrefix}* + - !Sub arn:${AWS::Partition}:events:${AWS::Region}:${AWS::AccountId}:rule/${PipelinePrefix}* - Effect: Allow Action: - "cloudformation:CancelUpdateStack" @@ -369,7 +360,7 @@ Resources: - "cloudformation:UpdateStack" - "cloudformation:UpdateTerminationProtection" Resource: - - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${PipelinePrefix}*/*" + - !Sub "arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/${PipelinePrefix}*/*" - Effect: Allow Action: - "cloudformation:ValidateTemplate" @@ -403,14 +394,14 @@ Resources: - "sns:GetTopicAttributes" - "sns:TagResource" Resource: - - !Sub arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelinePrefix}* + - !Sub arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:${PipelinePrefix}* - Effect: Allow Action: - "codebuild:CreateProject" - "codebuild:DeleteProject" - "codebuild:UpdateProject" Resource: - - !Sub arn:aws:codebuild:${AWS::Region}:${AWS::AccountId}:project/adf-* + - !Sub arn:${AWS::Partition}:codebuild:${AWS::Region}:${AWS::AccountId}:project/adf-* - Effect: Allow Action: - "iam:CreateRole" @@ -422,7 +413,7 @@ Resources: - "iam:PassRole" - "iam:PutRolePolicy" Resource: - - !Sub arn:aws:iam::${AWS::AccountId}:role/* + - !Sub arn:${AWS::Partition}:iam::${AWS::AccountId}:role/* Condition: StringEquals: aws:PrincipalOrgID: !Ref OrganizationId @@ -430,7 +421,7 @@ Resources: Action: - "iam:PassRole" # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_passrole.html Resource: - - arn:aws:iam::*:role/* # This role can pass to any other role in the organization. + - !Sub arn:${AWS::Partition}:iam::*:role/* # This role can pass to any other role in the organization. Condition: StringEquals: aws:PrincipalOrgID: !Ref OrganizationId @@ -448,8 +439,8 @@ Resources: - "codepipeline:TagResource" - "codepipeline:UpdatePipeline" Resource: - - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:webhook:adf-webhook-* - - !Sub arn:aws:codepipeline:${AWS::Region}:${AWS::AccountId}:${PipelinePrefix}* + - !Sub arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:webhook:adf-webhook-* + - !Sub arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${PipelinePrefix}* - Effect: Allow Action: - "codestar-connections:GetConnection" @@ -575,7 +566,7 @@ Resources: - s3:Get* - s3:List* Resource: - - !Sub "arn:aws:s3:::${PipelineBucket}/adf-build/templates/*" + - !Sub "arn:${AWS::Partition}:s3:::${PipelineBucket}/adf-build/templates/*" - Effect: Allow Sid: "CloudFormation" Action: @@ -593,8 +584,8 @@ Resources: - "cloudformation:UpdateStack" - "cloudformation:UpdateTerminationProtection" Resource: - - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/adf-codecommit-*/*" - - !Sub "arn:aws:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/adf-event-rule-${AWS::AccountId}-*/*" + - !Sub "arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/adf-codecommit-*/*" + - !Sub "arn:${AWS::Partition}:cloudformation:${AWS::Region}:${AWS::AccountId}:stack/adf-event-rule-${AWS::AccountId}-*/*" - Effect: Allow Sid: "CodeCommit" Action: @@ -622,9 +613,9 @@ Resources: - "ssm:GetParameters" - "ssm:GetParameter" Resource: - - "arn:aws:ssm:*:*:parameter/bucket_name" - - "arn:aws:ssm:*:*:parameter/deployment_account_id" - - "arn:aws:ssm:*:*:parameter/kms_arn" + - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/bucket_name" + - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/deployment_account_id" + - !Sub "arn:${AWS::Partition}:ssm:*:*:parameter/kms_arn" Roles: - !Ref AdfAutomationRole CodeCommitRepository: @@ -689,7 +680,7 @@ Resources: - cdk --version - chmod 755 adf-build/cdk/execute_pipeline_stacks.py adf-build/cdk/generate_pipeline_inputs.py adf-build/cdk/generate_pipeline_stacks.py adf-build/cdk/clean_pipelines.py - python adf-build/cdk/generate_pipeline_inputs.py - - cdk synth --app adf-build/cdk/generate_pipeline_stacks.py 1> /dev/null + - cdk synth --no-version-reporting --app adf-build/cdk/generate_pipeline_stacks.py 1> /dev/null - python adf-build/cdk/execute_pipeline_stacks.py post_build: commands: @@ -878,8 +869,8 @@ Resources: Action: - sts:AssumeRole Resource: - - arn:aws:iam::*:role/adf-cloudformation-role - - arn:aws:iam::*:role/adf-codecommit-role + - !Sub arn:${AWS::Partition}:iam::*:role/adf-cloudformation-role + - !Sub arn:${AWS::Partition}:iam::*:role/adf-codecommit-role Condition: StringEquals: aws:PrincipalOrgID: !Ref OrganizationId @@ -893,8 +884,8 @@ Resources: - s3:ReplicateObject - s3:ReplicateTags Resource: - - !Sub arn:aws:s3:::${PipelineBucket} - - !Sub arn:aws:s3:::${PipelineBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket} + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket}/* - Effect: Allow Sid: "CodeCommit" Action: @@ -945,8 +936,8 @@ Resources: StringEquals: aws:PrincipalOrgID: !Ref OrganizationId Resource: - - !Sub arn:aws:s3:::${PipelineBucket} - - !Sub arn:aws:s3:::${PipelineBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket} + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket}/* Principal: AWS: "*" SendSlackNotification: @@ -1055,18 +1046,18 @@ Resources: - Effect: Allow Action: - "secretsmanager:Get*" - Resource: !Sub "arn:aws:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:/adf/*" # Only allow Lambda access to get secrets that start with /adf/* + Resource: !Sub "arn:${AWS::Partition}:secretsmanager:${AWS::Region}:${AWS::AccountId}:secret:/adf/*" # Only allow Lambda access to get secrets that start with /adf/* - Effect: "Allow" Action: - "s3:Get*" - "s3:Put*" - "s3:List*" Resource: - - !Sub arn:aws:s3:::${PipelineBucket} - - !Sub arn:aws:s3:::${PipelineBucket}/* + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket} + - !Sub arn:${AWS::Partition}:s3:::${PipelineBucket}/* - Effect: "Allow" Action: "logs:*" - Resource: "arn:aws:logs:*:*:*" + Resource: !Sub "arn:${AWS::Partition}:logs:*:*:*" StatesExecutionRole: Type: "AWS::IAM::Role" Properties: @@ -1127,11 +1118,28 @@ Resources: ], "Default": "EnableCrossAccountAccess" }, - "EnableCrossAccountAccess": { - "Type": "Task", - "Resource": "${EnableCrossAccountAccess.Arn}", + "EnableCrossAccountAccessMap": { + "Type": "Map", "Next": "UpdateDeploymentPipelines", - "TimeoutSeconds": 900 + "Iterator": { + "StartAt": "EnableCrossAccountAccess", + "States": { + "EnableCrossAccountAccess": { + "Type": "Task", + "Resource": "${EnableCrossAccountAccess.Arn}", + "TimeoutSeconds": 900, + "End": true + } + } + }, + "ItemsPath": "$.account_ids", + "Parameters": { + "deployment_account_region.$": "$.deployment_account_region", + "deployment_account_id.$": "$.deployment_account_id", + "account_id.$": "$$.Map.Item.Value", + "regions.$": "$.regions" + }, + "ResultPath": null }, "UpdateDeploymentPipelines": { "Type": "Task", @@ -1156,9 +1164,9 @@ Resources: }, "NotifySuccess": { "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", + "Resource": "arn:${AWS::Partition}:states:::sns:publish", "Parameters": { - "TopicArn": "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", + "TopicArn": "arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", "Message.$": "$.message", "Subject": "Success - AWS Deployment Framework Bootstrap" }, @@ -1166,9 +1174,9 @@ Resources: }, "NotifyFailure": { "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", + "Resource": "arn:${AWS::Partition}:states:::sns:publish", "Parameters": { - "TopicArn": "arn:aws:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", + "TopicArn": "arn:${AWS::Partition}:sns:${AWS::Region}:${AWS::AccountId}:${PipelineSNSTopic.TopicName}", "Message.$": "$.error", "Subject": "Failure - AWS Deployment Framework Bootstrap" }, @@ -1232,18 +1240,8 @@ Resources: Statement: - Effect: Allow Action: codepipeline:StartPipelineExecution - Resource: - !Join [ - "", - [ - "arn:aws:codepipeline:", - !Ref "AWS::Region", - ":", - !Ref "AWS::AccountId", - ":", - !Ref CodePipeline, - ], - ] + Resource: !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}" + PipelineCloudWatchEventRule: Type: AWS::Events::Rule Properties: @@ -1252,18 +1250,7 @@ Resources: - aws.codecommit detail-type: - "CodeCommit Repository State Change" - resources: - - !Join [ - "", - [ - "arn:aws:codecommit:", - !Ref "AWS::Region", - ":", - !Ref "AWS::AccountId", - ":", - !GetAtt CodeCommitRepository.Name, - ], - ] + resources: !Sub "arn:${AWS::Partition}:codecommit:${AWS::Region}:${AWS::AccountId}:${CodeCommitRepository.Name}" detail: event: - referenceCreated @@ -1273,71 +1260,10 @@ Resources: referenceName: - master Targets: - - Arn: - !Join [ - "", - [ - "arn:aws:codepipeline:", - !Ref "AWS::Region", - ":", - !Ref "AWS::AccountId", - ":", - !Ref CodePipeline, - ], - ] + - Arn: !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}" RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline - TerraformLockTable: - Type: "AWS::DynamoDB::Table" - Properties: - AttributeDefinitions: - - AttributeName: LockID - AttributeType: S - KeySchema: - - AttributeName: LockID - KeyType: HASH - ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 - TableName: adf-tflocktable - ADFTerraformRole: - # ADF role to run terraform pipelines - Type: AWS::IAM::Role - Properties: - RoleName: "adf-terraform-role" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Sid: "AssumeRole" - Principal: - AWS: - # This would allow all codebuild projects to be able to assume this role - - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role - # - !Sub arn:aws:iam::${DeploymentAccountId}:role/my-custom-codebuild-role - # The above role would be created on the deployment account for the purpose deploying this custom resource via codebuild - Action: - - sts:AssumeRole - Path: / - ADFTerraformPolicy: - Type: AWS::IAM::Policy - Properties: - PolicyName: "adf-terraform-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "ec2:*" - - "s3:*" - - "ssm:*" - - "sso:*" - - "identitystore:*" - - "organizations:*" - Resource: - - "*" - Roles: - - !Ref ADFTerraformRole + Outputs: ADFVersionNumber: Value: !Ref ADFVersion @@ -1390,4 +1316,4 @@ Outputs: Description: "The Kms Key Arn" Value: !GetAtt KMSKey.Arn Export: - Name: "KmsKeyArn" + Name: "KmsKeyArn" \ No newline at end of file From 70196b826511bd214964fc3d773874e5f89e9e01 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 14:20:54 +0100 Subject: [PATCH 048/179] removed artifact from tf_apply --- samples/sample-terraform/tf_apply.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml index cccaca324..fe89d4b70 100644 --- a/samples/sample-terraform/tf_apply.yml +++ b/samples/sample-terraform/tf_apply.yml @@ -16,5 +16,3 @@ phases: - python adf-build/helpers/terraform/get_accounts.py - bash adf-build/helpers/terraform/adf_terraform.sh -artifacts: - files: "**/*" From 90431352fb728dcadde6a6a33b7098ed6d5fb7fb Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 14:38:44 +0100 Subject: [PATCH 049/179] added ondemand capacity to DynamoDB table --- .../adf-bootstrap/deployment/global.yml | 58 +++++++++++++++++++ .../adf-bootstrap/deployment/regional.yml | 4 +- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index cf627354d..57562ffad 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -207,6 +207,15 @@ Resources: PolicyDocument: Version: "2012-10-17" Statement: + - Effect: Allow + Sid: "DynamoDB" + Action: + - dynamodb:PutItem + - dynamodb:GetItem + - dynamodb:DeleteItem + - dynamodb:DescribeTable + Resource: + - !Sub "arn:aws:dynamodb:*:${AWS::AccountId}:table/adf-tflocktable*" - Effect: Allow Sid: "S3" Action: @@ -1263,6 +1272,55 @@ Resources: - Arn: !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}" RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline + TerraformLockTable: + Type: "AWS::DynamoDB::Table" + Properties: + AttributeDefinitions: + - AttributeName: LockID + AttributeType: S + KeySchema: + - AttributeName: LockID + KeyType: HASH + BillingMode: PAY_PER_REQUEST + TableName: adf-tflocktable + ADFTerraformRole: + # ADF role to run terraform pipelines + Type: AWS::IAM::Role + Properties: + RoleName: "adf-terraform-role" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Sid: "AssumeRole" + Principal: + AWS: + # This would allow all codebuild projects to be able to assume this role + - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role + # - !Sub arn:aws:iam::${DeploymentAccountId}:role/my-custom-codebuild-role + # The above role would be created on the deployment account for the purpose deploying this custom resource via codebuild + Action: + - sts:AssumeRole + Path: / + ADFTerraformPolicy: + Type: AWS::IAM::Policy + Properties: + PolicyName: "adf-terraform-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "ec2:*" + - "s3:*" + - "ssm:*" + - "sso:*" + - "identitystore:*" + - "organizations:*" + Resource: + - "*" + Roles: + - !Ref ADFTerraformRole Outputs: ADFVersionNumber: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml index e998655f7..4fbd3894d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml @@ -98,9 +98,7 @@ Resources: KeySchema: - AttributeName: LockID KeyType: HASH - ProvisionedThroughput: - ReadCapacityUnits: 5 - WriteCapacityUnits: 5 + BillingMode: PAY_PER_REQUEST TableName: adf-tflocktable Outputs: DeploymentFrameworkRegionalS3Bucket: From 50e17b311111779d0845844d547bccb8eb5cd911 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 14:42:52 +0100 Subject: [PATCH 050/179] removed terraform role. This could be added in global-iam by the user --- .../adf-bootstrap/deployment/global.yml | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 57562ffad..aa054d47f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -1283,44 +1283,6 @@ Resources: KeyType: HASH BillingMode: PAY_PER_REQUEST TableName: adf-tflocktable - ADFTerraformRole: - # ADF role to run terraform pipelines - Type: AWS::IAM::Role - Properties: - RoleName: "adf-terraform-role" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Sid: "AssumeRole" - Principal: - AWS: - # This would allow all codebuild projects to be able to assume this role - - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role - # - !Sub arn:aws:iam::${DeploymentAccountId}:role/my-custom-codebuild-role - # The above role would be created on the deployment account for the purpose deploying this custom resource via codebuild - Action: - - sts:AssumeRole - Path: / - ADFTerraformPolicy: - Type: AWS::IAM::Policy - Properties: - PolicyName: "adf-terraform-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "ec2:*" - - "s3:*" - - "ssm:*" - - "sso:*" - - "identitystore:*" - - "organizations:*" - Resource: - - "*" - Roles: - - !Ref ADFTerraformRole Outputs: ADFVersionNumber: From a81899eea4e38863557d04ae5e860553f65130a9 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 14:57:24 +0100 Subject: [PATCH 051/179] added an example of terraform role. It must be uncommented by the user to enable tf extension --- .../deployment/example-global-iam.yml | 38 ++++++++++++++++- .../adf-bootstrap/example-global-iam.yml | 42 +++++++++++++++++++ 2 files changed, 79 insertions(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index 9d6ad6c7b..a635cb2b3 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -23,4 +23,40 @@ Resources: Resource: - "*" Roles: - - adf-cloudformation-deployment-role \ No newline at end of file + - adf-cloudformation-deployment-role + + # ADFTerraformRole: + # # This is the role that will be used to deploy Terraform resources from within + # # the target account. You should scope this policy depending on what you would + # # like to deploy within certain Organizational Units. + # # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. + # # Uncomment this line if you want to enable the terraform extentions + # Type: AWS::IAM::Role + # Properties: + # RoleName: "adf-terraform-role" + # AssumeRolePolicyDocument: + # Version: "2012-10-17" + # Statement: + # - Effect: Allow + # Sid: "AssumeRole" + # Principal: + # AWS: + # - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role + # Action: + # - sts:AssumeRole + # Path: / + # ADFTerraformPolicy: + # Type: AWS::IAM::Policy + # Properties: + # PolicyName: "adf-terraform-policy" + # PolicyDocument: + # Version: "2012-10-17" + # Statement: + # - Effect: Allow + # Action: + # - "ec2:*" + # - "s3:*" + # Resource: + # - "*" + # Roles: + # - !Ref ADFTerraformRole \ No newline at end of file diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml index c3649bcfb..a78f4a49f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml @@ -3,6 +3,13 @@ AWSTemplateFormatVersion: "2010-09-09" Description: ADF CloudFormation Template (Global) for IAM in the Deployment Account + +Parameters: + DeploymentAccountId: + Type: "AWS::SSM::Parameter::Value" + Description: Deployment Account ID + Default: deployment_account_id + Resources: CloudFormationDeploymentPolicy: # This is the policy that will be used to deploy CloudFormation resources from @@ -34,6 +41,41 @@ Resources: - "*" Roles: - adf-cloudformation-deployment-role # This role is created in the global.yml and is the default cloudformation deployment role for ADF. + # ADFTerraformRole: + # # This is the role that will be used to deploy Terraform resources from within + # # the target account. You should scope this policy depending on what you would + # # like to deploy within certain Organizational Units. + # # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. + # # Uncomment this line if you want to enable the terraform extentions + # Type: AWS::IAM::Role + # Properties: + # RoleName: "adf-terraform-role" + # AssumeRolePolicyDocument: + # Version: "2012-10-17" + # Statement: + # - Effect: Allow + # Sid: "AssumeRole" + # Principal: + # AWS: + # - !Sub arn:aws:iam::${DeploymentAccountId}:role/adf-codebuild-role + # Action: + # - sts:AssumeRole + # Path: / + # ADFTerraformPolicy: + # Type: AWS::IAM::Policy + # Properties: + # PolicyName: "adf-terraform-policy" + # PolicyDocument: + # Version: "2012-10-17" + # Statement: + # - Effect: Allow + # Action: + # - "ec2:*" + # - "s3:*" + # Resource: + # - "*" + # Roles: + # - !Ref ADFTerraformRole # MyExampleCustomRole: # # Am example custom role that you would need to create in order to deploy custom resources in other AWS Accounts within the organization. From 5138c978a7596b50fad800c18def0be1173cb2e7 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 15:12:28 +0100 Subject: [PATCH 052/179] added tf pre-requisites to user-guide --- docs/user-guide.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/user-guide.md b/docs/user-guide.md index a45067b14..9aa22ef22 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -673,6 +673,13 @@ By passing in the Repository name _(repository)_ we are overriding the **name** ### Terraform pipeline +**Pre-requisites** +To enable ADF Terraform extension the following steps are required: +- Rename file example-global-iam.yml to global-iam.yml in the following path aws-deployment-framework-bootstrap/adf-bootstrap/ and comment out the Cloudformation resources ADFTerraformRole and ADFTerraformPolicy +- Rename file example-global-iam.yml to global-iam.yml in the following path aws-deployment-framework-bootstrap/adf-bootstrap/deployment and comment out the Cloudformation resources ADFTerraformRole and ADFTerraformPolicy + +NOTE: ADFTerraformPolicy IAM policy is a sample. This policies should NOT be used for purposes other than testing. You should scope this policy depending on what you would like to deploy within certain Organizational Units. + **Overview** ADF support the deployment of terraform code to multiple accounts and regions through terraform pipelines. From 6568537423f213257f51109e270844c179230e65 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 15:20:15 +0100 Subject: [PATCH 053/179] global vars to upper case --- .../shared/helpers/terraform/get_accounts.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 68aea95fe..c4e799b22 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -9,9 +9,9 @@ LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.INFO) -management_acc_id = os.environ["MANAGEMENT_ACCOUNT_ID"] +MANAGEMENT_ACC_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] if("TARGET_OUS" in os.environ): - ou_path = os.environ["TARGET_OUS"] + OU_PATH = os.environ["TARGET_OUS"] sts = boto3.client('sts') def main(): @@ -27,7 +27,7 @@ def main(): def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:aws:sts::{management_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) @@ -40,10 +40,10 @@ def get_accounts(): account_details = [] # [{'AccountId':'123','Email':''},{'AccountId':'456','Email':''}] LOGGER.info( "Management Account ID: %s", - management_acc_id + MANAGEMENT_ACC_ID ) # Assume a role into the management accounts role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:aws:sts::{management_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') for account in paginator(organizations.list_accounts): if account['Status'] == 'ACTIVE': account_details.append({ @@ -55,7 +55,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:aws:sts::{management_acc_id}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') # Read organization root id root_ids = [] for id in paginator(organizations.list_roots): @@ -63,12 +63,12 @@ def get_accounts_from_ous(): 'AccountId': id['Id'] }) root_id = root_ids[0]['AccountId'] - for path in ou_path.split(','): - # Set initial OU to start looking for given ou_path + for path in OU_PATH.split(','): + # Set initial OU to start looking for given OU_PATH if parent_ou_id is None: parent_ou_id = root_id - # Parse ou_path and find the ID + # Parse OU_PATH and find the ID ou_hierarchy = path.strip('/').split('/') hierarchy_index = 0 if(path.strip() == '/'): From 07d42a1ea4d3575df2453c2d264697c4ac18144c Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 22:33:14 +0100 Subject: [PATCH 054/179] added tfrun function to avoid code repetition and add functionality to copy tf plan to S3 bucket in the deployment account --- .../shared/helpers/terraform/adf_terraform.sh | 138 +++++------------- 1 file changed, 33 insertions(+), 105 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 6f6e87c38..bf3e6547d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -8,7 +8,7 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name /cross_region/s3_regional_bucket/"$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) - mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + mkdir -p "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" cd "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" || exit cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" # if account related variables exist copy the folder in the work directory @@ -21,19 +21,45 @@ tfinit(){ -backend-config "region=$AWS_REGION" \ -backend-config "key=$ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" \ -backend-config "dynamodb_table=adf-tflocktable" - + echo "Bucket: $S3_BUCKET_REGION_NAME" echo "Region: $AWS_REGION" echo "Key: $ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" echo "DynamoDB table: adf-tflocktable" } tfplan(){ + DATE=$(date +%Y-%m-%d) + TS=$(date +%Y%m%d%H%M%S) bash "$CURRENT"/adf-build/helpers/sts.sh "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" - terraform plan -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" + terraform plan -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" 2>&1 | tee -a "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-"$TS".log + # Save terraform plan to S3 bucket + aws s3 cp "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-"$TS".log s3://"$S3_BUCKET_REGION_NAME"/"$ADF_PROJECT_NAME"/tf-plan/"$DATE"/"$TF_VAR_TARGET_ACCOUNT_ID"/"$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-"$TS".log + echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" } tfapply(){ terraform apply "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" } +tfrun(){ + export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID + echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" + if [[ "$TF_STAGE" = "plan" ]] + then + set -e + tfinit + tfplan + set +e + elif [[ "$TF_STAGE" = "apply" ]] + then + set -e + tfinit + tfplan + tfapply + set +e + else + echo "Invalid terraform command" + exit 1 + fi +} # if REGIONS is not defined as pipeline parameters use default region if [[ -z "$REGIONS" ]] @@ -51,40 +77,7 @@ do echo "Apply to all accounts" for ACCOUNT_ID in $(jq '.[].AccountId' "$CURRENT"/accounts.json | sed 's/"//g' ) do - export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID - echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" - if [[ "$TF_STAGE" = "plan" ]] - then - tfinit - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfplan - if [[ $? -ne 0 ]] - then - exit 1 - fi - elif [[ "$TF_STAGE" = "apply" ]] - then - tfinit - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfplan - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfapply - if [[ $? -ne 0 ]] - then - exit 1 - fi - else - echo "Invalid terraform command" - fi + tfrun done fi @@ -92,42 +85,10 @@ do then # apply only on a subset of accounts (TARGET_ACCOUNTS) echo "List of target account: $TARGET_ACCOUNTS" + #for i in "${account_list[@]}" for ACCOUNT_ID in $(echo $TARGET_ACCOUNTS | sed "s/,/ /g") do - export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID - echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" - if [[ "$TF_STAGE" = "plan" ]] - then - tfinit - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfplan - if [[ $? -ne 0 ]] - then - exit 1 - fi - elif [[ "$TF_STAGE" = "apply" ]] - then - tfinit - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfplan - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfapply - if [[ $? -ne 0 ]] - then - exit 1 - fi - else - echo "Invalid terraform command" - fi + tfrun done fi @@ -136,40 +97,7 @@ do echo "List target OUs: $TARGET_OUS" for ACCOUNT_ID in $(jq '.[].AccountId' "$CURRENT"/accounts_from_ous.json | sed 's/"//g' ) do - export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID - echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" - if [[ "$TF_STAGE" = "plan" ]] - then - tfinit - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfplan - if [[ $? -ne 0 ]] - then - exit 1 - fi - elif [[ "$TF_STAGE" = "apply" ]] - then - tfinit - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfplan - if [[ $? -ne 0 ]] - then - exit 1 - fi - tfapply - if [[ $? -ne 0 ]] - then - exit 1 - fi - else - echo "Invalid terraform command" - fi + tfrun done fi done From 0b1f697aef083bda260b6c0ba692f337ab4a0ffd Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 12 Nov 2021 22:53:21 +0100 Subject: [PATCH 055/179] clean documentation --- docs/user-guide.md | 299 ++++++++++++++++++++++++--------------------- 1 file changed, 161 insertions(+), 138 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 9aa22ef22..e25c712b1 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -29,7 +29,7 @@ ## Deployment Map -The `deployment_map.yml` file _(or [files](#additional-deployment-maps))_ lives in the repository named `aws-deployment-framework-pipelines` on the Deployment Account. These files are the general pipeline definitions that are responsible for mapping the specific pipelines to their deployment targets along with their respective parameters. The [AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/home.html) will synthesize during the CodeBuild step within the `aws-deployment-framework-pipelines` pipeline. Prior to the CDK creating these pipeline templates, a input generation step will run to parse the deployment_map.yml files, it will then assume a readonly role on the master account in the Organization that will have access to resolve the accounts in the AWS Organizations OU's specified in the mapping file. It will return the account name and ID for each of the accounts and pass those values into the input files that will go on to be main CDK applications inputs. +The `deployment_map.yml` file *(or [files](#additional-deployment-maps))* lives in the repository named `aws-deployment-framework-pipelines` on the Deployment Account. These files are the general pipeline definitions that are responsible for mapping the specific pipelines to their deployment targets along with their respective parameters. The [AWS CDK](https://docs.aws.amazon.com/cdk/latest/guide/home.html) will synthesize during the CodeBuild step within the `aws-deployment-framework-pipelines` pipeline. Prior to the CDK creating these pipeline templates, a input generation step will run to parse the deployment_map.yml files, it will then assume a readonly role on the master account in the Organization that will have access to resolve the accounts in the AWS Organizations OU's specified in the mapping file. It will return the account name and ID for each of the accounts and pass those values into the input files that will go on to be main CDK applications inputs. The deployment map file defines the pipelines along with their inputs, providers to use and their configuration. It also defines the targets of the @@ -53,7 +53,7 @@ pipelines: properties: account_id: 111112233332 # The AWS Account where the source code will be in a CodeCommit Repository params: - notification_endpoint: janes_team@doe.com # Optional + notification_endpoint: janes_team@doe.com # Optional tags: foo: bar # Pipelines support tagging targets: @@ -72,17 +72,17 @@ pipelines: oauth_token_path: /adf/github_token # The path in AWS Secrets Manager that holds the GitHub Oauth token, ADF only has access to /adf/ prefix in Secrets Manager json_field: token # The field (key) name of the json object stored in AWS Secrets Manager that holds the Oauth token params: - notification_endpoint: joes_team@company.nl + notification_endpoint: joes_team@company.nl targets: - path: /banking/testing name: fancy-name #Optional way to pass a name for this stage in the pipeline ``` -In the above example we are creating two pipelines with AWS CodePipeline. The first one will deploy from a repository named **iam** that lives in the account **123456789101**. This CodeCommit Repository will automatically be created by default in the 123456789101 AWS Account if it does not exist. The automatic repository creation occurs if you enable `'auto-create-repositories'` (which is enabled by default). The `iam` pipeline will use AWS CodeCommit as its source and deploy in 3 steps. The first stage of the deployment will occur against all AWS Accounts that are in the `/security` Organization unit and be targeted to the `eu-west-1` region. After that, there is a manual approval phase which is denoted by the keyword `approval`. The next step will be targeted to the accounts within the `/banking/testing` OU _(in your default deployment account region)_ region. By providing a simple path without a region definition it will default to the region chosen as the deployment account region in your [adfconfig](./admin-guide/adfconfig.yml). Any failure during the pipeline will cause it to halt. +In the above example we are creating two pipelines with AWS CodePipeline. The first one will deploy from a repository named **iam** that lives in the account **123456789101**. This CodeCommit Repository will automatically be created by default in the 123456789101 AWS Account if it does not exist. The automatic repository creation occurs if you enable `'auto-create-repositories'` (which is enabled by default). The `iam` pipeline will use AWS CodeCommit as its source and deploy in 3 steps. The first stage of the deployment will occur against all AWS Accounts that are in the `/security` Organization unit and be targeted to the `eu-west-1` region. After that, there is a manual approval phase which is denoted by the keyword `approval`. The next step will be targeted to the accounts within the `/banking/testing` OU *(in your default deployment account region)* region. By providing a simple path without a region definition it will default to the region chosen as the deployment account region in your [adfconfig](./admin-guide/adfconfig.yml). Any failure during the pipeline will cause it to halt. -The second pipeline (_vpc_) example deploys to an OU path `/banking/testing`. You can choose between an absolute path in your AWS Organization, AWS Account ID or an array of OUs or IDs. This pipeline also uses Github as a source rather than AWS CodeCommit. When generating the pipeline, ADF expects [GitHub Token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) to be placed in AWS Secrets Manager in a path prefixed with `/adf/`. +The second pipeline (*vpc*) example deploys to an OU path `/banking/testing`. You can choose between an absolute path in your AWS Organization, AWS Account ID or an array of OUs or IDs. This pipeline also uses Github as a source rather than AWS CodeCommit. When generating the pipeline, ADF expects [GitHub Token](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) to be placed in AWS Secrets Manager in a path prefixed with `/adf/`. -By default, the above pipelines will be created to deploy CloudFormation using a change in two actions _(Create then Execute)_. +By default, the above pipelines will be created to deploy CloudFormation using a change in two actions *(Create then Execute)*. #### Targeting via Tags @@ -91,12 +91,13 @@ Tags on AWS Accounts can also be used to define stages within a pipeline. For ex We do that with the following syntax: ```yaml -- name: vpc-for-foo-team - default_providers: ... - targets: - - tags: # Using tags to define the stage rather than a path or account id - cost-center: foo-team - name: foo-team # You can optionally use the name key to give this stage some meaningful name + - name: vpc-for-foo-team + default_providers: + ... + targets: + - tags: # Using tags to define the stage rather than a path or account id + cost-center: foo-team + name: foo-team # You can optionally use the name key to give this stage some meaningful name ``` Adding or Removing Tags to an AWS Account in AWS Organizations will automatically trigger a run of the bootstrap pipeline which will in turn execute the pipeline generation pipeline in the deployment account. @@ -145,11 +146,11 @@ pipelines: The pipeline `sample-ec2-java-app-codedeploy` has a `default_providers` key that defines the high-level structure of the pipeline. -It explicitly defines the source _(requirement for all pipelines)_ and also +It explicitly defines the source *(requirement for all pipelines)* and also defines what type of build will occur along with any associated parameters. In this example, we're explicitly saying we want to use AWS CodeBuild -_(which is also the default)_ and also to use a specific Docker Image for build +*(which is also the default)* and also to use a specific Docker Image for build stage. The default deployment provider for this pipeline is configured to be `codedeploy` in this example. This means that any of the targets of the pipeline will use AWS CodeDeploy as their default @@ -157,7 +158,7 @@ pipeline will use AWS CodeDeploy as their default In the targets section itself we have the opportunity to override the provider itself or pass in any additional properties to that provider. In this example -we are passing in `application_name` and _deployment_group_name_ as properties +we are passing in `application_name` and *deployment_group_name* as properties to CodeDeploy for this specific stage. The `properties` can either be defined by changing the `default_providers` configuration or get updated at the stage level. Stage level config overrides default provider config. @@ -170,14 +171,14 @@ For detailed information on providers and their supported properties, see the ### Targets Syntax -The Deployment Map has a shorthand syntax along with a more detailed version when you need extra configuration for the _targets_ key as detailed below: +The Deployment Map has a shorthand syntax along with a more detailed version when you need extra configuration for the *targets* key as detailed below: **Shorthand:** ```yaml targets: - 9999999999 # Single Account, Deployment Account Region - - /my_ou/production # Group of Accounts, Deployment Account Region + - /my_ou/production # Group of Accounts, Deployment Account Region ``` **Detailed:** @@ -203,62 +204,68 @@ Pipelines also have parameters that don't relate to a specific stage but rather The following are the available pipeline parameters: -- _notification_endpoint_ _(String)_ defaults to none. - - > Can either be a valid email address or a string that represents the name of a Slack Channel. In order to integrate ADF with Slack see [Integrating with Slack](./admin-guide.md) in the admin guide. By Default, Notifications will be sent when pipelines Start, Complete or Fail. - -- _schedule_ _(String)_ defaults to none. - +- *notification_endpoint* *(String) | (Dict) * defaults to none. + > Can either be a valid email address or a string that represents the name of a Slack Channel. + > A more complex configuration can be provided to integrate with Slack via AWS ChatBot. + > ```yaml + > notification_endpoint: + > type: chat_bot + > target: example_slack_channel # This is the name of an slack channel configuration you created within the AWS Chat Bot service. This needs to be created before you apply the changes to the deployment map. + > ``` + > + > In order to integrate ADF with Slack see [Integrating with Slack](./admin-guide.md#integrating-with-slack-with-aws-chatbot) in the admin guide. By default, notifications will be sent when pipelines Start, Complete, or Fail. + +- *schedule* *(String)* defaults to none. > If the Pipeline should execute on a specific Schedule. Schedules are defined by using a Rate or an Expression. See [here](https://docs.aws.amazon.com/AmazonCloudWatch/latest/events/ScheduledEvents.html#RateExpressions) for more information on how to define Rate or an Expression. -- _restart_execution_on_update_ _(Boolean)_ default: `False`. +- *restart_execution_on_update* *(Boolean)* default: `False`. > If the Pipeline should start a new execution if its structure is updated. Pipelines can often update their structure if targets of the pipeline are Organizational Unit paths. This setting allows pipelines to automatically run once an AWS Account has been moved in or out of a targeted OU. ### Completion Triggers -Pipelines can also trigger other pipelines upon completion. To do this, use the _completion_trigger_ key on the pipeline definition. For example: +Pipelines can also trigger other pipelines upon completion. To do this, use the *completion_trigger* key on the pipeline definition. For example: ```yaml -- name: ami-builder - default_providers: - source: - provider: codecommit - properties: - account_id: 222222222222 - build: - provider: codebuild - role: packer - size: medium - params: - schedule: rate(7 days) - completion_trigger: # What should happen when this pipeline completes - pipelines: - - my-web-app-pipeline # Start this pipeline - -- name: my-web-app-pipeline - default_providers: - source: - provider: github - properties: - repository: my-web-app - owner: cool_coder - oauth_token_path: /adf/github_token - json_field: token - targets: - - path: /banking/testing - name: web-app-testing + - name: ami-builder + default_providers: + source: + provider: codecommit + properties: + account_id: 222222222222 + build: + provider: codebuild + role: packer + size: medium + params: + schedule: rate(7 days) + completion_trigger: # What should happen when this pipeline completes + pipelines: + - my-web-app-pipeline # Start this pipeline + + - name: my-web-app-pipeline + default_providers: + source: + provider: github + properties: + repository: my-web-app + owner: cool_coder + oauth_token_path: /adf/github_token + json_field: token + targets: + - path: /banking/testing + name: web-app-testing ``` -In the above example, the _ami-builder_ pipeline runs every 7 days based on its schedule. When it completes, it executes the _my-web-app-pipeline_ pipeline as defined in its _completion_trigger_ property. +In the above example, the *ami-builder* pipeline runs every 7 days based on its schedule. When it completes, it executes the *my-web-app-pipeline* pipeline as defined in its *completion_trigger* property. ### Additional Deployment Maps You can also create additional deployment map files. -These can live in a folder in the pipelines repository called _deployment_maps_. +These can live in a folder in the pipelines repository called *deployment_maps*. These are entirely optional, but can help split up complex environments with many pipelines. For example, you might have a map used for infrastructure type pipelines and one used for deploying applications. -These additional deployment map files can have any name, as long as they end with _.yml_. +These additional deployment map files can have any name, as long as they end with *.yml*. Taking it a step further, you can create a map per service. So you can organize these deployment map files inside your preferred directory structure @@ -282,7 +289,7 @@ Source entities for pipelines can consist of AWS CodeCommit Repositories, Amazon ### Removing Pipelines -If you decide you no longer require a specific pipeline you can remove it from the deployment*map.yml file and commit those changes back to the \_aws-deployment-framework-pipelines* repository _(on the deployment account)_ in order for it to be cleaned up. The resources that were created as outputs from this pipeline will **not** be removed by this process. +If you decide you no longer require a specific pipeline you can remove it from the deployment_map.yml file and commit those changes back to the *aws-deployment-framework-pipelines* repository *(on the deployment account)* in order for it to be cleaned up. The resources that were created as outputs from this pipeline will **not** be removed by this process. ## Deploying via Pipelines @@ -302,24 +309,24 @@ phases: - pip install -r adf-build/requirements.txt -q # Install Requirements via requirements.txt - python adf-build/generate_params.py # Generate Parameter files dynamically artifacts: - files: "**/*" # Package up all outputs and pass them along to next stage + files: '**/*' # Package up all outputs and pass them along to next stage ``` -In the example we have three steps to our install phase in our build, the remaining phases and steps you add are up to you. In the above steps we simply bring in the shared modules we will need to run our main function in _generate_params.py_. The $S3_BUCKET_NAME variable is available in AWS CodeBuild as we pass this in from our initial creation of the that defines the CodeBuild Project. You do not need to change this. +In the example we have three steps to our install phase in our build, the remaining phases and steps you add are up to you. In the above steps we simply bring in the shared modules we will need to run our main function in *generate_params.py*. The $S3_BUCKET_NAME variable is available in AWS CodeBuild as we pass this in from our initial creation of the that defines the CodeBuild Project. You do not need to change this. -Other packages such as [cfn-lint](https://github.com/awslabs/cfn-python-lint) can be installed in order to validate that our CloudFormation templates are up to standard and do not contain any obvious errors. If you wish to add in any extra packages you can add them to the _requirements.txt_ in the `bootstrap_repository` which is brought down into AWS CodeBuild and installed. Otherwise you can add them into any pipelines specific buildspec.yml. +Other packages such as [cfn-lint](https://github.com/awslabs/cfn-python-lint) can be installed in order to validate that our CloudFormation templates are up to standard and do not contain any obvious errors. If you wish to add in any extra packages you can add them to the *requirements.txt* in the `bootstrap_repository` which is brought down into AWS CodeBuild and installed. Otherwise you can add them into any pipelines specific buildspec.yml. -If you wish to hide away the steps that can occur in AWS CodeBuild, you can move the _buildspec.yml_ content itself into the pipeline by using the _spec_inline_ property in your map files. By doing this, you can remove the option to have a buildspec.yml in the source repository at all. This is a potential way to enforce certain build steps for certain pipeline types. +If you wish to hide away the steps that can occur in AWS CodeBuild, you can move the *buildspec.yml* content itself into the pipeline by using the *spec_inline* property in your map files. By doing this, you can remove the option to have a buildspec.yml in the source repository at all. This is a potential way to enforce certain build steps for certain pipeline types. #### Custom Build Images - You can use [custom build](https://aws.amazon.com/blogs/devops/extending-aws-codebuild-with-custom-build-environments/) environments in AWS CodeBuild. This can be defined in the your deployment map files like so: ```yaml pipelines: - name: example-custom-image default_providers: - source: ... + source: + ... build: provider: codebuild image: @@ -329,22 +336,38 @@ pipelines: - ... ``` +Public images from docker hub can be defined in your deployment map like so: + +```yaml +pipelines: + - name: example-custom-image + default_providers: + source: + ... + build: + provider: codebuild + properties: + image: docker-hub://bitnami/mongodb + targets: + - ... +``` + ### CloudFormation Parameters and Tagging -When you define CloudFormation templates as artifacts to push through a pipeline you might want to have a set of parameters associated with the templates. You can utilize the `params` folder in your repository to add in parameters as you see fit. To avoid having to create a parameter file for each of the stacks you wish to deploy to, you can create a parameter file called `global.yml` _(or .json)_ any parameters defined in this file will be merged into the parameters for any specific account parameter file at build time. For example you might have a single parameter for a template called `CostCenter` the value of this will be the same across every deployment of your application however you might have another parameter called `InstanceType` that you want to be different per account. Using this example we can create a `global.yml` file that contains the following content: +When you define CloudFormation templates as artifacts to push through a pipeline you might want to have a set of parameters associated with the templates. You can utilize the `params` folder in your repository to add in parameters as you see fit. To avoid having to create a parameter file for each of the stacks you wish to deploy to, you can create a parameter file called `global.yml` *(or .json)* any parameters defined in this file will be merged into the parameters for any specific account parameter file at build time. For example you might have a single parameter for a template called `CostCenter` the value of this will be the same across every deployment of your application however you might have another parameter called `InstanceType` that you want to be different per account. Using this example we can create a `global.yml` file that contains the following content: ```yaml Parameters: - CostCenter: department-abc + CostCenter: department-abc ``` -This can be represented in _json_ in the same way if desired. +This can be represented in *json* in the same way if desired. ```json { - "Parameters": { - "CostCenter": "department-abc" - } + "Parameters": { + "CostCenter": "department-abc" + } } ``` @@ -352,20 +375,20 @@ Then we can have a more specific parameter for another account, this file should ```yaml Parameters: - InstanceType: m5.large + InstanceType: m5.large ``` When the stack is executed it will be executed with the following parameters: ```yaml Parameters: - InstanceType: m5.large - CostCenter: department-abc + InstanceType: m5.large + CostCenter: department-abc ``` -This aggregation of parameters works for a few different levels, where the most specific level takes precedence. In the example above, if _CostCenter_ is defined in both `global.yml` and `account.yml` _("account" here represents the name of the account)_ then the value in the `account.yml` file will take precedence. +This aggregation of parameters works for a few different levels, where the most specific level takes precedence. In the example above, if *CostCenter* is defined in both `global.yml` and `account.yml` *("account" here represents the name of the account)* then the value in the `account.yml` file will take precedence. -The different types of parameter files and their order of precedence _(in the tree below, the lowest level has the highest precedence)_ can be used to simplify how parameters are specified. For example, a parameter such as `Environment` might be the same for all accounts under a certain OU, so placing it under a single `ou.yml` params file means you don't need to populate it for each account under that OU. +The different types of parameter files and their order of precedence *(in the tree below, the lowest level has the highest precedence)* can be used to simplify how parameters are specified. For example, a parameter such as `Environment` might be the same for all accounts under a certain OU, so placing it under a single `ou.yml` params file means you don't need to populate it for each account under that OU. **Note:** When using OU parameter files, the OU must be specified in the deployment map as a target. If only the account number is in the deployment map the corresponding OU parameter file will not be referenced. @@ -387,53 +410,53 @@ This concept also works for applying **Tags** to the resources within your stack ```yml Parameters: - CostCenter: "123" - Environment: testing + CostCenter: '123' + Environment: testing Tags: - TagKey: TagValue - MyKey: MyValue + TagKey: TagValue + MyKey: MyValue ``` -Again this example in _json_ would look like: +Again this example in *json* would look like: ```json { - "Parameters": { - "CostCenter": "123", - "Environment": "testing" - }, - "Tags": { - "TagKey": "TagValue", - "MyKey": "MyValue" - } + "Parameters": { + "CostCenter": "123", + "Environment": "testing" + }, + "Tags": { + "TagKey": "TagValue", + "MyKey": "MyValue" + } } ``` This means that all resources that support tags within your CloudFormation stack will be tagged as defined above. -It is important to keep in mind that each Deployment Provider _(Code Deploy, CloudFormation, Service Catalog etc)_ have their [own parameter structure](https://docs.aws.amazon.com/codepipeline/latest/userguide/reference-pipeline-structure.html) and configuration files. For example, Service catalog allows you to pass a configuration file as such: +It is important to keep in mind that each Deployment Provider *(Code Deploy, CloudFormation, Service Catalog etc)* have their [own parameter structure](https://docs.aws.amazon.com/codepipeline/latest/userguide/reference-pipeline-structure.html) and configuration files. For example, Service catalog allows you to pass a configuration file as such: ```json { - "SchemaVersion": "1.1", - "ProductVersionName": "test", - "ProductVersionDescription": "My awesome product", - "ProductType": "CLOUD_FORMATION_TEMPLATE", - "Properties": { - "TemplateFilePath": "/template.yml" - } + "SchemaVersion": "1.1", + "ProductVersionName": "test", + "ProductVersionDescription": "My awesome product", + "ProductType": "CLOUD_FORMATION_TEMPLATE", + "Properties": { + "TemplateFilePath": "/template.yml" + } } ``` -You can create the above parameter files if you are deploying products to your Service Catalog's in the same fashion as with CloudFormation _(global.yml etc)_. +You can create the above parameter files if you are deploying products to your Service Catalog's in the same fashion as with CloudFormation *(global.yml etc)*. For more examples of parameters and their usage see the `samples` folder in the root of the repository. -_Note:_ Currently only Strings type values are supported as parameters to CloudFormation templates when deploying via AWS CodePipeline. +*Note:* Currently only Strings type values are supported as parameters to CloudFormation templates when deploying via AWS CodePipeline. ### Serverless Transforms -If the template that is being deployed contains a transform, such as a Serverless Transform it needs to be packaged and uploaded to S3 in every region where it will be deployed. This can be achieved by setting the `CONTAINS_TRANSFORM` environment variable to _True_ in your pipeline definition with a deployment map file. Once the environment variable has been set, within your _buildspec.yml_ file you will need to use the _package_transform.sh_ helper script (`bash adf-build/helpers/package_transform.sh`). This script will package your template to each region and transparently generate a region specific template for the pipeline deploy stages. +If the template that is being deployed contains a transform, such as a Serverless Transform it needs to be packaged and uploaded to S3 in every region where it will be deployed. This can be achieved by setting the `CONTAINS_TRANSFORM` environment variable to *True* in your pipeline definition with a deployment map file. Once the environment variable has been set, within your *buildspec.yml* file you will need to use the *package_transform.sh* helper script (`bash adf-build/helpers/package_transform.sh`). This script will package your template to each region and transparently generate a region specific template for the pipeline deploy stages. ```yaml pipelines: @@ -454,7 +477,7 @@ pipelines: ### Parameter Injection -Parameter injection solves problems that occur with Cross Account parameter access. This concept allows the resolution of values directly from SSM Parameter Store within the Deployment account into Parameter files _(eg global.json, account-name.json)_ and also importing of output values from CloudFormation stacks across accounts and regions. +Parameter injection solves problems that occur with Cross Account parameter access. This concept allows the resolution of values directly from SSM Parameter Store within the Deployment account into Parameter files *(eg global.json, account-name.json)* and also importing of output values from CloudFormation stacks across accounts and regions. #### Retrieving parameter values @@ -462,16 +485,16 @@ If you wish to resolve values from Parameter Store on the Deployment Account dir ```yaml Parameters: - Environment: development - InstanceType: m5.large - SomeValueFromSSM: resolve:/my/path/to/value + Environment: development + InstanceType: m5.large + SomeValueFromSSM: resolve:/my/path/to/value ``` -When you use the special keyword **"resolve:"**, the value in the specified path will be fetched from Parameter Store on the deployment account during the CodeBuild Containers execution and populated into the parameter file for each account you have defined. If you plan on using any sensitive data, ensure you are using the [NoEcho](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html) property to ensure it is kept out of the console and logs. Resolving parameters across regions is also possible using the notation of _resolve:region:/my/path/to/value_. This allows you to fetch values from the deployment account in other regions other than the main deployment region. +When you use the special keyword **"resolve:"**, the value in the specified path will be fetched from Parameter Store on the deployment account during the CodeBuild Containers execution and populated into the parameter file for each account you have defined. If you plan on using any sensitive data, ensure you are using the [NoEcho](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/parameters-section-structure.html) property to ensure it is kept out of the console and logs. Resolving parameters across regions is also possible using the notation of *resolve:region:/my/path/to/value*. This allows you to fetch values from the deployment account in other regions other than the main deployment region. To highlight an example of how Parameter Injection can work well, think of the following scenario: You have some value that you wish to rotate on a monthly basis. You have some automation in place that updates the value of a Parameter store parameter on a schedule. Each time this pipeline runs it will check for that value and update the resources accordingly, effectively detaching the parameters from the pipeline itself. -There is also the concept of optionally resolving or importing values. This can be achieved by ending the import or resolve function with a **?**. For example, if you want to resolve a value from Parameter Store that might or might not yet exist you can use an optional resolve _(eg resolve:/my/path/to/myMagicKey?)_. If the key _myMagicKey_ does not exist in Parameter Store then an empty string will be returned as the value. +There is also the concept of optionally resolving or importing values. This can be achieved by ending the import or resolve function with a **?**. For example, if you want to resolve a value from Parameter Store that might or might not yet exist you can use an optional resolve *(eg resolve:/my/path/to/myMagicKey?)*. If the key *myMagicKey* does not exist in Parameter Store then an empty string will be returned as the value. #### Importing output values @@ -479,24 +502,24 @@ Parameter injection is also useful for importing output values from CloudFormati ```yaml Parameters: - BucketInLoggingAccount: "import:123456789101:eu-west-1:stack_name:output_key" + BucketInLoggingAccount: 'import:123456789101:eu-west-1:stack_name:output_key' ``` -In the above example _123456789101_ is the AWS Account Id in which we want to pull a value from, _eu-west-1_ is the region, stack*name is the CloudFormation stack name and \_output_key* is the output key name _(not export name)_. Again, this concept works with the optional style syntax _(eg, import:123456789101:eu-west-1:stack_name:output_key?)_ if the key _output_key_ does not exist at the point in time when this specific import is executed, it will return an empty string as the parameter value rather than an error since it is considered optional. +In the above example *123456789101* is the AWS Account Id in which we want to pull a value from, *eu-west-1* is the region, stack_name is the CloudFormation stack name and *output_key* is the output key name *(not export name)*. Again, this concept works with the optional style syntax *(eg, import:123456789101:eu-west-1:stack_name:output_key?)* if the key *output_key* does not exist at the point in time when this specific import is executed, it will return an empty string as the parameter value rather than an error since it is considered optional. #### Uploading assets -Another built-in function is **upload**, You can use _upload_ to perform an automated upload of a resource such as a template or file into Amazon S3 as part of the build process. -Once the upload is complete, the Amazon S3 URL for the object will be put in place of the _upload_ string in the parameter file. +Another built-in function is **upload**, You can use *upload* to perform an automated upload of a resource such as a template or file into Amazon S3 as part of the build process. +Once the upload is complete, the Amazon S3 URL for the object will be put in place of the *upload* string in the parameter file. -For example, If you are deploying products that will be made available via Service Catalog to many teams throughout your organization _(see samples)_ you will need to reference the AWS CloudFormation template URL of the product as part of the template that creates the product definition. The problem that the **upload** function is solving in this case is that the template URL of the product cannot exist at this point, since the file has not yet been uploaded to S3. +For example, If you are deploying products that will be made available via Service Catalog to many teams throughout your organization *(see samples)* you will need to reference the AWS CloudFormation template URL of the product as part of the template that creates the product definition. The problem that the **upload** function is solving in this case is that the template URL of the product cannot exist at this point, since the file has not yet been uploaded to S3. ```yaml Parameters: - ProductYTemplateURL: "upload:path:productY/template.yml" + ProductYTemplateURL: 'upload:path:productY/template.yml' ``` -In the above example, we are calling the **upload** function on a file called `template.yml` that lives in the _productY_ folder within our repository and then returning the path style URL from S3 (indicated by the word _path_ in the string). The string _"upload:path:productY/template.yml"_ will be replaced by the URL of the object in S3 once it has been uploaded. +In the above example, we are calling the **upload** function on a file called `template.yml` that lives in the *productY* folder within our repository and then returning the path style URL from S3 (indicated by the word *path* in the string). The string *"upload:path:productY/template.yml"* will be replaced by the URL of the object in S3 once it has been uploaded. Syntax: @@ -510,25 +533,25 @@ upload:${region}:${style}:${local_path} There are five different styles that one could choose from. -- `path` style, as shown in the example above, will return the S3 path to the object as. +* `path` style, as shown in the example above, will return the S3 path to the object as. This is referred to as the classic [Path Style method](https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html). - - In case the bucket is stored in us-east-1, it will return: + * In case the bucket is stored in us-east-1, it will return: `https://s3.amazonaws.com/${bucket}/${key}` - - In case the bucket is stored in any other region, it will return: + * In case the bucket is stored in any other region, it will return: `https://s3-${region}.amazonaws.com/${bucket}/${key}` -- `virtual-hosted` style, will return the S3 location using the virtual hosted bucket domain. - - In case the bucket is stored in us-east-1, it will return: +* `virtual-hosted` style, will return the S3 location using the virtual hosted bucket domain. + * In case the bucket is stored in us-east-1, it will return: `https://${bucket}.s3.amazonaws.com/${key}` - - In case the bucket is stored in any other region, it will return: + * In case the bucket is stored in any other region, it will return: `https://${bucket}.s3-${region}.amazonaws.com/${key}` -- `s3-url` style, will return the S3 location using S3 URL with the `s3://` protocol. - As an example, this style is required for [CloudFormation AWS::Include transform](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/create-reusable-transform-function-snippets-and-add-to-your-template-with-aws-include-transform.html). - - It returns: `s3://${bucket}/${key}` -- `s3-uri` style, will return the S3 location using S3 URI without specifying a protocol. +* `s3-url` style, will return the S3 location using S3 URL with the `s3://` protocol. + As an example, this style is required for [CloudFormation AWS::Include transform](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/create-reusable-transform-function-snippets-and-add-to-your-template-with-aws-include-transform.html). + * It returns: `s3://${bucket}/${key}` +* `s3-uri` style, will return the S3 location using S3 URI without specifying a protocol. As an example, this style is required for [CodeBuild project source locations](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-codebuild-project-source.html#cfn-codebuild-project-source-location). - - It returns: `${bucket}/${key}` -- `s3-key-only` style, similar to `s3-uri` but it will only return the `key` value. - - It returns: `${key}` + * It returns: `${bucket}/${key}` +* `s3-key-only` style, similar to `s3-uri` but it will only return the `key` value. + * It returns: `${key}` The `region` is optional. This allows you to upload files to S3 Buckets within specific regions by @@ -540,11 +563,11 @@ the location where `adf-build/generate-params.py` scripts gets executed from. As shown in the example shared above, the file to upload would be the `productY/template.yml` file that is stored in the root of the repository. -The bucket being used to hold the uploaded object is the same Amazon S3 Bucket that holds deployment artifacts _(On the Deployment Account)_ for the specific region which they are intended to be deployed to. Files that are uploaded using this functionality will receive a random name each time they are uploaded. +The bucket being used to hold the uploaded object is the same Amazon S3 Bucket that holds deployment artifacts *(On the Deployment Account)* for the specific region which they are intended to be deployed to. Files that are uploaded using this functionality will receive a random name each time they are uploaded. ### Nested CloudFormation Stacks -AWS CloudFormation allows stacks to create other stacks via the [nested stacks](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html) feature. ADF supports a single entry template which defaults to `template.yml`, the stacks that you wish to nest will need to spawn from this template. Nested stacks allow users to pass a `TemplateURL` value that points directly to another CloudFormation template that is either in S3 or on the File System. If you reference a template on the file system you will need to use the `package_transform.sh` helper script during AWS CodeBuild execution _(during the build phase)_ in your pipeline to package up the contents of your templates into finalized artifacts. +AWS CloudFormation allows stacks to create other stacks via the [nested stacks](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/using-cfn-nested-stacks.html) feature. ADF supports a single entry template which defaults to `template.yml`, the stacks that you wish to nest will need to spawn from this template. Nested stacks allow users to pass a `TemplateURL` value that points directly to another CloudFormation template that is either in S3 or on the File System. If you reference a template on the file system you will need to use the `package_transform.sh` helper script during AWS CodeBuild execution *(during the build phase)* in your pipeline to package up the contents of your templates into finalized artifacts. This can be achieved with a `buildspec.yml` like so: @@ -561,25 +584,25 @@ phases: commands: - bash adf-build/helpers/package_transform.sh artifacts: - files: "**/*" + files: '**/*' ``` This allows us to specify nested stacks that are in the same repository as our main `template.yml` in our like so: ```yaml -MyStack: - Type: "AWS::CloudFormation::Stack" - Properties: - TemplateURL: another_template.yml # file path to the nested stack template + MyStack: + Type: "AWS::CloudFormation::Stack" + Properties: + TemplateURL: another_template.yml # file path to the nested stack template ``` -When the `package_transform.sh` command is executed, the file will be packaged up and uploaded to Amazon S3. Its _TemplateURL_ key will be updated to point to the object in S3 and this will be a valid path when `template.yml` is executed in the deploy stages of your pipeline. +When the `package_transform.sh` command is executed, the file will be packaged up and uploaded to Amazon S3. Its *TemplateURL* key will be updated to point to the object in S3 and this will be a valid path when `template.yml` is executed in the deploy stages of your pipeline. ### Deploying Serverless Applications with SAM -Serverless Applications can also be deployed via ADF _(see samples)_. The only extra step required to deploy a SAM template is that you execute `bash adf-build/helpers/package_transform.sh` from within your build stage like so: +Serverless Applications can also be deployed via ADF *(see samples)*. The only extra step required to deploy a SAM template is that you execute `bash adf-build/helpers/package_transform.sh` from within your build stage like so: -For example, deploying a NodeJS Serverless Application from AWS CodeBuild with the _aws/codebuild/standard:2.0_ image can be done with a _buildspec.yml_ that looks like the following [read more](https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#runtime-versions-buildspec-file): +For example, deploying a NodeJS Serverless Application from AWS CodeBuild with the *aws/codebuild/standard:2.0* image can be done with a *buildspec.yml* that looks like the following [read more](https://docs.aws.amazon.com/codebuild/latest/userguide/build-spec-ref.html#runtime-versions-buildspec-file): ```yaml version: 0.2 @@ -598,12 +621,12 @@ phases: commands: - bash adf-build/helpers/package_transform.sh artifacts: - files: "**/*" + files: '**/*' ``` ### Using Anchors and Alias -You can take advantage of YAML Anchors and Alias' in the deployment map files. As you can see from the example below, The &generic_params and &generic_targets are anchors. They can be added to any mapping, sequence or scalar. Once you create an anchor, you can reference it anywhere within the map again with its alias *(eg *generic_params)\* to reproduce their values, similar to variables. +You can take advantage of YAML Anchors and Alias' in the deployment map files. As you can see from the example below, The &generic_params and &generic_targets are anchors. They can be added to any mapping, sequence or scalar. Once you create an anchor, you can reference it anywhere within the map again with its alias *(eg *generic_params)* to reproduce their values, similar to variables. ```yaml pipelines: @@ -669,7 +692,7 @@ pipelines: targets: *generic_targets ``` -By passing in the Repository name _(repository)_ we are overriding the **name** property which normally is the name of our associated repository. This will tie both of these pipelines to the single _sample-vpc_ repository on the _111111111111_ AWS Account. +By passing in the Repository name *(repository)* we are overriding the **name** property which normally is the name of our associated repository. This will tie both of these pipelines to the single *sample-vpc* repository on the *111111111111* AWS Account. ### Terraform pipeline @@ -775,4 +798,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named adf_locktable. +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named adf_locktable. \ No newline at end of file From ed32b14be59ec3d32eb1a3c274f57e38ea8a7872 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:19:06 +0100 Subject: [PATCH 056/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index e25c712b1..2d118f394 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -706,7 +706,7 @@ NOTE: ADFTerraformPolicy IAM policy is a sample. This policies should NOT be use **Overview** ADF support the deployment of terraform code to multiple accounts and regions through terraform pipelines. -The module consists of four build stages defined in the following file: +The module consists of four build stages defined in the following CodeBuild build specification: - `buildspec.yml`: install the version of terraform specified in the pipeline configuration - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. From 926c9e3af334ed8886d8cacbfe9f258346e2a5c1 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:19:40 +0100 Subject: [PATCH 057/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 2d118f394..369730867 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -710,8 +710,8 @@ The module consists of four build stages defined in the following CodeBuild buil - `buildspec.yml`: install the version of terraform specified in the pipeline configuration - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. -- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan -- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply +- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan. +- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply. An optional approval step could be added between plan and apply as shown in the pipeline definition below. From 8c653aa05f35000a0b57d28382246184fec38e72 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:20:13 +0100 Subject: [PATCH 058/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../shared/helpers/terraform/get_accounts.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index c4e799b22..d39d7e135 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -44,13 +44,15 @@ def get_accounts(): ) # Assume a role into the management accounts role to get account ID's and emails organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') - for account in paginator(organizations.list_accounts): - if account['Status'] == 'ACTIVE': - account_details.append({ - 'AccountId': account['Id'], - 'Email': account['Email'] - }) - return account_details + return list( + map( + lambda account: {'AccountId': account['Id'], 'Email': account['Email']}, + filter( + lambda account: account['Status'] == 'ACTIVE', + paginator(organizations.list_accounts) + ) + ) + ) def get_accounts_from_ous(): parent_ou_id = None From e3762e718f4274a72ff469794937ae3399759894 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:20:33 +0100 Subject: [PATCH 059/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../shared/helpers/terraform/get_accounts.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index d39d7e135..d59cc150b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -59,11 +59,12 @@ def get_accounts_from_ous(): account_list = [] organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') # Read organization root id - root_ids = [] - for id in paginator(organizations.list_roots): - root_ids.append({ - 'AccountId': id['Id'] - }) + root_ids = list( + map( + lambda root: {'AccountId': root['Id']]}, + paginator(organizations.list_roots) + ) + ) root_id = root_ids[0]['AccountId'] for path in OU_PATH.split(','): # Set initial OU to start looking for given OU_PATH From fb9759e723f56190fabeb6cbc2a93e1d9fb41d69 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:20:59 +0100 Subject: [PATCH 060/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index d59cc150b..dffc50c9a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -99,10 +99,10 @@ def get_boto3_client(service, role, session_name): DurationSeconds=900 ) session = boto3.Session( - aws_access_key_id=role['Credentials']['AccessKeyId'], - aws_secret_access_key=role['Credentials']['SecretAccessKey'], - aws_session_token=role['Credentials']['SessionToken'] - ) + aws_access_key_id=role['Credentials']['AccessKeyId'], + aws_secret_access_key=role['Credentials']['SecretAccessKey'], + aws_session_token=role['Credentials']['SessionToken'] + ) return session.client(service) From 7d5b8b0faf8c5b23ecb6cf083f1bcdc47af68050 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:21:21 +0100 Subject: [PATCH 061/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index dffc50c9a..818b3068a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -94,10 +94,10 @@ def get_accounts_from_ous(): def get_boto3_client(service, role, session_name): role = sts.assume_role( - RoleArn=role, - RoleSessionName=session_name, - DurationSeconds=900 - ) + RoleArn=role, + RoleSessionName=session_name, + DurationSeconds=900 + ) session = boto3.Session( aws_access_key_id=role['Credentials']['AccessKeyId'], aws_secret_access_key=role['Credentials']['SecretAccessKey'], From 44c540185bc52726744b6e1dcfca004a42721f76 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:21:36 +0100 Subject: [PATCH 062/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 818b3068a..206c4504e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -130,5 +130,6 @@ def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> li }) return account_list + if __name__ == "__main__": main() \ No newline at end of file From 9fd62e226d9475cd130ea6d21454b2000bf0854d Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:23:19 +0100 Subject: [PATCH 063/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 206c4504e..2f3cfa5b8 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -54,6 +54,7 @@ def get_accounts(): ) ) + def get_accounts_from_ous(): parent_ou_id = None account_list = [] From 3e7ada51415f1f41382b5c2db490cddaa5d2c6c3 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:23:57 +0100 Subject: [PATCH 064/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 2f3cfa5b8..21b676f80 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -39,9 +39,9 @@ def get_accounts(): # Return an array of objects like this: [{'AccountId':'xxx','Email':''}] account_details = [] # [{'AccountId':'123','Email':''},{'AccountId':'456','Email':''}] LOGGER.info( - "Management Account ID: %s", - MANAGEMENT_ACC_ID - ) + "Management Account ID: %s", + MANAGEMENT_ACC_ID + ) # Assume a role into the management accounts role to get account ID's and emails organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') return list( From 9180263b552cf85ac3b899d7e47c8586eb033f28 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:24:26 +0100 Subject: [PATCH 065/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 21b676f80..8845264e5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -37,7 +37,7 @@ def list_organizational_units_for_parent(parent_ou): def get_accounts(): # Return an array of objects like this: [{'AccountId':'xxx','Email':''}] - account_details = [] # [{'AccountId':'123','Email':''},{'AccountId':'456','Email':''}] + account_details = [] LOGGER.info( "Management Account ID: %s", MANAGEMENT_ACC_ID From e7e0679f6c3cabfb8a51666d1fdd63e52f293e2b Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:24:48 +0100 Subject: [PATCH 066/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 8845264e5..143804452 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -35,6 +35,7 @@ def list_organizational_units_for_parent(parent_ou): ] return organizational_units + def get_accounts(): # Return an array of objects like this: [{'AccountId':'xxx','Email':''}] account_details = [] From bafca0d1747dc5555a4d69bcbc2813ec3513b146 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:25:32 +0100 Subject: [PATCH 067/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 369730867..0dd0dc309 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -715,7 +715,7 @@ The module consists of four build stages defined in the following CodeBuild buil An optional approval step could be added between plan and apply as shown in the pipeline definition below. -Refer to sample-terraform repository in samples for an example of terraform pipeline. +Please look into the [sample-terraform](../samples/sample-terraform) pipeline for more details in the setup and integration. **Parameters** From e0be884883b12c6846cea55c9e0b2c46605e7461 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:26:08 +0100 Subject: [PATCH 068/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 0dd0dc309..07a4dcbc0 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -719,10 +719,10 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo **Parameters** -- TERRAFORM_VERSION: the terraform version used to deploy the resource -- TARGET_ACCOUNTS: comma separated list of target accounts -- TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) -- REGIONS: comma separated list of target regions. If this parameter is empty, the main ADF region is used. +- `TERRAFORM_VERSION`: the terraform version used to deploy the resource. +- `TARGET_ACCOUNTS`: comma separated list of target accounts. +- `TARGET_OUS`: comma separated list of target leaf OUs (parent OUs are supported). +- `REGIONS`: comma separated list of target regions. If this parameter is empty, the main ADF region is used. **Deployment procedure** From c315037f36152e75267393d242d492641e1bc240 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:26:36 +0100 Subject: [PATCH 069/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 07a4dcbc0..fa729e7ed 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -764,7 +764,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo 2. Add the project name in params/global.yml file 3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. -4. Add variable definition to tf\variables.tf file and variable values to tfvars/global.auto.tfvars +4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. - Local variables (per account) can be configured using the following naming convention From b0fcd779b75a612877347e8a481d3bbbaa1a9e58 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:26:56 +0100 Subject: [PATCH 070/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index fa729e7ed..f18b12157 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -780,7 +780,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 ``` -5. Push to your sample-terraform ADF repository +5. Push to your terraform ADF repository, for example the sample-terraform one. 6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. From 53572e772db4a0a9804317671745b9b2bccb7f53 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:27:55 +0100 Subject: [PATCH 071/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index f18b12157..92cff39ab 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -781,7 +781,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo ``` 5. Push to your terraform ADF repository, for example the sample-terraform one. -6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. +6. Pipeline contains a manual approval step between terraform plan and terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. From cfbbd37da60258062a958566d3173c25d412ebf6 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:28:16 +0100 Subject: [PATCH 072/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 92cff39ab..52961a7fe 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -798,4 +798,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named adf_locktable. \ No newline at end of file +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. Please note: usage of this locking table is charged on the deployment account. \ No newline at end of file From 17a6c69e9521a825bced3259b2a9f2de7aaf031f Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:28:40 +0100 Subject: [PATCH 073/179] Update samples/sample-terraform/tf/s3.tf Co-authored-by: Simon Kok --- samples/sample-terraform/tf/s3.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/samples/sample-terraform/tf/s3.tf b/samples/sample-terraform/tf/s3.tf index 0beb9d026..68730ecb1 100644 --- a/samples/sample-terraform/tf/s3.tf +++ b/samples/sample-terraform/tf/s3.tf @@ -8,4 +8,6 @@ resource "aws_s3_bucket_public_access_block" "s3-public-block" { block_public_acls = true block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true } From 3a434adac22ad844aa54100d85ab6a7ceb5c0a7f Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:29:43 +0100 Subject: [PATCH 074/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml Co-authored-by: Simon Kok --- .../bootstrap_repository/adf-bootstrap/example-global-iam.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml index a78f4a49f..ddac74877 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml @@ -46,7 +46,7 @@ Resources: # # the target account. You should scope this policy depending on what you would # # like to deploy within certain Organizational Units. # # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. - # # Uncomment this line if you want to enable the terraform extentions + # # Uncomment this line if you want to enable the terraform extensions # Type: AWS::IAM::Role # Properties: # RoleName: "adf-terraform-role" From e9d3750ed7de9feb58577ec31d2399cf3b01a32a Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:30:14 +0100 Subject: [PATCH 075/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index bf3e6547d..fa7547af6 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -69,8 +69,8 @@ fi echo "List of target regions: $REGIONS" for REGION in $(echo $REGIONS | sed "s/,/ /g") do - AWS_REGION=$REGION - export TF_VAR_TARGET_REGION=$REGION + AWS_REGION=$(echo -n $REGION | sed 's/^[ \t]*//;s/[ \t]*$//') # sed trims whitespaces + export TF_VAR_TARGET_REGION=$AWS_REGION # if TARGET_ACCOUNTS and TARGET_OUS are not defined apply to all accounts if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]] then From 256e75b28a8cab8a5c3d245872200c81ea2db933 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Fri, 19 Nov 2021 20:30:50 +0100 Subject: [PATCH 076/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 143804452..d7a5dffed 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -14,8 +14,8 @@ OU_PATH = os.environ["TARGET_OUS"] sts = boto3.client('sts') + def main(): - accounts = get_accounts() with open('accounts.json', 'w') as outfile: json.dump(accounts, outfile) From 4c034f075d3eac23f9ade9cd34451408f374b734 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Mon, 22 Nov 2021 22:43:20 +0100 Subject: [PATCH 077/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index fa7547af6..44134133e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -8,7 +8,7 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name /cross_region/s3_regional_bucket/"$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) - mkdir -p "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" cd "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" || exit cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" # if account related variables exist copy the folder in the work directory From 4a91739b7075bef7723136b1356ab8add0b5422a Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Mon, 22 Nov 2021 22:47:09 +0100 Subject: [PATCH 078/179] removed extra ] --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index d7a5dffed..4be7a1ae4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -63,7 +63,7 @@ def get_accounts_from_ous(): # Read organization root id root_ids = list( map( - lambda root: {'AccountId': root['Id']]}, + lambda root: {'AccountId': root['Id']}, paginator(organizations.list_roots) ) ) From f9bdf73111985429989d95750970ed6fbf5ecc67 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Mon, 22 Nov 2021 22:50:25 +0100 Subject: [PATCH 079/179] changed session name --- .../adf-build/shared/helpers/terraform/get_accounts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 4be7a1ae4..05d999138 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -27,7 +27,7 @@ def main(): def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getOrganizationUnits') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) @@ -44,7 +44,7 @@ def get_accounts(): MANAGEMENT_ACC_ID ) # Assume a role into the management accounts role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountIDs') return list( map( lambda account: {'AccountId': account['Id'], 'Email': account['Email']}, @@ -59,7 +59,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountID') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getRootAccountIDs') # Read organization root id root_ids = list( map( From d98feb64e881989b6853550c5bd0a3c17be8dc8c Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Mon, 22 Nov 2021 23:02:01 +0100 Subject: [PATCH 080/179] move TERRAFORM_VERSION variable to buildspec.yml --- docs/user-guide.md | 9 +++------ samples/sample-terraform/README.md | 8 +++----- samples/sample-terraform/buildspec.yml | 3 +++ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 52961a7fe..03ca93243 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1,6 +1,5 @@ # User Guide -- [User Guide](#user-guide) - [Deployment Map](#deployment-map) - [Targeting via Tags](#targeting-via-tags) - [Important Notes](#important-notes) @@ -737,9 +736,6 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo account_id: 111111111111 # source account id build: provider: codebuild - properties: - environment_variables: - TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0. deploy: provider: codebuild properties: @@ -780,8 +776,9 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 ``` -5. Push to your terraform ADF repository, for example the sample-terraform one. -6. Pipeline contains a manual approval step between terraform plan and terraform apply. Confirm to proceed. +5. Define the desired `TERRAFORM_VERSION` in the buildspec.yml file as shown in the sample-terraform example +6. Push to your terraform ADF repository, for example the sample-terraform one. +7. Pipeline contains a manual approval step between terraform plan and terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 63c98e056..c71824fbe 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -32,9 +32,6 @@ An optional approval step could be added between plan and apply as shown in the account_id: 111111111111 # source account id build: provider: codebuild - properties: - environment_variables: - TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0. deploy: provider: codebuild properties: @@ -75,8 +72,9 @@ An optional approval step could be added between plan and apply as shown in the └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 ``` -5. Push to your sample-terraform ADF repository -6. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. +5. Define the desired `TERRAFORM_VERSION` in the buildspec.yml file +6. Push to your sample-terraform ADF repository +7. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. diff --git a/samples/sample-terraform/buildspec.yml b/samples/sample-terraform/buildspec.yml index 245c84f9a..5a0eba67a 100644 --- a/samples/sample-terraform/buildspec.yml +++ b/samples/sample-terraform/buildspec.yml @@ -1,5 +1,8 @@ version: 0.2 +env: + variables: + TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0. phases: install: commands: From d5f72a24a10377587cdea7e4fe2f1f603f3c001f Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Mon, 22 Nov 2021 23:03:43 +0100 Subject: [PATCH 081/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 05d999138..b67dcd93d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -10,8 +10,7 @@ LOGGER.setLevel(logging.INFO) MANAGEMENT_ACC_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] -if("TARGET_OUS" in os.environ): - OU_PATH = os.environ["TARGET_OUS"] +OU_PATH = os.environ.get("TARGET_OUS"]) sts = boto3.client('sts') From 541262c12e90b02cd4e53929fe73ac48e171dc87 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Mon, 22 Nov 2021 23:04:54 +0100 Subject: [PATCH 082/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index b67dcd93d..bf87983de 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -19,7 +19,7 @@ def main(): with open('accounts.json', 'w') as outfile: json.dump(accounts, outfile) - if("TARGET_OUS" in os.environ): + if OU_PATH: accounts_from_ous = get_accounts_from_ous() with open('accounts_from_ous.json', 'w') as outfile: json.dump(accounts_from_ous, outfile) From 90e37867cb2de252ef261f124ac95d24f17479a2 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Mon, 22 Nov 2021 23:07:23 +0100 Subject: [PATCH 083/179] renamed environment variables --- .../shared/helpers/terraform/get_accounts.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index bf87983de..8e33ededd 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -9,8 +9,8 @@ LOGGER = logging.getLogger(__name__) LOGGER.setLevel(logging.INFO) -MANAGEMENT_ACC_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] -OU_PATH = os.environ.get("TARGET_OUS"]) +MANAGEMENT_ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] +TARGET_OUS = os.environ.get("TARGET_OUS") sts = boto3.client('sts') @@ -19,14 +19,14 @@ def main(): with open('accounts.json', 'w') as outfile: json.dump(accounts, outfile) - if OU_PATH: + if TARGET_OUS: accounts_from_ous = get_accounts_from_ous() with open('accounts_from_ous.json', 'w') as outfile: json.dump(accounts_from_ous, outfile) def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getOrganizationUnits') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getOrganizationUnits') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) @@ -40,10 +40,10 @@ def get_accounts(): account_details = [] LOGGER.info( "Management Account ID: %s", - MANAGEMENT_ACC_ID + MANAGEMENT_ACCOUNT_ID ) # Assume a role into the management accounts role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountIDs') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountIDs') return list( map( lambda account: {'AccountId': account['Id'], 'Email': account['Email']}, @@ -58,7 +58,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACC_ID}:role/OrganizationAccountAccessRole-readonly', 'getRootAccountIDs') + organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getRootAccountIDs') # Read organization root id root_ids = list( map( @@ -67,12 +67,12 @@ def get_accounts_from_ous(): ) ) root_id = root_ids[0]['AccountId'] - for path in OU_PATH.split(','): - # Set initial OU to start looking for given OU_PATH + for path in TARGET_OUS.split(','): + # Set initial OU to start looking for given TARGET_OUS if parent_ou_id is None: parent_ou_id = root_id - # Parse OU_PATH and find the ID + # Parse TARGET_OUS and find the ID ou_hierarchy = path.strip('/').split('/') hierarchy_index = 0 if(path.strip() == '/'): From 7401122c2c3de6b869f4dee495bba81d5335af62 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:36:35 +0100 Subject: [PATCH 084/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 8e33ededd..dc1a62af5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -117,7 +117,7 @@ def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> li ) for page in pages: for child in page['Children']: - account_list.extend(get_account_recursive(org_client, child['Id'], path+ou_id+'/')) + account_list.extend(get_account_recursive(org_client, child['Id'], f"{path}{ou_id}/")) # Get Accounts pages = paginator.paginate( From 0ea718bc3689cfed73496bc9f1bcf35b31192251 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:37:17 +0100 Subject: [PATCH 085/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/get_accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index dc1a62af5..623397954 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -1,7 +1,7 @@ -import boto3 import json -import os import logging +import os +import boto3 from paginator import paginator # Configure logging From d02171cfced70a8169e125a2046ec9d5e9528a95 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:38:10 +0100 Subject: [PATCH 086/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 44134133e..4ad43ac2d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -95,7 +95,7 @@ do if ! [[ -z "$TARGET_OUS" ]] then echo "List target OUs: $TARGET_OUS" - for ACCOUNT_ID in $(jq '.[].AccountId' "$CURRENT"/accounts_from_ous.json | sed 's/"//g' ) + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g' ) do tfrun done From 3d7fcb748362080a03f8b2826db2ef9824f5acd2 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:39:26 +0100 Subject: [PATCH 087/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 4ad43ac2d..09fff4e14 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -85,7 +85,6 @@ do then # apply only on a subset of accounts (TARGET_ACCOUNTS) echo "List of target account: $TARGET_ACCOUNTS" - #for i in "${account_list[@]}" for ACCOUNT_ID in $(echo $TARGET_ACCOUNTS | sed "s/,/ /g") do tfrun From abea92b93b47b40658f70c2a51eecd6ce87cdbcb Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:40:05 +0100 Subject: [PATCH 088/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 09fff4e14..fa5a00090 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -75,7 +75,7 @@ do if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]] then echo "Apply to all accounts" - for ACCOUNT_ID in $(jq '.[].AccountId' "$CURRENT"/accounts.json | sed 's/"//g' ) + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts.json" | sed 's/"//g' ) do tfrun done From b9da4dceb5ce929a74100be1d3e3569c46b2fbf5 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:40:36 +0100 Subject: [PATCH 089/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index fa5a00090..2aaf84113 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -56,7 +56,7 @@ tfrun(){ tfapply set +e else - echo "Invalid terraform command" + echo "Invalid Terraform stage: TF_STAGE = $TF_STAGE" exit 1 fi } From 4201301ad96753dd1eb9cd1099d846aa62edbf1d Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:40:59 +0100 Subject: [PATCH 090/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 03ca93243..afb4f0ec9 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -776,7 +776,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 ``` -5. Define the desired `TERRAFORM_VERSION` in the buildspec.yml file as shown in the sample-terraform example +5. Define the desired `TERRAFORM_VERSION` in the `buildspec.yml` file as shown in the sample-terraform example. ADF supports Terraform version v0.13.0 and later. 6. Push to your terraform ADF repository, for example the sample-terraform one. 7. Pipeline contains a manual approval step between terraform plan and terraform apply. Confirm to proceed. From de8ca78521ee49b402e28e0f6e23d5a9904b7df6 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 22:41:15 +0100 Subject: [PATCH 091/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index afb4f0ec9..532310bc1 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -758,7 +758,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo spec_filename: tf_apply.yml # terraform apply ``` -2. Add the project name in params/global.yml file +2. Add the project name in `params/global.yml` file. 3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. 4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. From d0a51b570a68278b675edd0527144240210ee77b Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 23 Nov 2021 22:46:48 +0100 Subject: [PATCH 092/179] fixed toc identation --- docs/user-guide.md | 52 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 532310bc1..bd3a99d92 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1,30 +1,30 @@ # User Guide - - [Deployment Map](#deployment-map) - - [Targeting via Tags](#targeting-via-tags) - - [Important Notes](#important-notes) - - [Zero-prefixed AWS Account Ids](#zero-prefixed-aws-account-ids) - - [Providers](#providers) - - [Targets Syntax](#targets-syntax) - - [Params](#params) - - [Completion Triggers](#completion-triggers) - - [Additional Deployment Maps](#additional-deployment-maps) - - [Repositories](#repositories) - - [Removing Pipelines](#removing-pipelines) - - [Deploying via Pipelines](#deploying-via-pipelines) - - [BuildSpec](#buildspec) - - [Custom Build Images](#custom-build-images) - - [CloudFormation Parameters and Tagging](#cloudformation-parameters-and-tagging) - - [Serverless Transforms](#serverless-transforms) - - [Parameter Injection](#parameter-injection) - - [Retrieving parameter values](#retrieving-parameter-values) - - [Importing output values](#importing-output-values) - - [Uploading assets](#uploading-assets) - - [Nested CloudFormation Stacks](#nested-cloudformation-stacks) - - [Deploying Serverless Applications with SAM](#deploying-serverless-applications-with-sam) - - [Using Anchors and Alias](#using-anchors-and-alias) - - [One to many relationships](#one-to-many-relationships) - - [Terraform pipeline](#terraform-pipeline) +- [Deployment Map](#deployment-map) + - [Targeting via Tags](#targeting-via-tags) + - [Important Notes](#important-notes) + - [Zero-prefixed AWS Account Ids](#zero-prefixed-aws-account-ids) + - [Providers](#providers) + - [Targets Syntax](#targets-syntax) + - [Params](#params) + - [Completion Triggers](#completion-triggers) + - [Additional Deployment Maps](#additional-deployment-maps) + - [Repositories](#repositories) + - [Removing Pipelines](#removing-pipelines) +- [Deploying via Pipelines](#deploying-via-pipelines) + - [BuildSpec](#buildspec) + - [Custom Build Images](#custom-build-images) + - [CloudFormation Parameters and Tagging](#cloudformation-parameters-and-tagging) + - [Serverless Transforms](#serverless-transforms) + - [Parameter Injection](#parameter-injection) + - [Retrieving parameter values](#retrieving-parameter-values) + - [Importing output values](#importing-output-values) + - [Uploading assets](#uploading-assets) + - [Nested CloudFormation Stacks](#nested-cloudformation-stacks) + - [Deploying Serverless Applications with SAM](#deploying-serverless-applications-with-sam) + - [Using Anchors and Alias](#using-anchors-and-alias) + - [One to many relationships](#one-to-many-relationships) + - [Terraform pipeline](#terraform-pipeline) ## Deployment Map @@ -795,4 +795,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. Please note: usage of this locking table is charged on the deployment account. \ No newline at end of file +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. Please note: usage of this locking table is charged on the deployment account. From 7a166a60d883af4ddd9aaac972232f386d61ee58 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 23 Nov 2021 23:02:34 +0100 Subject: [PATCH 093/179] added details to terraform sections --- docs/user-guide.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index bd3a99d92..41de0b4eb 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -718,10 +718,11 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo **Parameters** -- `TERRAFORM_VERSION`: the terraform version used to deploy the resource. +- `TERRAFORM_VERSION`: the terraform version used to deploy the resource. This parameter must be defined in the `buildspec.yml` file of the repository. - `TARGET_ACCOUNTS`: comma separated list of target accounts. - `TARGET_OUS`: comma separated list of target leaf OUs (parent OUs are supported). - `REGIONS`: comma separated list of target regions. If this parameter is empty, the main ADF region is used. +- `MANAGEMENT_ACCOUNT_ID`: id of the AWS Organizations management account. **Deployment procedure** @@ -759,7 +760,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo ``` 2. Add the project name in `params/global.yml` file. -3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. +3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf` which contain the definition of the remote state file location, terraform provider definition. Any change to these files could affect the standard functionalities of this module. 4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. - Local variables (per account) can be configured using the following naming convention From db13877a4dd949a7d2c3f015e8c2171987d44842 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:03:11 +0100 Subject: [PATCH 094/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 41de0b4eb..c052779e0 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -697,7 +697,7 @@ By passing in the Repository name *(repository)* we are overriding the **name** **Pre-requisites** To enable ADF Terraform extension the following steps are required: -- Rename file example-global-iam.yml to global-iam.yml in the following path aws-deployment-framework-bootstrap/adf-bootstrap/ and comment out the Cloudformation resources ADFTerraformRole and ADFTerraformPolicy +- Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/` and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. - Rename file example-global-iam.yml to global-iam.yml in the following path aws-deployment-framework-bootstrap/adf-bootstrap/deployment and comment out the Cloudformation resources ADFTerraformRole and ADFTerraformPolicy NOTE: ADFTerraformPolicy IAM policy is a sample. This policies should NOT be used for purposes other than testing. You should scope this policy depending on what you would like to deploy within certain Organizational Units. From 48ff0a5157678a960a3339cd93add64f5a451079 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:03:40 +0100 Subject: [PATCH 095/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index c052779e0..8ce6fccd7 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -698,7 +698,7 @@ By passing in the Repository name *(repository)* we are overriding the **name** **Pre-requisites** To enable ADF Terraform extension the following steps are required: - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/` and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. -- Rename file example-global-iam.yml to global-iam.yml in the following path aws-deployment-framework-bootstrap/adf-bootstrap/deployment and comment out the Cloudformation resources ADFTerraformRole and ADFTerraformPolicy +- Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/deployment` (please note: `deployment` at the end) and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. NOTE: ADFTerraformPolicy IAM policy is a sample. This policies should NOT be used for purposes other than testing. You should scope this policy depending on what you would like to deploy within certain Organizational Units. From 5466280636713baddba59a46332a50f2659d8332 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:03:54 +0100 Subject: [PATCH 096/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 8ce6fccd7..34c5d1364 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -700,7 +700,7 @@ To enable ADF Terraform extension the following steps are required: - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/` and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/deployment` (please note: `deployment` at the end) and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. -NOTE: ADFTerraformPolicy IAM policy is a sample. This policies should NOT be used for purposes other than testing. You should scope this policy depending on what you would like to deploy within certain Organizational Units. +Important note: `ADFTerraformPolicy` IAM policy is a sample. This policies should **NOT** be used for purposes other than testing. You should scope this policy depending on what you would like to deploy using Terraform within the selected Organizational Units. **Overview** From 31f52a3a83d9f0ecbe6c27f3ce98b09a75932831 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:04:39 +0100 Subject: [PATCH 097/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 34c5d1364..3af284b73 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -704,7 +704,7 @@ Important note: `ADFTerraformPolicy` IAM policy is a sample. This policies shoul **Overview** -ADF support the deployment of terraform code to multiple accounts and regions through terraform pipelines. +ADF support the deployment of Terraform code to multiple accounts and regions through Terraform pipelines. The module consists of four build stages defined in the following CodeBuild build specification: - `buildspec.yml`: install the version of terraform specified in the pipeline configuration From b9a6edf040897c62f21406c9e1bc3a523535c9d2 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:04:57 +0100 Subject: [PATCH 098/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 3af284b73..4fc6c7d60 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -707,7 +707,7 @@ Important note: `ADFTerraformPolicy` IAM policy is a sample. This policies shoul ADF support the deployment of Terraform code to multiple accounts and regions through Terraform pipelines. The module consists of four build stages defined in the following CodeBuild build specification: -- `buildspec.yml`: install the version of terraform specified in the pipeline configuration +- `buildspec.yml`: install the version of Terraform specified in the pipeline configuration. - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. - `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan. - `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply. From 544a71b4713b91aa6fb1427eecefd1ecea402564 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:05:14 +0100 Subject: [PATCH 099/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index c71824fbe..c920f5068 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -5,7 +5,7 @@ This repository contains a module that manage the deployment of terraform code to multiple accounts and regions. The module consists of four build stages defined in the following file: -- `buildspec.yml`: install the version of terraform specified in the pipeline configuration +- `buildspec.yml`: install the version of Terraform specified in the pipeline configuration. - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. - `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan - `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply From d5ed60ac517bc1451bd1e3a148ede6a7324f8ccc Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:05:28 +0100 Subject: [PATCH 100/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index c920f5068..23d5ef065 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -7,8 +7,8 @@ The module consists of four build stages defined in the following file: - `buildspec.yml`: install the version of Terraform specified in the pipeline configuration. - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. -- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan -- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply +- `tf_plan.yml`: get the list of accounts from the organization and run a Terraform plan. +- `tf_apply.yml`: get the list of accounts from the organization and run a Terraform plan and apply. An optional approval step could be added between plan and apply as shown in the pipeline definition below. From 1f3dba144ff5cebabcd40f7c2fad3f87922e5449 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 23 Nov 2021 23:10:45 +0100 Subject: [PATCH 101/179] renamed terraform to upper case and align content of README.md file in sample-terraform --- docs/user-guide.md | 16 +++++------ samples/sample-terraform/README.md | 45 +++++++++++++++--------------- 2 files changed, 31 insertions(+), 30 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 4fc6c7d60..416e9b635 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -718,7 +718,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo **Parameters** -- `TERRAFORM_VERSION`: the terraform version used to deploy the resource. This parameter must be defined in the `buildspec.yml` file of the repository. +- `TERRAFORM_VERSION`: the Terraform version used to deploy the resource. This parameter must be defined in the `buildspec.yml` file of the repository. - `TARGET_ACCOUNTS`: comma separated list of target accounts. - `TARGET_OUS`: comma separated list of target leaf OUs (parent OUs are supported). - `REGIONS`: comma separated list of target regions. If this parameter is empty, the main ADF region is used. @@ -749,24 +749,24 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo targets: - name: terraform-scan # optional properties: - spec_filename: tf_scan.yml # terraform scan + spec_filename: tf_scan.yml # Terraform scan - name: terraform-plan properties: - spec_filename: tf_plan.yml # terraform plan + spec_filename: tf_plan.yml # Terraform plan - approval # manual approval - name: terraform-apply properties: - spec_filename: tf_apply.yml # terraform apply + spec_filename: tf_apply.yml # Terraform apply ``` 2. Add the project name in `params/global.yml` file. -3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf` which contain the definition of the remote state file location, terraform provider definition. Any change to these files could affect the standard functionalities of this module. +3. Add Terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf` which contain the definition of the remote state file location, Terraform provider definition. Any change to these files could affect the standard functionalities of this module. 4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. - Local variables (per account) can be configured using the following naming convention ``` - tfvars <-- This folder contains the structure to define terraform variables + tfvars <-- This folder contains the structure to define Terraform variables │ └───global.auto.tfvars <-- this file contains global variables applied to all the target accounts │ @@ -778,8 +778,8 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo ``` 5. Define the desired `TERRAFORM_VERSION` in the `buildspec.yml` file as shown in the sample-terraform example. ADF supports Terraform version v0.13.0 and later. -6. Push to your terraform ADF repository, for example the sample-terraform one. -7. Pipeline contains a manual approval step between terraform plan and terraform apply. Confirm to proceed. +6. Push to your Terraform ADF repository, for example the sample-terraform one. +7. Pipeline contains a manual approval step between Terraform plan and Terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 23d5ef065..8bd6b48a7 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -1,25 +1,26 @@ # Terraform template -## Overview +**Overview** -This repository contains a module that manage the deployment of terraform code to multiple accounts and regions. -The module consists of four build stages defined in the following file: +ADF support the deployment of Terraform code to multiple accounts and regions through Terraform pipelines. +The module consists of four build stages defined in the following CodeBuild build specification: - `buildspec.yml`: install the version of Terraform specified in the pipeline configuration. - `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. -- `tf_plan.yml`: get the list of accounts from the organization and run a Terraform plan. -- `tf_apply.yml`: get the list of accounts from the organization and run a Terraform plan and apply. +- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan. +- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply. An optional approval step could be added between plan and apply as shown in the pipeline definition below. -## Parameters +**Parameters** -- TERRAFORM_VERSION: the terraform version used to deploy the resource -- TARGET_ACCOUNTS: comma separated list of target accounts -- TARGET_OUS: comma separated list of target leaf OUs (parent OUs are supported) -- REGIONS: comma separated list of target regions. If this parameter is empty, the main ADF region is used. +- `TERRAFORM_VERSION`: the Terraform version used to deploy the resource. This parameter must be defined in the `buildspec.yml` file of the repository. +- `TARGET_ACCOUNTS`: comma separated list of target accounts. +- `TARGET_OUS`: comma separated list of target leaf OUs (parent OUs are supported). +- `REGIONS`: comma separated list of target regions. If this parameter is empty, the main ADF region is used. +- `MANAGEMENT_ACCOUNT_ID`: id of the AWS Organizations management account. -### Deployment procedure +**Deployment procedure** 1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: @@ -44,24 +45,24 @@ An optional approval step could be added between plan and apply as shown in the targets: - name: terraform-scan # optional properties: - spec_filename: tf_scan.yml # terraform scan + spec_filename: tf_scan.yml # Terraform scan - name: terraform-plan properties: - spec_filename: tf_plan.yml # terraform plan + spec_filename: tf_plan.yml # Terraform plan - approval # manual approval - name: terraform-apply properties: - spec_filename: tf_apply.yml # terraform apply + spec_filename: tf_apply.yml # Terraform apply ``` -2. Add the project name in params/global.yml file -3. Add terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf`. -4. Add variable definition to tf\variables.tf file and variable values to tfvars/global.auto.tfvars +2. Add the project name in `params/global.yml` file. +3. Add Terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf` which contain the definition of the remote state file location, Terraform provider definition. Any change to these files could affect the standard functionalities of this module. +4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. - Local variables (per account) can be configured using the following naming convention ``` - tfvars <-- This folder contains the structure to define terraform variables + tfvars <-- This folder contains the structure to define Terraform variables │ └───global.auto.tfvars <-- this file contains global variables applied to all the target accounts │ @@ -72,9 +73,9 @@ An optional approval step could be added between plan and apply as shown in the └──────│ local.auto.tfvars <-- this file contains variables related to account 222222222222 ``` -5. Define the desired `TERRAFORM_VERSION` in the buildspec.yml file -6. Push to your sample-terraform ADF repository -7. Pipeline contains a manual step approval between terraform plan and terraform apply. Confirm to proceed. +5. Define the desired `TERRAFORM_VERSION` in the `buildspec.yml` file as shown in the sample-terraform example. ADF supports Terraform version v0.13.0 and later. +6. Push to your Terraform ADF repository, for example the sample-terraform one. +7. Pipeline contains a manual approval step between Terraform plan and Terraform apply. Confirm to proceed. Terraform state files are stored in the regional S3 buckets in the deployment account. One state file per account/region/module is created. @@ -91,4 +92,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named adf_locktable. +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. Please note: usage of this locking table is charged on the deployment account. From 1d0496c9661b09f8897d63531550854cd1bf6a21 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:12:13 +0100 Subject: [PATCH 102/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 8bd6b48a7..5750868cd 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -59,7 +59,7 @@ An optional approval step could be added between plan and apply as shown in the 3. Add Terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf` which contain the definition of the remote state file location, Terraform provider definition. Any change to these files could affect the standard functionalities of this module. 4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. - - Local variables (per account) can be configured using the following naming convention + - Local variables (per account) can be configured using the following naming convention: ``` tfvars <-- This folder contains the structure to define Terraform variables From 135c6b3aa5eec2bad6580b6ffe29fb5fc1cb0203 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:12:27 +0100 Subject: [PATCH 103/179] Update samples/sample-terraform/buildspec.yml Co-authored-by: Simon Kok --- samples/sample-terraform/buildspec.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/buildspec.yml b/samples/sample-terraform/buildspec.yml index 5a0eba67a..8eabedcfd 100644 --- a/samples/sample-terraform/buildspec.yml +++ b/samples/sample-terraform/buildspec.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TERRAFORM_VERSION: "1.0.10" # terraform version. The module support terraform version greater than 0.13.0. + TERRAFORM_VERSION: "1.0.10" # Terraform version to use. ADF supports Terraform version v0.13.0 and later. phases: install: commands: From 4b8a1fd83e20ca92b54af1f3c55c7dd347ff0a41 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:12:56 +0100 Subject: [PATCH 104/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 2aaf84113..f28ae67b9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -7,7 +7,7 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store - S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name /cross_region/s3_regional_bucket/"$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) + S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" cd "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" || exit cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" From ba5881ca565febe53b8ce55475085a61a5ad863e Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:13:12 +0100 Subject: [PATCH 105/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index f28ae67b9..db54b37c1 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -9,7 +9,7 @@ tfinit(){ # retrieve regional S3 bucket name from parameter store S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - cd "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" || exit + cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" # if account related variables exist copy the folder in the work directory if [ -d "$CURRENT/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then From f99f07ac267e7de90be570741e4f47e95c27bf6e Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:13:29 +0100 Subject: [PATCH 106/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index db54b37c1..9aae780dc 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -10,7 +10,7 @@ tfinit(){ S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit - cp -R "$CURRENT"/tf/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + cp -R "$CURRENT/tf/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" # if account related variables exist copy the folder in the work directory if [ -d "$CURRENT/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then cp -R "$CURRENT"/tfvars/"$TF_VAR_TARGET_ACCOUNT_ID"/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" From 33de671ab54399fa3fa57e1245632bb594a00d06 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:13:44 +0100 Subject: [PATCH 107/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 9aae780dc..61554d90e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -13,7 +13,7 @@ tfinit(){ cp -R "$CURRENT/tf/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" # if account related variables exist copy the folder in the work directory if [ -d "$CURRENT/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then - cp -R "$CURRENT"/tfvars/"$TF_VAR_TARGET_ACCOUNT_ID"/* "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" fi cp -R "$CURRENT"/tfvars/global.auto.tfvars "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" terraform init \ From 722fa0cab60226df2eacb26d96e08f0cf710bbbe Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:13:59 +0100 Subject: [PATCH 108/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 61554d90e..6bef7c668 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -15,7 +15,7 @@ tfinit(){ if [ -d "$CURRENT/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" fi - cp -R "$CURRENT"/tfvars/global.auto.tfvars "$CURRENT"/tmp/"$TF_VAR_TARGET_ACCOUNT_ID"-"$AWS_REGION" + cp -R "${CURRENT}/tfvars/global.auto.tfvars" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" terraform init \ -backend-config "bucket=$S3_BUCKET_REGION_NAME" \ -backend-config "region=$AWS_REGION" \ From fed666a13743c14473f9827aeaab1e710adecda9 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:14:20 +0100 Subject: [PATCH 109/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 6bef7c668..8314a008a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -30,10 +30,10 @@ tfinit(){ tfplan(){ DATE=$(date +%Y-%m-%d) TS=$(date +%Y%m%d%H%M%S) - bash "$CURRENT"/adf-build/helpers/sts.sh "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" - terraform plan -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" 2>&1 | tee -a "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-"$TS".log - # Save terraform plan to S3 bucket - aws s3 cp "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-"$TS".log s3://"$S3_BUCKET_REGION_NAME"/"$ADF_PROJECT_NAME"/tf-plan/"$DATE"/"$TF_VAR_TARGET_ACCOUNT_ID"/"$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-"$TS".log + bash "$CURRENT/adf-build/helpers/sts.sh" "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" + terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" + # Save Terraform plan results to the S3 bucket + aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" } tfapply(){ From 8ec3acd6d80e0bfff4a2479d0f8ad550bc8dbfc5 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Tue, 23 Nov 2021 23:14:38 +0100 Subject: [PATCH 110/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 8314a008a..04793e114 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -37,7 +37,7 @@ tfplan(){ echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" } tfapply(){ - terraform apply "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID" + terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" } tfrun(){ export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID From 2e3e139bed01cd90fc04118fc256973292402df8 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 23 Nov 2021 23:16:28 +0100 Subject: [PATCH 111/179] removed Terraform IAM role and policy --- .../adf-bootstrap/global.yml | 38 ------------------- 1 file changed, 38 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml index a584fe83f..8886892a5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/global.yml @@ -317,41 +317,3 @@ Resources: - "*" Roles: - !Ref ReadOnlyAutomationRole - ADFTerraformRole: - # ADF role to run terraform pipelines - Type: AWS::IAM::Role - Properties: - RoleName: "adf-terraform-role" - AssumeRolePolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Sid: "AssumeRole" - Principal: - AWS: - # This would allow all codebuild projects to be able to assume this role - - !Sub arn:aws:iam::${DeploymentAccountId}:role/adf-codebuild-role - # - !Sub arn:aws:iam::${DeploymentAccountId}:role/my-custom-codebuild-role - # The above role would be created on the deployment account for the purpose deploying this custom resource via codebuild - Action: - - sts:AssumeRole - Path: / - ADFTerraformPolicy: - Type: AWS::IAM::Policy - Properties: - PolicyName: "adf-terraform-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Action: - - "ec2:*" - - "s3:*" - - "ssm:*" - - "sso:*" - - "identitystore:*" - - "organizations:*" - Resource: - - "*" - Roles: - - !Ref ADFTerraformRole From c8aa79487f859efc4fe017b20ca5b81dcca9dcc2 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 23 Nov 2021 23:21:45 +0100 Subject: [PATCH 112/179] added support to partitions --- .../adf-build/shared/helpers/terraform/get_accounts.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 623397954..ab336540a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -3,6 +3,7 @@ import os import boto3 from paginator import paginator +from partition import get_partition # Configure logging logging.basicConfig(level=logging.INFO) @@ -11,6 +12,8 @@ MANAGEMENT_ACCOUNT_ID = os.environ["MANAGEMENT_ACCOUNT_ID"] TARGET_OUS = os.environ.get("TARGET_OUS") +REGION_DEFAULT = os.environ["AWS_REGION"] +PARTITION = get_partition(REGION_DEFAULT) sts = boto3.client('sts') @@ -26,7 +29,7 @@ def main(): def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getOrganizationUnits') + organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getOrganizationUnits') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) @@ -43,7 +46,7 @@ def get_accounts(): MANAGEMENT_ACCOUNT_ID ) # Assume a role into the management accounts role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountIDs') + organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountIDs') return list( map( lambda account: {'AccountId': account['Id'], 'Email': account['Email']}, @@ -58,7 +61,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:aws:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getRootAccountIDs') + organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getRootAccountIDs') # Read organization root id root_ids = list( map( From 2cccef32767f4e43ad4c3bdd86dde2018e0ef7ec Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:59:25 +0100 Subject: [PATCH 113/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 416e9b635..02be81886 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -708,7 +708,7 @@ ADF support the deployment of Terraform code to multiple accounts and regions th The module consists of four build stages defined in the following CodeBuild build specification: - `buildspec.yml`: install the version of Terraform specified in the pipeline configuration. -- `tf_scan.yml`: (optional) scans for vulnerabilities in the terraform code using the terrascan application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF terraform pipelines. +- `tf_scan.yml`: (optional) scans for vulnerabilities in the Terraform code using the [Terrascan](https://github.com/accurics/terrascan) application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF Terraform pipelines. - `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan. - `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply. From 75f97dcb385d5e40b1386ceff94e9a0da40d86d5 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:59:32 +0100 Subject: [PATCH 114/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 02be81886..5a42788e2 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -709,8 +709,8 @@ The module consists of four build stages defined in the following CodeBuild buil - `buildspec.yml`: install the version of Terraform specified in the pipeline configuration. - `tf_scan.yml`: (optional) scans for vulnerabilities in the Terraform code using the [Terrascan](https://github.com/accurics/terrascan) application. If vulnerabilities are found, it will fail and block further execution in the pipeline. It is recommended to enable this step in all ADF Terraform pipelines. -- `tf_plan.yml`: get the list of accounts from the organization and run a terraform plan. -- `tf_apply.yml`: get the list of accounts from the organization and run a terraform plan and apply. +- `tf_plan.yml`: get the list of accounts from the organization and run a Terraform plan. +- `tf_apply.yml`: get the list of accounts from the organization and run a Terraform plan and apply. An optional approval step could be added between plan and apply as shown in the pipeline definition below. From 81c20d906e0b1fa018155c9182a9a97edafe08a4 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:59:38 +0100 Subject: [PATCH 115/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 5a42788e2..d8774a384 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -760,7 +760,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo ``` 2. Add the project name in `params/global.yml` file. -3. Add Terraform code to the `tf` folder. Do not make changes to `backend.tf` file and `main.tf` which contain the definition of the remote state file location, Terraform provider definition. Any change to these files could affect the standard functionalities of this module. +3. Add Terraform code to the `tf` folder. **Please note**: Do not make changes to `backend.tf` file and `main.tf` in the root folder of the sample. These contain the definition of the remote state file location and the Terraform provider definition. Any change to these files could disrupt the standard functionalities of this module. 4. Add variable definition to `tf/variables.tf` file and variable values to `tfvars/global.auto.tfvars`. - Local variables (per account) can be configured using the following naming convention From 9313a17488b20eaf24b853581ce76259fa294a05 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:59:44 +0100 Subject: [PATCH 116/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index d8774a384..ce47133da 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -796,4 +796,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. Please note: usage of this locking table is charged on the deployment account. +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. **Please note**: usage of this locking table is charged on the deployment account. From 196efb97c9a70dd923bda4277dbe0de5e7db359b Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:59:50 +0100 Subject: [PATCH 117/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index ce47133da..a51255519 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -695,7 +695,7 @@ By passing in the Repository name *(repository)* we are overriding the **name** ### Terraform pipeline -**Pre-requisites** +#### Prerequisites To enable ADF Terraform extension the following steps are required: - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/` and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/deployment` (please note: `deployment` at the end) and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. From b9dafee539dcff015853224accfe5f09e475a87c Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 11:59:56 +0100 Subject: [PATCH 118/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 5750868cd..382b7898d 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -20,7 +20,7 @@ An optional approval step could be added between plan and apply as shown in the - `REGIONS`: comma separated list of target regions. If this parameter is empty, the main ADF region is used. - `MANAGEMENT_ACCOUNT_ID`: id of the AWS Organizations management account. -**Deployment procedure** +## Deployment procedure 1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: From 139895739229aee0cb022d4be2826f6cf4e82c59 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:00:01 +0100 Subject: [PATCH 119/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 382b7898d..d800a5d78 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -92,4 +92,4 @@ The following state files are created: - 222222222222 main region (eu-west-1) adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate - 222222222222 secondary region (us-east-1) adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. Please note: usage of this locking table is charged on the deployment account. +A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. **Please note**: usage of this locking table is charged on the deployment account. From fb84415080615e06d444599d4e99ec15b1155dc3 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:04:09 +0100 Subject: [PATCH 120/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml Co-authored-by: Simon Kok --- .../adf-bootstrap/deployment/example-global-iam.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index a635cb2b3..814654bcb 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -30,7 +30,7 @@ Resources: # # the target account. You should scope this policy depending on what you would # # like to deploy within certain Organizational Units. # # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. - # # Uncomment this line if you want to enable the terraform extentions + # # Uncomment the `ADFTerraformRole` and `ADFTerraformPolicy` resources if you want to enable Terraform support in ADF. # Type: AWS::IAM::Role # Properties: # RoleName: "adf-terraform-role" From fb4c66113555d9b8657c419539edd157449262d7 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Wed, 24 Nov 2021 12:05:44 +0100 Subject: [PATCH 121/179] removed TerraformLockTable resource as already defined in regional.yml --- .../adf-bootstrap/deployment/global.yml | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index aa054d47f..8c2f63c87 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -1272,17 +1272,6 @@ Resources: - Arn: !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}" RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline - TerraformLockTable: - Type: "AWS::DynamoDB::Table" - Properties: - AttributeDefinitions: - - AttributeName: LockID - AttributeType: S - KeySchema: - - AttributeName: LockID - KeyType: HASH - BillingMode: PAY_PER_REQUEST - TableName: adf-tflocktable Outputs: ADFVersionNumber: From f0d44f5fdbfe27d1a4999af38ad2eb76f27f2d10 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:06:22 +0100 Subject: [PATCH 122/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index a51255519..de7ee4b90 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -702,7 +702,7 @@ To enable ADF Terraform extension the following steps are required: Important note: `ADFTerraformPolicy` IAM policy is a sample. This policies should **NOT** be used for purposes other than testing. You should scope this policy depending on what you would like to deploy using Terraform within the selected Organizational Units. -**Overview** +#### Overview ADF support the deployment of Terraform code to multiple accounts and regions through Terraform pipelines. The module consists of four build stages defined in the following CodeBuild build specification: From 9d2c71d8bf9de672b40638b7bf9ae082b09f580c Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:06:30 +0100 Subject: [PATCH 123/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index de7ee4b90..15c339cab 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -716,7 +716,7 @@ An optional approval step could be added between plan and apply as shown in the Please look into the [sample-terraform](../samples/sample-terraform) pipeline for more details in the setup and integration. -**Parameters** +#### Parameters - `TERRAFORM_VERSION`: the Terraform version used to deploy the resource. This parameter must be defined in the `buildspec.yml` file of the repository. - `TARGET_ACCOUNTS`: comma separated list of target accounts. From beb9791c43b5558b5c4d2c7620b3b0fe404bf44e Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:06:38 +0100 Subject: [PATCH 124/179] Update docs/user-guide.md Co-authored-by: Simon Kok --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 15c339cab..63588eb81 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -724,7 +724,7 @@ Please look into the [sample-terraform](../samples/sample-terraform) pipeline fo - `REGIONS`: comma separated list of target regions. If this parameter is empty, the main ADF region is used. - `MANAGEMENT_ACCOUNT_ID`: id of the AWS Organizations management account. -**Deployment procedure** +#### Deployment procedure 1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the example: From fb743acbfed1539591ca17b9fa2f821c9bd04819 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:06:45 +0100 Subject: [PATCH 125/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index d800a5d78..3379b581e 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -1,6 +1,6 @@ # Terraform template -**Overview** +## Overview ADF support the deployment of Terraform code to multiple accounts and regions through Terraform pipelines. The module consists of four build stages defined in the following CodeBuild build specification: From 17818b9b39c3d897a25bfaf9b5798a00ad58a699 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Wed, 24 Nov 2021 12:06:54 +0100 Subject: [PATCH 126/179] Update samples/sample-terraform/README.md Co-authored-by: Simon Kok --- samples/sample-terraform/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 3379b581e..feb5ae6fe 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -12,7 +12,7 @@ The module consists of four build stages defined in the following CodeBuild buil An optional approval step could be added between plan and apply as shown in the pipeline definition below. -**Parameters** +## Parameters - `TERRAFORM_VERSION`: the Terraform version used to deploy the resource. This parameter must be defined in the `buildspec.yml` file of the repository. - `TARGET_ACCOUNTS`: comma separated list of target accounts. From 55c544deee47a06ee1a4d671fefcf4b99fc590d9 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 22:42:14 +0100 Subject: [PATCH 127/179] added newline character --- .../bootstrap_repository/adf-bootstrap/deployment/global.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 8c2f63c87..c67b8eac7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -1325,4 +1325,4 @@ Outputs: Description: "The Kms Key Arn" Value: !GetAtt KMSKey.Arn Export: - Name: "KmsKeyArn" \ No newline at end of file + Name: "KmsKeyArn" From e2a93b14e737b48cd2eb98a73a8283c12226c0a0 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 22:45:34 +0100 Subject: [PATCH 128/179] added newline character --- .../adf-build/shared/helpers/terraform/paginator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py index 18e8cd552..e54b6db5c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py @@ -11,4 +11,4 @@ def paginator(method, **kwargs): iterator = client.get_paginator(method.__name__) for page in iterator.paginate(**kwargs).result_key_iters(): for result in page: - yield result \ No newline at end of file + yield result From f98f3e50adfc415ab7c3860a943ceaacb6296048 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 22:51:28 +0100 Subject: [PATCH 129/179] added newline character --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index ab336540a..6c8e645d6 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -136,4 +136,4 @@ def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> li if __name__ == "__main__": - main() \ No newline at end of file + main() From 131c526f6fa17b68f95ac9f12070614ec6cba59e Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 23:00:56 +0100 Subject: [PATCH 130/179] added docstring --- .../shared/helpers/terraform/get_accounts.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 6c8e645d6..26ca665bb 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -1,3 +1,10 @@ +# Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: MIT-0 + +""" +Module used to get accounts list from target OUs. +""" + import json import logging import os @@ -69,7 +76,7 @@ def get_accounts_from_ous(): paginator(organizations.list_roots) ) ) - root_id = root_ids[0]['AccountId'] + root_id = root_ids[0]['AccountId'] for path in TARGET_OUS.split(','): # Set initial OU to start looking for given TARGET_OUS if parent_ou_id is None: @@ -78,7 +85,7 @@ def get_accounts_from_ous(): # Parse TARGET_OUS and find the ID ou_hierarchy = path.strip('/').split('/') hierarchy_index = 0 - if(path.strip() == '/'): + if path.strip() == '/': account_list.extend(get_account_recursive(organizations, parent_ou_id, '/')) else: while hierarchy_index < len(ou_hierarchy): From 4d9719030384c0fd69fe486a6e1887161e90c98b Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 23:29:21 +0100 Subject: [PATCH 131/179] read CROSS_ACCOUNT_ACCESS_ROLE from parameter store --- .../shared/helpers/terraform/get_accounts.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 26ca665bb..bdd88ea7e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -22,6 +22,9 @@ REGION_DEFAULT = os.environ["AWS_REGION"] PARTITION = get_partition(REGION_DEFAULT) sts = boto3.client('sts') +ssm = boto3.client('ssm') +response = ssm.get_parameter(Name='cross_account_access_role') +CROSS_ACCOUNT_ACCESS_ROLE = response['Parameter']['Value'] def main(): @@ -32,11 +35,10 @@ def main(): if TARGET_OUS: accounts_from_ous = get_accounts_from_ous() with open('accounts_from_ous.json', 'w') as outfile: - json.dump(accounts_from_ous, outfile) - + json.dump(accounts_from_ous, outfile) def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getOrganizationUnits') + organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getOrganizationUnits') organizational_units = [ ou for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) @@ -47,13 +49,12 @@ def list_organizational_units_for_parent(parent_ou): def get_accounts(): # Return an array of objects like this: [{'AccountId':'xxx','Email':''}] - account_details = [] LOGGER.info( "Management Account ID: %s", MANAGEMENT_ACCOUNT_ID ) # Assume a role into the management accounts role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getaccountIDs') + organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getaccountIDs') return list( map( lambda account: {'AccountId': account['Id'], 'Email': account['Email']}, @@ -68,7 +69,7 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/OrganizationAccountAccessRole-readonly', 'getRootAccountIDs') + organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getRootAccountIDs') # Read organization root id root_ids = list( map( From 3fcda2d89437e9246d548650b30ea80d6c1281ad Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 23:34:49 +0100 Subject: [PATCH 132/179] replaced sample adf terraform policy example --- .../adf-bootstrap/example-global-iam.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml index b9233112a..02d8b3a93 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml @@ -70,8 +70,18 @@ Resources: # Statement: # - Effect: Allow # Action: - # - "ec2:*" - # - "s3:*" + # - "events:PutRule" + # - "events:PutTargets" + # - "events:DeleteRule" + # - "events:DescribeRule" + # - "events:ListTagsForResource" + # - "events:ListTargetsByRule" + # - "events:RemoveTargets" + # - "sns:CreateTopic" + # - "sns:GetTopicAttributes" + # - "sns:SetTopicAttributes" + # - "sns:ListTagsForResource" + # - "sns:DeleteTopic" # Resource: # - "*" # Roles: From e1ee1a65a91df3bc378ab48c4931c510e315011f Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 23:35:02 +0100 Subject: [PATCH 133/179] replaced sample adf terraform policy example --- .../deployment/example-global-iam.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index 814654bcb..ed01dd00d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -54,8 +54,18 @@ Resources: # Statement: # - Effect: Allow # Action: - # - "ec2:*" - # - "s3:*" + # - "events:PutRule" + # - "events:PutTargets" + # - "events:DeleteRule" + # - "events:DescribeRule" + # - "events:ListTagsForResource" + # - "events:ListTargetsByRule" + # - "events:RemoveTargets" + # - "sns:CreateTopic" + # - "sns:GetTopicAttributes" + # - "sns:SetTopicAttributes" + # - "sns:ListTagsForResource" + # - "sns:DeleteTopic" # Resource: # - "*" # Roles: From 868d88fed2ce1a54cd2716fa998f0d895233bb96 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 23:38:45 +0100 Subject: [PATCH 134/179] added details to adf terraform role description --- .../adf-bootstrap/deployment/example-global-iam.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index ed01dd00d..f5cf05033 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -30,7 +30,9 @@ Resources: # # the target account. You should scope this policy depending on what you would # # like to deploy within certain Organizational Units. # # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. - # # Uncomment the `ADFTerraformRole` and `ADFTerraformPolicy` resources if you want to enable Terraform support in ADF. + # # Uncomment the `ADFTerraformRole` and `ADFTerraformPolicy` resources if you want to enable Terraform support in ADF + # # and deploy resources in Deployment account using Terraform. If you need to push to other account, you don't need to + # # add this policy # Type: AWS::IAM::Role # Properties: # RoleName: "adf-terraform-role" From a327b30abd98f846fda2e35fe9535fcd3a75eb0f Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 23 Dec 2021 23:57:51 +0100 Subject: [PATCH 135/179] removed trailing spaces --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index bdd88ea7e..8b8adbace 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -35,7 +35,7 @@ def main(): if TARGET_OUS: accounts_from_ous = get_accounts_from_ous() with open('accounts_from_ous.json', 'w') as outfile: - json.dump(accounts_from_ous, outfile) + json.dump(accounts_from_ous, outfile) def list_organizational_units_for_parent(parent_ou): organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getOrganizationUnits') From c05057243c9c17ebfc27e29e374ca25195e4b3a3 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 00:01:26 +0100 Subject: [PATCH 136/179] added aws partition variable --- samples/sample-terraform/tf/main.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/samples/sample-terraform/tf/main.tf b/samples/sample-terraform/tf/main.tf index 0c1cb8a42..2886e3dba 100644 --- a/samples/sample-terraform/tf/main.tf +++ b/samples/sample-terraform/tf/main.tf @@ -1,3 +1,5 @@ +data "aws_partition" "current" {} + terraform { required_providers { aws = { @@ -9,6 +11,6 @@ terraform { } provider "aws" { assume_role { - role_arn = "arn:aws:iam::${var.TARGET_ACCOUNT_ID}:role/${var.TARGET_ACCOUNT_ROLE}" + role_arn = "arn:${data.aws_partition.current}:iam::${var.TARGET_ACCOUNT_ID}:role/${var.TARGET_ACCOUNT_ROLE}" } } From 35d314810af4140b88ded89363897a87a14260b1 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 00:04:34 +0100 Subject: [PATCH 137/179] added init stage only as option --- .../shared/helpers/terraform/adf_terraform.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 04793e114..59e15ce57 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -42,7 +42,12 @@ tfapply(){ tfrun(){ export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" - if [[ "$TF_STAGE" = "plan" ]] + if [[ "$TF_STAGE" = "init" ]] + then + set -e + tfinit + set +e + elif [[ "$TF_STAGE" = "plan" ]] then set -e tfinit @@ -67,9 +72,9 @@ then REGIONS=$AWS_DEFAULT_REGION fi echo "List of target regions: $REGIONS" -for REGION in $(echo $REGIONS | sed "s/,/ /g") +for REGION in $(echo "$REGIONS" | sed "s/,/ /g") do - AWS_REGION=$(echo -n $REGION | sed 's/^[ \t]*//;s/[ \t]*$//') # sed trims whitespaces + AWS_REGION=$(echo -n "$REGION" | sed 's/^[ \t]*//;s/[ \t]*$//') # sed trims whitespaces export TF_VAR_TARGET_REGION=$AWS_REGION # if TARGET_ACCOUNTS and TARGET_OUS are not defined apply to all accounts if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]] @@ -85,7 +90,7 @@ do then # apply only on a subset of accounts (TARGET_ACCOUNTS) echo "List of target account: $TARGET_ACCOUNTS" - for ACCOUNT_ID in $(echo $TARGET_ACCOUNTS | sed "s/,/ /g") + for ACCOUNT_ID in $(echo "$TARGET_ACCOUNTS" | sed "s/,/ /g") do tfrun done From d50cc354787c0b4f369041c36a3d8561aca9705a Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:17:43 +0100 Subject: [PATCH 138/179] import extensions parameter in SSM Parameter Store of Management account --- .../bootstrap_repository/adf-build/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py index d1995a0d4..37dfb265d 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py @@ -38,6 +38,7 @@ def __init__(self, parameter_store=None, config_path=None): self.protected = None self.target_regions = [] self.cross_account_access_role = None + self.extensions = None self._load_config_file() def store_config(self): @@ -108,6 +109,7 @@ def _parse_config(self): self.notification_endpoint = self.config.get( 'main-notification-endpoint')[0].get('target') self.notification_channel = None if self.notification_type == 'email' else self.notification_endpoint + self.extensions = self.config_contents.get('extensions', None) self._validate() @@ -143,6 +145,10 @@ def _store_config(self): "config_contents", "config_path", "notification_endpoint", - "notification_type" + "notification_type", + "extensions" ): self.parameters_client.put_parameter(key, str(value)) + for extension, attributes in self.extensions.items(): + for attribute in attributes: + self.parameters_client.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) From 9742f979d3669ceaa7a89f625b38add3cda02ad4 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:18:15 +0100 Subject: [PATCH 139/179] import extensions parameter in SSM Parameter Store of Deployment account (all regions) --- .../initial_commit/bootstrap_repository/adf-build/main.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index f448da84f..74608c0e9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -120,6 +120,10 @@ def prepare_deployment_account(sts, deployment_account_id, config): deployment_account_parameter_store.put_parameter( 'organization_id', os.environ["ORGANIZATION_ID"] ) + if hasattr(config, 'extensions'): + for extension, attributes in config.extensions.items(): + for attribute in attributes: + deployment_account_parameter_store.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) deployment_account_parameter_store = ParameterStore( config.deployment_account_region, @@ -164,6 +168,10 @@ def prepare_deployment_account(sts, deployment_account_id, config): '/notification_endpoint/main' if item == 'notification_channel' else item, str(getattr(config, item)) ) + if hasattr(config, 'extensions'): + for extension, attributes in config.extensions.items(): + for attribute in attributes: + deployment_account_parameter_store.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) return deployment_account_role From a0b660b5843541627e317f11e5594600840aab2c Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:19:14 +0100 Subject: [PATCH 140/179] added condition on DynamoDB table. Deploy only if Tf extension is enabled --- .../adf-bootstrap/deployment/regional.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml index 91dc1d1af..63e98f531 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml @@ -7,6 +7,13 @@ Parameters: OrganizationId: Type: "AWS::SSM::Parameter::Value" Default: organization_id + ADFTerraformExtension: + Type: "AWS::SSM::Parameter::Value" + Default: /adf/extensions/terraform/enabled +Conditions: + ADFTerraformExtensionEnabled: !Equals + - !Ref ADFTerraformExtension + - True Resources: DeploymentFrameworkRegionalS3Bucket: Type: AWS::S3::Bucket @@ -90,6 +97,7 @@ Resources: StringEquals: aws:PrincipalOrgID: !Ref OrganizationId TerraformLockTable: + Condition: ADFTerraformExtensionEnabled Type: "AWS::DynamoDB::Table" Properties: AttributeDefinitions: From b1e0020a5ea7e0b549ef2d5b76a5639d108b9c07 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:21:11 +0100 Subject: [PATCH 141/179] added example in adfconfig related to Tf extension --- .../initial_commit/bootstrap_repository/example-adfconfig.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml index be25be2d5..b8ae2968c 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml @@ -18,3 +18,7 @@ config: scm: # Source control management auto-create-repositories: enabled # if true and using codecommit as source the repository will be automatically created default-scm-branch: master # allow to specify custom default branch if a pipeline specific configuration is not provided. If the parameter is not specified it defaults to the "master" branch. + +extensions: + terraform: + enabled: False # if true resources needed to run terraform template will be deployed \ No newline at end of file From 5eccd06418523aed2531a3eb4e2c62b6aba261e4 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:24:54 +0100 Subject: [PATCH 142/179] added details related to terraform extension --- docs/user-guide.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/user-guide.md b/docs/user-guide.md index ca44d0347..9b4684b25 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -26,6 +26,10 @@ - [Using Anchors and Alias](#using-anchors-and-alias) - [One to many relationships](#one-to-many-relationships) - [Terraform pipeline](#terraform-pipeline) + - [Prerequisites](#prerequisites) + - [Overview](#overview) + - [Parameters](#parameters) + - [Deployment procedure](#deployment-procedure) ## Deployment Map @@ -754,6 +758,7 @@ By passing in the Repository name *(repository)* we are overriding the **name** #### Prerequisites To enable ADF Terraform extension the following steps are required: +- Enable ADF Terraform extension. Set the parameter extensions > terraform > enabled to True to deploy all the necessary resources - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/` and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/deployment` (please note: `deployment` at the end) and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. From 6e2eb5cf4e5a71ac025e2cd04366e12f96a41c97 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:27:28 +0100 Subject: [PATCH 143/179] added details related to terraform extension --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 9b4684b25..abf38d826 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -758,7 +758,7 @@ By passing in the Repository name *(repository)* we are overriding the **name** #### Prerequisites To enable ADF Terraform extension the following steps are required: -- Enable ADF Terraform extension. Set the parameter extensions > terraform > enabled to True to deploy all the necessary resources +- Enable ADF Terraform extension. Set the parameter extensions > terraform > enabled to True in adfconfig.yml file, as shown in example-adfconfig.yml, to deploy all the necessary resources - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/` and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. - Rename file `example-global-iam.yml` to `global-iam.yml` in the following path `aws-deployment-framework-bootstrap/adf-bootstrap/deployment` (please note: `deployment` at the end) and ensure the CloudFormation resources `ADFTerraformRole` and `ADFTerraformPolicy` are no longer commented out. From 9b6debbffd62be047a14a968b4287f924cc5327f Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 24 Dec 2021 08:37:51 +0100 Subject: [PATCH 144/179] fixed indentation --- .../initial_commit/bootstrap_repository/adf-build/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index 74608c0e9..d18842a1f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -170,8 +170,8 @@ def prepare_deployment_account(sts, deployment_account_id, config): ) if hasattr(config, 'extensions'): for extension, attributes in config.extensions.items(): - for attribute in attributes: - deployment_account_parameter_store.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) + for attribute in attributes: + deployment_account_parameter_store.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) return deployment_account_role From 18fc837c3e5327881908d2929e647b9765c95b45 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Mon, 3 Jan 2022 22:29:51 +0100 Subject: [PATCH 145/179] renamed paginator variable --- .../adf-build/shared/helpers/terraform/get_accounts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 8b8adbace..3c6c14a51 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -121,8 +121,8 @@ def get_boto3_client(service, role, session_name): def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> list: account_list = [] # Get OUs - paginator = org_client.get_paginator('list_children') - pages = paginator.paginate( + paginator_item = org_client.get_paginator('list_children') + pages = paginator_item.paginate( ParentId=ou_id, ChildType='ORGANIZATIONAL_UNIT' ) @@ -131,7 +131,7 @@ def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> li account_list.extend(get_account_recursive(org_client, child['Id'], f"{path}{ou_id}/")) # Get Accounts - pages = paginator.paginate( + pages = paginator_item.paginate( ParentId=ou_id, ChildType='ACCOUNT' ) From 2f2f38bc50c9b59dcf94c89611bdd305598666ae Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Mon, 7 Mar 2022 14:20:27 +0100 Subject: [PATCH 146/179] Resolve W1514 - use open with encoding --- .../adf-build/shared/helpers/terraform/get_accounts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 3c6c14a51..c9fa282b4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -29,12 +29,12 @@ def main(): accounts = get_accounts() - with open('accounts.json', 'w') as outfile: + with open('accounts.json', 'w', encoding='utf-8') as outfile: json.dump(accounts, outfile) if TARGET_OUS: accounts_from_ous = get_accounts_from_ous() - with open('accounts_from_ous.json', 'w') as outfile: + with open('accounts_from_ous.json', 'w', encoding='utf-8') as outfile: json.dump(accounts_from_ous, outfile) def list_organizational_units_for_parent(parent_ou): From f6c436c071a307c95d7e052e82fb601f313e769b Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Mon, 7 Mar 2022 14:25:14 +0100 Subject: [PATCH 147/179] Fix user guide target via tags --- docs/user-guide.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 852729fdd..82638ceda 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1,7 +1,7 @@ # User Guide - [Deployment Map](#deployment-map) - - [Targeting via Tags](#targeting-via-tags) + - [Targeting via Tags](#targeting-via-tags) - [Important Notes](#important-notes) - [Zero-prefixed AWS Account Ids](#zero-prefixed-aws-account-ids) - [Providers](#providers) @@ -88,7 +88,7 @@ The second pipeline (*vpc*) example deploys to an OU path `/banking/testing`. Yo By default, the above pipelines will be created to deploy CloudFormation using a change in two actions *(Create then Execute)*. -#### Targeting via Tags +### Targeting via Tags Tags on AWS Accounts can also be used to define stages within a pipeline. For example, we might want to create a pipeline that targets all AWS Accounts with the tag `cost-center` and value of `foo-team`. You cannot use a combination of `path/target` and `tags`. From 975c754383fe041276842926769ce4131901789c Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 27 May 2022 17:32:09 +0200 Subject: [PATCH 148/179] changed default codebuild image to STANDARD_5_0 --- .../adf-build/shared/cdk/cdk_constructs/adf_codebuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py index ea847c6e0..b6b314754 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py @@ -18,7 +18,7 @@ ADF_DEPLOYMENT_REGION = os.environ["AWS_REGION"] ADF_DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] -DEFAULT_CODEBUILD_IMAGE = "UBUNTU_14_04_PYTHON_3_7_1" +DEFAULT_CODEBUILD_IMAGE = "STANDARD_5_0" DEFAULT_BUILD_SPEC_FILENAME = 'buildspec.yml' DEFAULT_DEPLOY_SPEC_FILENAME = 'deployspec.yml' From 3eb4732d82f7d7d2cbf69262b6d8b041b445c7ff Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Fri, 16 Sep 2022 16:56:42 +0200 Subject: [PATCH 149/179] Fix YAML lint issues --- .../bootstrap_repository/adf-bootstrap/deployment/regional.yml | 2 +- .../initial_commit/bootstrap_repository/example-adfconfig.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml index 41fe4e906..faa105f6a 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml @@ -17,7 +17,7 @@ Conditions: ADFTerraformExtensionEnabled: Fn::Equals: - !Ref ADFTerraformExtension - - True + - true Resources: DeploymentFrameworkRegionalS3Bucket: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml index 06cf6258c..0b48e654b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/example-adfconfig.yml @@ -31,4 +31,4 @@ config: extensions: terraform: - enabled: False # If true resources needed to run Terraform template will be deployed + enabled: false # If true resources needed to run Terraform template will be deployed From 836bdb0376807992fd5f96570abbf8851975af34 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Fri, 16 Sep 2022 17:14:42 +0200 Subject: [PATCH 150/179] Fix CFN lint issues --- samples/sample-terraform/params/global.yml | 2 +- .../bootstrap_repository/adf-bootstrap/deployment/global.yml | 2 +- .../bootstrap_repository/adf-bootstrap/example-global-iam.yml | 4 ++++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/samples/sample-terraform/params/global.yml b/samples/sample-terraform/params/global.yml index 1aab20ee4..3243d2830 100644 --- a/samples/sample-terraform/params/global.yml +++ b/samples/sample-terraform/params/global.yml @@ -1,2 +1,2 @@ Parameters: - ProjectName: "sample-terraform" # $ADF_PROJECT_NAME + ProjectName: "sample-terraform" diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index 342c530ac..b0f4431f3 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -275,7 +275,7 @@ Resources: - dynamodb:DeleteItem - dynamodb:DescribeTable Resource: - - !Sub "arn:aws:dynamodb:*:${AWS::AccountId}:table/adf-tflocktable*" + - !Sub "arn:${AWS::Partition}:dynamodb:*:${AWS::AccountId}:table/adf-tflocktable*" - Effect: Allow Sid: "S3" Action: diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml index 44bab7010..316a5c9c5 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/example-global-iam.yml @@ -128,3 +128,7 @@ Resources: # - "*" # Roles: # - !Ref MyExampleCustomRole + +Outputs: + DeploymentAccountId: + Value: !Ref DeploymentAccountId From 0aa1f83d2b47463976dfdf32df030a150c653149 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Fri, 16 Sep 2022 17:26:22 +0200 Subject: [PATCH 151/179] Fix MegaLint issues --- docs/user-guide.md | 7 +++--- samples/sample-terraform/README.md | 2 +- samples/sample-terraform/tf_apply.yml | 5 ++--- samples/sample-terraform/tf_plan.yml | 4 ++-- .../shared/helpers/terraform/adf_terraform.sh | 22 +++++++++---------- 5 files changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index efb983a01..ffbd96213 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1039,6 +1039,7 @@ the `111111111111` AWS Account. #### Prerequisites To enable ADF Terraform extension the following steps are required: + - Enable ADF Terraform extension. Set the parameter `extensions > terraform > enabled` to `True` in the `adfconfig.yml` file, as shown in the `example-adfconfig.yml`, to deploy all the necessary @@ -1096,8 +1097,7 @@ pipeline for more details in the setup and integration. #### Deployment procedure -1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as in the - example: +Example Terraform deployment map: ```yaml - name: sample-terraform @@ -1132,6 +1132,7 @@ pipeline for more details in the setup and integration. spec_filename: tf_apply.yml # Terraform apply ``` +1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as shown above. 2. Add the project name in `params/global.yml` file. 3. Add Terraform code to the `tf` folder. **Please note**: Do not make changes to `backend.tf` file and `main.tf` in the root folder of the sample. @@ -1144,7 +1145,7 @@ pipeline for more details in the setup and integration. - Local variables (per account) can be configured using the following naming convention - ``` + ```txt tfvars <-- This folder contains the structure to define Terraform │ variables │ diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index 412798abe..bd9fb116a 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -3,7 +3,7 @@ ## Overview Please read the[user guide on ADF's support for -Terraform](../docs/user-guide.md#terraform-pipeline) before you proceed. +Terraform](../../docs/user-guide.md#terraform-pipeline) before you proceed. ## Deployment procedure diff --git a/samples/sample-terraform/tf_apply.yml b/samples/sample-terraform/tf_apply.yml index fe89d4b70..cd9fe1ef8 100644 --- a/samples/sample-terraform/tf_apply.yml +++ b/samples/sample-terraform/tf_apply.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources + TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role Terraform will assume to deploy resources TF_IN_AUTOMATION: true TF_CLI_ARGS: "-no-color" TF_STAGE: "apply" @@ -10,9 +10,8 @@ env: phases: install: runtime-versions: - python: 3.8 + python: 3.9 build: commands: - python adf-build/helpers/terraform/get_accounts.py - bash adf-build/helpers/terraform/adf_terraform.sh - diff --git a/samples/sample-terraform/tf_plan.yml b/samples/sample-terraform/tf_plan.yml index a4acfecf2..a6eb85e59 100644 --- a/samples/sample-terraform/tf_plan.yml +++ b/samples/sample-terraform/tf_plan.yml @@ -2,7 +2,7 @@ version: 0.2 env: variables: - TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role terraform will assume to deploy resources + TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role Terraform will assume to deploy resources TF_IN_AUTOMATION: true TF_STAGE: "plan" TF_CLI_ARGS: "-no-color" @@ -10,7 +10,7 @@ env: phases: install: runtime-versions: - python: 3.8 + python: 3.9 build: commands: - python adf-build/helpers/terraform/get_accounts.py diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 59e15ce57..6547a5419 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -2,7 +2,7 @@ PATH=$PATH:$(pwd) export PATH CURRENT=$(pwd) -terraform --version +terraform --version echo "Terraform stage: $TF_STAGE" tfinit(){ @@ -21,17 +21,17 @@ tfinit(){ -backend-config "region=$AWS_REGION" \ -backend-config "key=$ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" \ -backend-config "dynamodb_table=adf-tflocktable" - + echo "Bucket: $S3_BUCKET_REGION_NAME" echo "Region: $AWS_REGION" echo "Key: $ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" echo "DynamoDB table: adf-tflocktable" } -tfplan(){ +tfplan(){ DATE=$(date +%Y-%m-%d) TS=$(date +%Y%m%d%H%M%S) bash "$CURRENT/adf-build/helpers/sts.sh" "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" - terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" + terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" # Save Terraform plan results to the S3 bucket aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" @@ -60,7 +60,7 @@ tfrun(){ tfplan tfapply set +e - else + else echo "Invalid Terraform stage: TF_STAGE = $TF_STAGE" exit 1 fi @@ -73,14 +73,14 @@ then fi echo "List of target regions: $REGIONS" for REGION in $(echo "$REGIONS" | sed "s/,/ /g") -do +do AWS_REGION=$(echo -n "$REGION" | sed 's/^[ \t]*//;s/[ \t]*$//') # sed trims whitespaces export TF_VAR_TARGET_REGION=$AWS_REGION # if TARGET_ACCOUNTS and TARGET_OUS are not defined apply to all accounts if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]] then - echo "Apply to all accounts" - for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts.json" | sed 's/"//g' ) + echo "Apply to all accounts" + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts.json" | sed 's/"//g' ) do tfrun done @@ -91,15 +91,15 @@ do # apply only on a subset of accounts (TARGET_ACCOUNTS) echo "List of target account: $TARGET_ACCOUNTS" for ACCOUNT_ID in $(echo "$TARGET_ACCOUNTS" | sed "s/,/ /g") - do + do tfrun done fi if ! [[ -z "$TARGET_OUS" ]] then - echo "List target OUs: $TARGET_OUS" - for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g' ) + echo "List target OUs: $TARGET_OUS" + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g' ) do tfrun done From 43e1cfdd735d3f7ba1c0635ef47e216c29eca84c Mon Sep 17 00:00:00 2001 From: Stewart Wallace Date: Tue, 20 Sep 2022 11:38:14 +0100 Subject: [PATCH 152/179] Adding in default values for extensions --- .../bootstrap_repository/adf-build/config.py | 96 +++++++++---------- .../adf-build/tests/test_config.py | 51 +++++++--- 2 files changed, 86 insertions(+), 61 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py index d2e1dc22f..27a4bfa8f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py @@ -15,6 +15,7 @@ ADF_VERSION = os.environ["ADF_VERSION"] LOGGER = configure_logger(__name__) +AVAILABLE_EXTENSIONS = ["terraform"] class Config: @@ -24,10 +25,9 @@ class Config: def __init__(self, parameter_store=None, config_path=None): self.parameters_client = parameter_store or ParameterStore( - os.environ["AWS_REGION"], - boto3 + os.environ["AWS_REGION"], boto3 ) - self.config_path = config_path or './adfconfig.yml' + self.config_path = config_path or "./adfconfig.yml" self.organization_id = os.environ["ORGANIZATION_ID"] self.client_deployment_region = None self.notification_type = None @@ -51,34 +51,35 @@ def _validate(self): Validates the adfconfig.yml file """ if None in ( - self.cross_account_access_role, - self.config, - self.deployment_account_region, - self.organization_id, - self.target_regions, - self.config.get('moves'), - self.config.get('main-notification-endpoint') + self.cross_account_access_role, + self.config, + self.deployment_account_region, + self.organization_id, + self.target_regions, + self.config.get("moves"), + self.config.get("main-notification-endpoint"), ): raise InvalidConfigError( - 'adfconfig.yml is missing required properties. ' - 'Please see the documentation.' + "adfconfig.yml is missing required properties. " + "Please see the documentation." ) from None try: - if self.config.get('scp'): - assert self.config.get('scp').get( - 'keep-default-scp') in ['enabled', 'disabled'] + if self.config.get("scp"): + assert self.config.get("scp").get("keep-default-scp") in [ + "enabled", + "disabled", + ] except AssertionError: raise InvalidConfigError( - 'Configuration settings for organizations should be either ' - 'enabled or disabled' + "Configuration settings for organizations should be either " + "enabled or disabled" ) from None if isinstance(self.deployment_account_region, list): if len(self.deployment_account_region) > 1: raise InvalidConfigError( - 'ADF currently only supports a single ' - 'Deployment Account region' + "ADF currently only supports a single " "Deployment Account region" ) from None [self.deployment_account_region] = self.deployment_account_region @@ -89,7 +90,7 @@ def _load_config_file(self): """ Loads the adfconfig.yml file and executes _parse_config """ - with open(self.config_path, encoding='utf-8') as config: + with open(self.config_path, encoding="utf-8") as config: self.config_contents = yaml.load(config, Loader=yaml.FullLoader) self._parse_config() @@ -97,40 +98,40 @@ def _parse_config(self): """ Parses the adfconfig.yml file and executes _validate """ - regions = self.config_contents.get('regions', {}).get('targets', []) - self.deployment_account_region = ( - self.config_contents.get('regions', {}).get('deployment-account') - ) - self.target_regions = ( - [] if regions[0] is None - else regions + regions = self.config_contents.get("regions", {}).get("targets", []) + self.deployment_account_region = self.config_contents.get("regions", {}).get( + "deployment-account" ) - self.cross_account_access_role = ( - self.config_contents.get('roles', {}).get('cross-account-access') + self.target_regions = [] if regions[0] is None else regions + self.cross_account_access_role = self.config_contents.get("roles", {}).get( + "cross-account-access" ) - self.config = self.config_contents.get('config') - self.protected = self.config.get('protected', []) + self.config = self.config_contents.get("config") + self.protected = self.config.get("protected", []) # TODO Investigate why this only considers the first notification # endpoint. Seems like a bug, it should support multiple. - adf_config_notification_type = ( - self.config.get('main-notification-endpoint')[0].get('type') - ) + adf_config_notification_type = self.config.get("main-notification-endpoint")[ + 0 + ].get("type") self.notification_type = ( - 'lambda' if adf_config_notification_type == 'slack' - else 'email' - ) - self.notification_endpoint = ( - self.config.get('main-notification-endpoint')[0].get('target') + "lambda" if adf_config_notification_type == "slack" else "email" ) + self.notification_endpoint = self.config.get("main-notification-endpoint")[ + 0 + ].get("target") self.notification_channel = ( - None if self.notification_type == 'email' - else self.notification_endpoint + None if self.notification_type == "email" else self.notification_endpoint ) - self.extensions = self.config_contents.get('extensions', None) + self.extensions = self.config_contents.get("extensions", dict()) + self._configure_default_extensions_behavior() self._validate() + def _configure_default_extensions_behavior(self): + for unconfigured_extension in AVAILABLE_EXTENSIONS - self.extensions.keys(): + self.extensions[unconfigured_extension] = {"enabled": False} + def _store_cross_region_config(self): """ Stores cross_account_access_role Parameter @@ -138,16 +139,11 @@ def _store_cross_region_config(self): in deployment account main region. """ self.client_deployment_region = ParameterStore( - self.deployment_account_region, - boto3 - ) - self.client_deployment_region.put_parameter( - 'adf_version', - ADF_VERSION + self.deployment_account_region, boto3 ) + self.client_deployment_region.put_parameter("adf_version", ADF_VERSION) self.client_deployment_region.put_parameter( - 'cross_account_access_role', - self.cross_account_access_role + "cross_account_access_role", self.cross_account_access_role ) def _store_config(self): diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py index 80da0b8ae..4b4dfa0d7 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py @@ -4,6 +4,7 @@ # pylint: skip-file import os +from pickle import TRUE from errors import InvalidConfigError from pytest import fixture, raises @@ -16,16 +17,18 @@ def cls(): parameter_store = Mock() return Config( parameter_store=parameter_store, - config_path='{0}/stubs/stub_adfconfig.yml'.format( + config_path="{0}/stubs/stub_adfconfig.yml".format( os.path.dirname(os.path.realpath(__file__)) - ) + ), ) def test_validation(cls): - assert cls.config.get( - 'main-notification-endpoint')[0].get('target') == 'john@example.com' - assert cls.config.get('moves')[0].get('action') == 'remove-base' + assert ( + cls.config.get("main-notification-endpoint")[0].get("target") + == "john@example.com" + ) + assert cls.config.get("moves")[0].get("action") == "remove-base" def test_validation_list_deployment_target(cls): @@ -41,33 +44,59 @@ def test_validation_list_deployment_account_target(cls): def test_raise_validation_remove_moves(cls): - cls.config_contents.get('config').pop('moves', None) + cls.config_contents.get("config").pop("moves", None) with raises(InvalidConfigError): assert cls._parse_config() def test_raise_validation_remove_roles(cls): - cls.config_contents.get('roles', None).pop('cross-account-access', None) + cls.config_contents.get("roles", None).pop("cross-account-access", None) with raises(InvalidConfigError): assert cls._parse_config() def test_raise_validation_remove_deployment_target_region(cls): - cls.config_contents.get('regions', None).pop('deployment-account', None) + cls.config_contents.get("regions", None).pop("deployment-account", None) with raises(InvalidConfigError): assert cls._parse_config() def test_raise_validation_length_deployment_target_region(cls): cls.config_contents["regions"]["deployment-account"] = [ - 'region1', - 'region2', + "region1", + "region2", ] with raises(InvalidConfigError): assert cls._parse_config() def test_raise_validation_organizations_scp(cls): - cls.config_contents['config']['scp']['keep-default-scp'] = 'blah' + cls.config_contents["config"]["scp"]["keep-default-scp"] = "blah" with raises(InvalidConfigError): assert cls._parse_config() + + +def test_extensions_default_configuration(cls): + cls._parse_config() + assert cls.extensions == {"terraform": {"enabled": False}} + + +def test_extensions_default_configuration_with_custom(cls): + cls.config_contents["extensions"] = {"another_extensions": {"enabled": True}} + cls._parse_config() + assert cls.extensions == { + "another_extensions": {"enabled": True}, + "terraform": {"enabled": False}, + } + + +def test_extensions_default_configuration_wont_override_custom(cls): + cls.config_contents["extensions"] = { + "another_extensions": {"enabled": True}, + "terraform": {"enabled": True}, + } + cls._parse_config() + assert cls.extensions == { + "another_extensions": {"enabled": True}, + "terraform": {"enabled": True}, + } From 74b8049cb9517f3d1262a349f242c53c3b4bf9eb Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Thu, 3 Nov 2022 14:58:21 +0100 Subject: [PATCH 153/179] Remove redundant paginator This is already available at: src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/python/paginator.py --- .../shared/helpers/terraform/paginator.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py deleted file mode 100644 index e54b6db5c..000000000 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/paginator.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: MIT-0 - -""" -Paginator used with certain boto3 calls -when pagination is required -""" - -def paginator(method, **kwargs): - client = method.__self__ - iterator = client.get_paginator(method.__name__) - for page in iterator.paginate(**kwargs).result_key_iters(): - for result in page: - yield result From 0e433a88015d3a765287b1386944aaf277f43546 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Thu, 3 Nov 2022 15:02:35 +0100 Subject: [PATCH 154/179] Fix TF doc comments and links --- docs/user-guide.md | 18 +++++++++--------- samples/sample-terraform/README.md | 12 ++++++------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index ffbd96213..bb57b89a9 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -77,7 +77,7 @@ pipelines: params: notification_endpoint: janes_team@example.com # Optional tags: - foo: bar # Pipelines support tagging + foo: bar # Pipelines support tagging targets: - path: /security regions: eu-west-1 @@ -105,7 +105,7 @@ pipelines: notification_endpoint: joes_team@example.com targets: - path: /banking/testing - name: fancy-name #Optional way to pass a name for this stage in the pipeline + name: fancy-name # Optional way to pass a name for this stage in the pipeline ``` In the above example we are creating two pipelines with AWS CodePipeline. The @@ -251,7 +251,7 @@ targets: targets: - target: 9999999999 # Target and Path keys can be used interchangeably regions: eu-west-1 - name: my-special-account # Defaults to adf-cloudformation-deployment-role + name: my-special-account # Defaults to adf-cloudformation-deployment-role # If you intend to override the provider for this stage # (see providers guide for available providers) provider: some_provider @@ -364,7 +364,7 @@ pipelines: triggers: on_complete: pipelines: - - my-web-app-pipeline # Start this pipeline + - my-web-app-pipeline # Start this pipeline - name: my-web-app-pipeline default_providers: @@ -395,7 +395,7 @@ pipelines: # and what should be triggered when it completes completion_triggers: pipelines: - - my-web-app-pipeline # Start this pipeline + - my-web-app-pipeline # Start this pipeline - name: my-web-app-pipeline # Same configuration as defined above. @@ -545,7 +545,7 @@ pipelines: provider: codebuild image: repository_arn: arn:aws:ecr:region:111111111111:repository/test - tag: latest # optional (defaults to latest) + tag: latest # optional (defaults to latest) targets: - # ... ``` @@ -912,7 +912,7 @@ main `template.yml` in our like so: MyStack: Type: "AWS::CloudFormation::Stack" Properties: - TemplateURL: another_template.yml # file path to the nested stack template + TemplateURL: another_template.yml # file path to the nested stack template ``` When the `package_transform.sh` command is executed, the file will be packaged @@ -1120,13 +1120,13 @@ Example Terraform deployment map: # For example, "eu-west-1,us-east-1". REGIONS: eu-west-1 targets: - - name: terraform-scan # optional + - name: terraform-scan # optional properties: spec_filename: tf_scan.yml # Terraform scan - name: terraform-plan properties: spec_filename: tf_plan.yml # Terraform plan - - approval # manual approval + - approval # manual approval - name: terraform-apply properties: spec_filename: tf_apply.yml # Terraform apply diff --git a/samples/sample-terraform/README.md b/samples/sample-terraform/README.md index bd9fb116a..1a02bdff0 100644 --- a/samples/sample-terraform/README.md +++ b/samples/sample-terraform/README.md @@ -2,7 +2,7 @@ ## Overview -Please read the[user guide on ADF's support for +Please read the [user guide on ADF's support for Terraform](../../docs/user-guide.md#terraform-pipeline) before you proceed. ## Deployment procedure @@ -31,13 +31,13 @@ example: # "eu-west-1,us-east-1" REGIONS: eu-west-1 targets: - - name: terraform-scan # optional + - name: terraform-scan # optional properties: spec_filename: tf_scan.yml # Terraform scan - name: terraform-plan properties: spec_filename: tf_plan.yml # Terraform plan - - approval # manual approval + - approval # manual approval - name: terraform-apply properties: spec_filename: tf_apply.yml # Terraform apply @@ -45,6 +45,6 @@ example: The sample uses the following configuration, please update accordingly: -- Project name: sample-tf-module -- Target accounts: 111111111111, 222222222222 -- Target regions: eu-west-1 (main ADF region), us-east-1 +- Project name: `sample-tf-module` +- Target accounts: `111111111111` and `222222222222` +- Target regions: `eu-west-1` (the main ADF deployment region) and `us-east-1` From 81a34659325c95d898c83dbdcca7a9421cd5361f Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Thu, 3 Nov 2022 15:05:47 +0100 Subject: [PATCH 155/179] Fix TF line length findings --- samples/sample-terraform/tf/main.tf | 2 +- .../bootstrap_repository/adf-build/config.py | 3 +- .../shared/helpers/terraform/get_accounts.py | 65 +++++++++++++++---- .../adf-build/tests/test_config.py | 6 +- 4 files changed, 62 insertions(+), 14 deletions(-) diff --git a/samples/sample-terraform/tf/main.tf b/samples/sample-terraform/tf/main.tf index 2886e3dba..dc4c6d66b 100644 --- a/samples/sample-terraform/tf/main.tf +++ b/samples/sample-terraform/tf/main.tf @@ -11,6 +11,6 @@ terraform { } provider "aws" { assume_role { - role_arn = "arn:${data.aws_partition.current}:iam::${var.TARGET_ACCOUNT_ID}:role/${var.TARGET_ACCOUNT_ROLE}" + role_arn = "arn:${data.aws_partition.current}:iam::${var.TARGET_ACCOUNT_ID}:role/${var.TARGET_ACCOUNT_ROLE}" } } diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py index 287357625..3346850a9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/config.py @@ -79,7 +79,8 @@ def _validate(self): if isinstance(self.deployment_account_region, list): if len(self.deployment_account_region) > 1: raise InvalidConfigError( - "ADF currently only supports a single " "Deployment Account region" + "ADF currently only supports a single " + "Deployment Account region" ) from None [self.deployment_account_region] = self.deployment_account_region diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index c9fa282b4..2e7b02bd4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -37,11 +37,23 @@ def main(): with open('accounts_from_ous.json', 'w', encoding='utf-8') as outfile: json.dump(accounts_from_ous, outfile) + def list_organizational_units_for_parent(parent_ou): - organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getOrganizationUnits') + organizations = get_boto3_client( + 'organizations', + ( + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' + ), + 'getOrganizationUnits', + ) organizational_units = [ ou - for org_units in organizations.get_paginator("list_organizational_units_for_parent").paginate(ParentId=parent_ou) + for org_units in ( + organizations + .get_paginator("list_organizational_units_for_parent") + .paginate(ParentId=parent_ou) + ) for ou in org_units['OrganizationalUnits'] ] return organizational_units @@ -53,11 +65,22 @@ def get_accounts(): "Management Account ID: %s", MANAGEMENT_ACCOUNT_ID ) - # Assume a role into the management accounts role to get account ID's and emails - organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getaccountIDs') + # Assume a role into the management accounts role to get account ID's + # and emails + organizations = get_boto3_client( + 'organizations', + ( + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' + ), + 'getaccountIDs', + ) return list( map( - lambda account: {'AccountId': account['Id'], 'Email': account['Email']}, + lambda account: { + 'AccountId': account['Id'], + 'Email': account['Email'], + }, filter( lambda account: account['Status'] == 'ACTIVE', paginator(organizations.list_accounts) @@ -69,7 +92,14 @@ def get_accounts(): def get_accounts_from_ous(): parent_ou_id = None account_list = [] - organizations = get_boto3_client('organizations', f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getRootAccountIDs') + organizations = get_boto3_client( + 'organizations', + ( + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', + ), + 'getRootAccountIDs', + ) # Read organization root id root_ids = list( map( @@ -87,7 +117,9 @@ def get_accounts_from_ous(): ou_hierarchy = path.strip('/').split('/') hierarchy_index = 0 if path.strip() == '/': - account_list.extend(get_account_recursive(organizations, parent_ou_id, '/')) + account_list.extend( + get_account_recursive(organizations, parent_ou_id, '/') + ) else: while hierarchy_index < len(ou_hierarchy): org_units = list_organizational_units_for_parent(parent_ou_id) @@ -97,10 +129,15 @@ def get_accounts_from_ous(): hierarchy_index += 1 break else: - raise ValueError(f'Could not find ou with name {ou_hierarchy} in OU list {org_units}.') + raise ValueError( + f'Could not find ou with name {ou_hierarchy} in ' + f'OU list {org_units}.' + ) - account_list.extend(get_account_recursive(organizations, parent_ou_id, '/')) - parent_ou_id=None + account_list.extend( + get_account_recursive(organizations, parent_ou_id, '/'), + ) + parent_ou_id = None return account_list @@ -128,7 +165,13 @@ def get_account_recursive(org_client: boto3.client, ou_id: str, path: str) -> li ) for page in pages: for child in page['Children']: - account_list.extend(get_account_recursive(org_client, child['Id'], f"{path}{ou_id}/")) + account_list.extend( + get_account_recursive( + org_client, + child['Id'], + f"{path}{ou_id}/", + ) + ) # Get Accounts pages = paginator_item.paginate( diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py index c779c8058..11f6f19e8 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/tests/test_config.py @@ -81,7 +81,11 @@ def test_extensions_default_configuration(cls): def test_extensions_default_configuration_with_custom(cls): - cls.config_contents["extensions"] = {"another_extensions": {"enabled": True}} + cls.config_contents["extensions"] = { + "another_extensions": { + "enabled": True, + }, + } cls._parse_config() assert cls.extensions == { "another_extensions": {"enabled": True}, From a5694a4215ec593bbf88d9db29257ccfa0f4acd2 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Thu, 3 Nov 2022 15:06:19 +0100 Subject: [PATCH 156/179] Revert default CodeBuild container, would introduce a breaking change The default CodeBuild container image to use cannot be changed without introducing a breaking change. A breaking change would imply a major version release. Since we are adding TF support in v3.2.0, a minor version release, we cannot modify this yet. It is on the roadmap though for the next major release :). --- .../adf-build/shared/cdk/cdk_constructs/adf_codebuild.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py index a7a87879d..d6b82a15f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/cdk/cdk_constructs/adf_codebuild.py @@ -19,7 +19,7 @@ ADF_DEPLOYMENT_REGION = os.environ["AWS_REGION"] ADF_DEPLOYMENT_ACCOUNT_ID = os.environ["ACCOUNT_ID"] -DEFAULT_CODEBUILD_IMAGE = "STANDARD_5_0" +DEFAULT_CODEBUILD_IMAGE = "UBUNTU_14_04_PYTHON_3_7_1" DEFAULT_BUILD_SPEC_FILENAME = 'buildspec.yml' DEFAULT_DEPLOY_SPEC_FILENAME = 'deployspec.yml' From 3be75e37c217a1749e5d0741625be7b4deb6eedc Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Thu, 3 Nov 2022 15:08:54 +0100 Subject: [PATCH 157/179] Replace redundant code writing extension parameters --- .../bootstrap_repository/adf-build/main.py | 22 ++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py index 72506cda6..07c4475df 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/main.py @@ -122,10 +122,7 @@ def prepare_deployment_account(sts, deployment_account_id, config): deployment_account_parameter_store.put_parameter( 'organization_id', os.environ["ORGANIZATION_ID"] ) - if hasattr(config, 'extensions'): - for extension, attributes in config.extensions.items(): - for attribute in attributes: - deployment_account_parameter_store.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) + _store_extension_parameters(deployment_account_parameter_store, config) deployment_account_parameter_store = ParameterStore( config.deployment_account_region, @@ -170,14 +167,23 @@ def prepare_deployment_account(sts, deployment_account_id, config): '/notification_endpoint/main' if item == 'notification_channel' else item, str(getattr(config, item)) ) - if hasattr(config, 'extensions'): - for extension, attributes in config.extensions.items(): - for attribute in attributes: - deployment_account_parameter_store.put_parameter(f"/adf/extensions/{extension}/{attribute}", str(attributes[attribute])) + _store_extension_parameters(deployment_account_parameter_store, config) return deployment_account_role +def _store_extension_parameters(parameter_store, config): + if not hasattr(config, 'extensions'): + return + + for extension, attributes in config.extensions.items(): + for attribute in attributes: + parameter_store.put_parameter( + f"/adf/extensions/{extension}/{attribute}", + str(attributes[attribute]), + ) + + def worker_thread( account_id, sts, From ee1a0f8187bcc4d118c9b733c2c10c037bf8e998 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 13:53:57 +0100 Subject: [PATCH 158/179] fixed bash syntax --- .../shared/helpers/terraform/adf_terraform.sh | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 6547a5419..32c2d2a7b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -8,14 +8,16 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) - mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit - cp -R "$CURRENT/tf/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + mkdir -p "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" + cd "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" || exit + cp -R "$CURRENT"/tf/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" # if account related variables exist copy the folder in the work directory - if [ -d "$CURRENT/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then - cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + if [ -d "$CURRENT"/tfvars/"$TF_VAR_TARGET_ACCOUNT_ID" ]; then + cp -R "${CURRENT}"/tfvars/"${TF_VAR_TARGET_ACCOUNT_ID}"/* "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" + fi + if [ -f "$CURRENT"/tfvars/global.auto.tfvars ]; then + cp -R "${CURRENT}"/tfvars/global.auto.tfvars "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" fi - cp -R "${CURRENT}/tfvars/global.auto.tfvars" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" terraform init \ -backend-config "bucket=$S3_BUCKET_REGION_NAME" \ -backend-config "region=$AWS_REGION" \ @@ -30,14 +32,14 @@ tfinit(){ tfplan(){ DATE=$(date +%Y-%m-%d) TS=$(date +%Y%m%d%H%M%S) - bash "$CURRENT/adf-build/helpers/sts.sh" "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" - terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" + bash "$CURRENT"/adf-build/helpers/sts.sh "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" + terraform plan -out "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}"-"${TS}".log # Save Terraform plan results to the S3 bucket - aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" + aws s3 cp "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}"-"${TS}".log s3://"${S3_BUCKET_REGION_NAME}"/"${ADF_PROJECT_NAME}"/tf-plan/"${DATE}"/"${TF_VAR_TARGET_ACCOUNT_ID}"/"${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}"-"${TS}".log echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" } tfapply(){ - terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" + terraform apply "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}" } tfrun(){ export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID @@ -99,7 +101,7 @@ do if ! [[ -z "$TARGET_OUS" ]] then echo "List target OUs: $TARGET_OUS" - for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g' ) + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}"/accounts_from_ous.json | sed 's/"//g' ) do tfrun done From 5e95abf1b15cc0615a473389820874fdbbc22398 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 13:54:27 +0100 Subject: [PATCH 159/179] fixed parameter type to string --- .../shared/helpers/terraform/get_accounts.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 2e7b02bd4..d854d8b02 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -41,10 +41,7 @@ def main(): def list_organizational_units_for_parent(parent_ou): organizations = get_boto3_client( 'organizations', - ( - f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' - f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' - ), + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getOrganizationUnits', ) organizational_units = [ @@ -69,10 +66,7 @@ def get_accounts(): # and emails organizations = get_boto3_client( 'organizations', - ( - f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' - f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' - ), + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getaccountIDs', ) return list( @@ -94,10 +88,7 @@ def get_accounts_from_ous(): account_list = [] organizations = get_boto3_client( 'organizations', - ( - f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' - f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', - ), + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', 'getRootAccountIDs', ) # Read organization root id From 87bb7a5d47faf2da920e151d03c6aa6463c1a788 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 14:20:56 +0100 Subject: [PATCH 160/179] merge to main repo --- .../deployment/example-global-iam.yml | 56 ------------------- 1 file changed, 56 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index e1516dcaf..9d635b677 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -28,7 +28,6 @@ Resources: Roles: - adf-cloudformation-deployment-role -<<<<<<< HEAD ## # Begin of Terraform resources ## @@ -142,58 +141,3 @@ Resources: ec2:AuthorizedService: "codebuild.amazonaws.com" Roles: - adf-codebuild-role -======= -## -# Begin of Terraform resources -## -# ADFTerraformRole: -# # This is the role that will be used to deploy Terraform resources from within -# # the target account. You should scope this policy depending on what you would -# # like to deploy within certain Organizational Units. -# # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. -# # Uncomment the `ADFTerraformRole` and `ADFTerraformPolicy` resources if you want to enable Terraform support in ADF -# # and deploy resources in Deployment account using Terraform. If you need to push to other account, you don't need to -# # add this policy -# Type: AWS::IAM::Role -# Properties: -# RoleName: "adf-terraform-role" -# AssumeRolePolicyDocument: -# Version: "2012-10-17" -# Statement: -# - Effect: Allow -# Sid: "AssumeRole" -# Principal: -# AWS: -# - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role -# Action: -# - sts:AssumeRole -# Path: / -# ADFTerraformPolicy: -# Type: AWS::IAM::Policy -# Properties: -# PolicyName: "adf-terraform-policy" -# PolicyDocument: -# Version: "2012-10-17" -# Statement: -# - Effect: Allow -# Action: -# - "events:PutRule" -# - "events:PutTargets" -# - "events:DeleteRule" -# - "events:DescribeRule" -# - "events:ListTagsForResource" -# - "events:ListTargetsByRule" -# - "events:RemoveTargets" -# - "sns:CreateTopic" -# - "sns:GetTopicAttributes" -# - "sns:SetTopicAttributes" -# - "sns:ListTagsForResource" -# - "sns:DeleteTopic" -# Resource: -# - "*" -# Roles: -# - !Ref ADFTerraformRole -## -# End of Terraform resources -## ->>>>>>> 349601ed827dff4c1173f1d9d13dd1e2c8027559 From 14012156c9d59b44292cba83e852e3f534cb774d Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 14:26:18 +0100 Subject: [PATCH 161/179] merge user_guide --- docs/user-guide.md | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 993d09897..3af3fa47b 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -26,19 +26,11 @@ - [Deploying Serverless Applications with SAM](#deploying-serverless-applications-with-sam) - [Using YAML Anchors and Aliases](#using-yaml-anchors-and-aliases) - [One to many relationships](#one-to-many-relationships) -<<<<<<< HEAD - - [Terraform pipeline](#terraform-pipeline) - - [Prerequisites](#prerequisites) - - [Overview](#overview) - - [Parameters](#parameters) - - [Deployment procedure](#deployment-procedure) -======= - [Terraform pipeline](#terraform-pipeline) - [Prerequisites](#prerequisites) - [Overview](#overview) - [Parameters](#parameters) - [Deployment procedure](#deployment-procedure) ->>>>>>> 349601ed827dff4c1173f1d9d13dd1e2c8027559 ## Deployment Map @@ -1188,15 +1180,6 @@ e.g. The following state files are created: - 111111111111 main region (eu-west-1) -<<<<<<< HEAD - -> adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/111111111111.tfstate -- 111111111111 secondary region (us-east-1) - -> adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/111111111111.tfstate -- 222222222222 main region (eu-west-1) - -> adf-global-base-deployment-pipelinebucketxyz/sample-tf-module/222222222222.tfstate -- 222222222222 secondary region (us-east-1) - -> adf-regional-base-deploy-deploymentframeworkregio-jsm/sample-tf-module/222222222222.tfstate -======= -> `adf-global-base-deployment-pipeline-bucket-xyz/sample-tf-module/111111111111.tfstate` - 111111111111 secondary region (us-east-1) -> `adf-regional-base-deploy-deployment-framework-region-jsm/sample-tf-module/111111111111.tfstate` @@ -1204,8 +1187,7 @@ The following state files are created: -> `adf-global-base-deployment-pipeline-bucket-xyz/sample-tf-module/222222222222.tfstate` - 222222222222 secondary region (us-east-1) -> `adf-regional-base-deploy-deployment-framework-region-jsm/sample-tf-module/222222222222.tfstate` ->>>>>>> 349601ed827dff4c1173f1d9d13dd1e2c8027559 A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. **Please note**: usage -of this locking table is charged on the deployment account. +of this locking table is charged on the deployment account. \ No newline at end of file From 624c48f82545ac31ec38b704033d6147ae040361 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 14:27:38 +0100 Subject: [PATCH 162/179] merge to master --- .../deployment/example-global-iam.yml | 166 ++++++------------ 1 file changed, 53 insertions(+), 113 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index 9d635b677..75439caa1 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -28,116 +28,56 @@ Resources: Roles: - adf-cloudformation-deployment-role - ## - # Begin of Terraform resources - ## - # ADFTerraformRole: - # # This is the role that will be used to deploy Terraform resources from within - # # the target account. You should scope this policy depending on what you would - # # like to deploy within certain Organizational Units. - # # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. - # # Uncomment the `ADFTerraformRole` and `ADFTerraformPolicy` resources if you want to enable Terraform support in ADF - # # and deploy resources in Deployment account using Terraform. If you need to push to other account, you don't need to - # # add this policy - # Type: AWS::IAM::Role - # Properties: - # RoleName: "adf-terraform-role" - # AssumeRolePolicyDocument: - # Version: "2012-10-17" - # Statement: - # - Effect: Allow - # Sid: "AssumeRole" - # Principal: - # AWS: - # - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role - # Action: - # - sts:AssumeRole - # Path: / - # ADFTerraformPolicy: - # Type: AWS::IAM::Policy - # Properties: - # PolicyName: "adf-terraform-policy" - # PolicyDocument: - # Version: "2012-10-17" - # Statement: - # - Effect: Allow - # Action: - # - "events:PutRule" - # - "events:PutTargets" - # - "events:DeleteRule" - # - "events:DescribeRule" - # - "events:ListTagsForResource" - # - "events:ListTargetsByRule" - # - "events:RemoveTargets" - # - "sns:CreateTopic" - # - "sns:GetTopicAttributes" - # - "sns:SetTopicAttributes" - # - "sns:ListTagsForResource" - # - "sns:DeleteTopic" - # Resource: - # - "*" - # Roles: - # - !Ref ADFTerraformRole - ## - # End of Terraform resources - ## - - ## - # Begin of VPC CodeBuild support IAM permissions - ## - PipelineProvisionerResourcePolicy: - Type: AWS::IAM::Policy - Properties: - PolicyName: "adf-pipeline-provisioner-codebuild-role-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Sid: "CodeBuildVPC" - Action: - - "ec2:AuthorizeSecurityGroupEgress" - - "ec2:AuthorizeSecurityGroupIngress" - - "ec2:CreateSecurityGroup" - - "ec2:CreateTags" - - "ec2:DeleteSecurityGroup" - - "ec2:DeleteSecurityGroup" - - "ec2:DeleteTags" - - "ec2:Describe*" - - "ec2:List*" - - "ec2:RevokeSecurityGroupEgress" - - "ec2:RevokeSecurityGroupIngress" - Resource: - - "*" - Roles: - - adf-pipeline-provisioner-codebuild-role - - CodeBuildResourcePolicy: - Type: AWS::IAM::Policy - Properties: - PolicyName: "adf-codebuild-role-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: Allow - Sid: "CodeBuildVPC" - Action: - - "ec2:CreateNetworkInterface" - - "ec2:DescribeDhcpOptions" - - "ec2:DescribeNetworkInterfaces" - - "ec2:DeleteNetworkInterface" - - "ec2:DescribeSubnets" - - "ec2:DescribeSecurityGroups" - - "ec2:DescribeVpcs" - Resource: - - "*" - - Effect: Allow - Sid: "CodeBuildENI" - Action: - - "ec2:CreateNetworkInterfacePermission" - Resource: - - "*" - Condition: - StringEquals: - ec2:AuthorizedService: "codebuild.amazonaws.com" - Roles: - - adf-codebuild-role +## +# Begin of Terraform resources +## +# ADFTerraformRole: +# # This is the role that will be used to deploy Terraform resources from within +# # the target account. You should scope this policy depending on what you would +# # like to deploy within certain Organizational Units. +# # NOTE: below is a sample IAM policy. This policies should NOT be used for purposes other than testing. +# # Uncomment the `ADFTerraformRole` and `ADFTerraformPolicy` resources if you want to enable Terraform support in ADF +# # and deploy resources in Deployment account using Terraform. If you need to push to other account, you don't need to +# # add this policy +# Type: AWS::IAM::Role +# Properties: +# RoleName: "adf-terraform-role" +# AssumeRolePolicyDocument: +# Version: "2012-10-17" +# Statement: +# - Effect: Allow +# Sid: "AssumeRole" +# Principal: +# AWS: +# - !Sub arn:aws:iam::${AWS::AccountId}:role/adf-codebuild-role +# Action: +# - sts:AssumeRole +# Path: / +# ADFTerraformPolicy: +# Type: AWS::IAM::Policy +# Properties: +# PolicyName: "adf-terraform-policy" +# PolicyDocument: +# Version: "2012-10-17" +# Statement: +# - Effect: Allow +# Action: +# - "events:PutRule" +# - "events:PutTargets" +# - "events:DeleteRule" +# - "events:DescribeRule" +# - "events:ListTagsForResource" +# - "events:ListTargetsByRule" +# - "events:RemoveTargets" +# - "sns:CreateTopic" +# - "sns:GetTopicAttributes" +# - "sns:SetTopicAttributes" +# - "sns:ListTagsForResource" +# - "sns:DeleteTopic" +# Resource: +# - "*" +# Roles: +# - !Ref ADFTerraformRole +## +# End of Terraform resources +## \ No newline at end of file From 506810daced0509aa709fae94e3b7620cfdb40c1 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 14:29:08 +0100 Subject: [PATCH 163/179] merge to master2 --- .../adf-bootstrap/deployment/example-global-iam.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml index 75439caa1..5620bf404 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/example-global-iam.yml @@ -80,4 +80,4 @@ Resources: # - !Ref ADFTerraformRole ## # End of Terraform resources -## \ No newline at end of file +## From 079559a20aba4bfcce968048051673c85518dd8b Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 14:48:54 +0100 Subject: [PATCH 164/179] fixed double quotes bash syntax --- .../shared/helpers/terraform/adf_terraform.sh | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 32c2d2a7b..ced4210de 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -8,15 +8,15 @@ echo "Terraform stage: $TF_STAGE" tfinit(){ # retrieve regional S3 bucket name from parameter store S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) - mkdir -p "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" - cd "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" || exit - cp -R "$CURRENT"/tf/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit + cp -R "${CURRENT}/tf/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" # if account related variables exist copy the folder in the work directory - if [ -d "$CURRENT"/tfvars/"$TF_VAR_TARGET_ACCOUNT_ID" ]; then - cp -R "${CURRENT}"/tfvars/"${TF_VAR_TARGET_ACCOUNT_ID}"/* "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" + if [ -d "${CURRENT}/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then + cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}"/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" fi - if [ -f "$CURRENT"/tfvars/global.auto.tfvars ]; then - cp -R "${CURRENT}"/tfvars/global.auto.tfvars "${CURRENT}"/tmp/"${TF_VAR_TARGET_ACCOUNT_ID}"-"${AWS_REGION}" + if [ -f "${CURRENT}/tfvars/global.auto.tfvars" ]; then + cp -R "${CURRENT}/tfvars/global.auto.tfvars" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" fi terraform init \ -backend-config "bucket=$S3_BUCKET_REGION_NAME" \ @@ -32,14 +32,14 @@ tfinit(){ tfplan(){ DATE=$(date +%Y-%m-%d) TS=$(date +%Y%m%d%H%M%S) - bash "$CURRENT"/adf-build/helpers/sts.sh "$TF_VAR_TARGET_ACCOUNT_ID" "$TF_VAR_TARGET_ACCOUNT_ROLE" - terraform plan -out "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}"-"${TS}".log + bash "${CURRENT}/adf-build/helpers/sts.sh" "${TF_VAR_TARGET_ACCOUNT_ID}" "${TF_VAR_TARGET_ACCOUNT_ROLE}" + terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}".log # Save Terraform plan results to the S3 bucket - aws s3 cp "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}"-"${TS}".log s3://"${S3_BUCKET_REGION_NAME}"/"${ADF_PROJECT_NAME}"/tf-plan/"${DATE}"/"${TF_VAR_TARGET_ACCOUNT_ID}"/"${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}"-"${TS}".log + aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" } tfapply(){ - terraform apply "${ADF_PROJECT_NAME}"-"${TF_VAR_TARGET_ACCOUNT_ID}" + terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" } tfrun(){ export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID @@ -101,7 +101,7 @@ do if ! [[ -z "$TARGET_OUS" ]] then echo "List target OUs: $TARGET_OUS" - for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}"/accounts_from_ous.json | sed 's/"//g' ) + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g' ) do tfrun done From fe8036d997da0a94ac12339a5dd98e77fddd7fc7 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 16:05:03 +0100 Subject: [PATCH 165/179] added newline --- docs/user-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 3af3fa47b..6b66c1cf2 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1190,4 +1190,4 @@ The following state files are created: A DynamoDB table is created to manage the lock of the state file. It is deployed in every ADF regions named `adf_locktable`. **Please note**: usage -of this locking table is charged on the deployment account. \ No newline at end of file +of this locking table is charged on the deployment account. From d104f37817647a34d26e6f3406d04f95936d5f85 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Tue, 31 Jan 2023 16:35:32 +0100 Subject: [PATCH 166/179] fixed bash syntax --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index ced4210de..8bb12b89f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -10,7 +10,7 @@ tfinit(){ S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit - cp -R "${CURRENT}/tf/*" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + cp -R "${CURRENT}"/tf/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" # if account related variables exist copy the folder in the work directory if [ -d "${CURRENT}/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}"/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" From 83a2b28f11444239c7585671040c6959c9bcc26e Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 3 Feb 2023 23:55:50 +0100 Subject: [PATCH 167/179] added tfstate-lockfile DynamoDB table --- .../adf-bootstrap/deployment/global.yml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index c6762d6d1..bc916d658 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -63,6 +63,14 @@ Parameters: Description: The Prefix that will be attached to stacks deployed via ADF. Type: String Default: "adf-" + + ADFTerraformExtension: + Type: "AWS::SSM::Parameter::Value" + Default: /adf/extensions/terraform/enabled + +Conditions: + ADFTerraformExtensionEnabled: + !Equals [!Ref ADFTerraformExtension, "True"] Globals: Function: @@ -1457,6 +1465,21 @@ Resources: - Arn: !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}" RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline + + TerraformLockTable: + Condition: ADFTerraformExtensionEnabled + Type: "AWS::DynamoDB::Table" + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + AttributeDefinitions: + - AttributeName: LockID + AttributeType: S + KeySchema: + - AttributeName: LockID + KeyType: HASH + BillingMode: PAY_PER_REQUEST + TableName: adf-tflocktable Outputs: ADFVersionNumber: From 911af7573b926e93cfd1aa20db30f3bcad9986ed Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 3 Feb 2023 23:56:22 +0100 Subject: [PATCH 168/179] fixed ADFTerraformExtensionEnabled condition --- .../adf-bootstrap/deployment/regional.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml index faa105f6a..74eab131b 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/regional.yml @@ -15,9 +15,7 @@ Parameters: Conditions: ADFTerraformExtensionEnabled: - Fn::Equals: - - !Ref ADFTerraformExtension - - true + !Equals [!Ref ADFTerraformExtension, "True"] Resources: DeploymentFrameworkRegionalS3Bucket: From 36477b30e0940ba4320a1a41e5aec6725c3c3c43 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Sat, 4 Feb 2023 00:02:23 +0100 Subject: [PATCH 169/179] removed trailing whitespace --- .../bootstrap_repository/adf-bootstrap/deployment/global.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml index bc916d658..39a733b88 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/global.yml @@ -63,7 +63,7 @@ Parameters: Description: The Prefix that will be attached to stacks deployed via ADF. Type: String Default: "adf-" - + ADFTerraformExtension: Type: "AWS::SSM::Parameter::Value" Default: /adf/extensions/terraform/enabled @@ -1465,7 +1465,7 @@ Resources: - Arn: !Sub "arn:${AWS::Partition}:codepipeline:${AWS::Region}:${AWS::AccountId}:${CodePipeline}" RoleArn: !GetAtt PipelineCloudWatchEventRole.Arn Id: adf-codepipeline-trigger-pipeline - + TerraformLockTable: Condition: ADFTerraformExtensionEnabled Type: "AWS::DynamoDB::Table" From af9a68a925726b977849894fb135cfb58cc07af0 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Mon, 20 Mar 2023 22:27:38 +0100 Subject: [PATCH 170/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 8bb12b89f..5c5ef0a58 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -12,7 +12,7 @@ tfinit(){ cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit cp -R "${CURRENT}"/tf/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" # if account related variables exist copy the folder in the work directory - if [ -d "${CURRENT}/tfvars/$TF_VAR_TARGET_ACCOUNT_ID" ]; then + if [ -d "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}" ]; then cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}"/* "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" fi if [ -f "${CURRENT}/tfvars/global.auto.tfvars" ]; then From a3925333c0cd206bcb5a2356ad7a4221fca4e912 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Mon, 20 Mar 2023 22:27:53 +0100 Subject: [PATCH 171/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh Co-authored-by: Simon Kok --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 5c5ef0a58..89f91ea88 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -33,7 +33,7 @@ tfplan(){ DATE=$(date +%Y-%m-%d) TS=$(date +%Y%m%d%H%M%S) bash "${CURRENT}/adf-build/helpers/sts.sh" "${TF_VAR_TARGET_ACCOUNT_ID}" "${TF_VAR_TARGET_ACCOUNT_ROLE}" - terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}".log + terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" # Save Terraform plan results to the S3 bucket aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" From cdea90a36125a07cbbd0ab065e53817d3eb13202 Mon Sep 17 00:00:00 2001 From: stemons <75492869+stemons@users.noreply.github.com> Date: Mon, 20 Mar 2023 22:39:12 +0100 Subject: [PATCH 172/179] Update get_accounts.py --- .../shared/helpers/terraform/get_accounts.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index d854d8b02..b8fda6e3f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -41,7 +41,10 @@ def main(): def list_organizational_units_for_parent(parent_ou): organizations = get_boto3_client( 'organizations', - f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', + ( + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' + ), 'getOrganizationUnits', ) organizational_units = [ @@ -66,7 +69,10 @@ def get_accounts(): # and emails organizations = get_boto3_client( 'organizations', - f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', + ( + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' + ), 'getaccountIDs', ) return list( @@ -88,7 +94,10 @@ def get_accounts_from_ous(): account_list = [] organizations = get_boto3_client( 'organizations', - f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', + ( + f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' + ), 'getRootAccountIDs', ) # Read organization root id From 491d26e58955ea9354c17100ebf832fa023dfff0 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Wed, 19 Apr 2023 12:14:51 +0200 Subject: [PATCH 173/179] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index b8fda6e3f..2e7b02bd4 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -96,7 +96,7 @@ def get_accounts_from_ous(): 'organizations', ( f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' - f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', ), 'getRootAccountIDs', ) From 51e6d85ae8fbf9ce7323437c774db43d6f297918 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 4 May 2023 22:20:29 +0200 Subject: [PATCH 174/179] added terraform destroy capabilities to terraform script --- .../shared/helpers/terraform/adf_terraform.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 89f91ea88..9df861b4e 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -41,6 +41,12 @@ tfplan(){ tfapply(){ terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" } +tfplandestroy(){ + terraform plan -destroy -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy +} +tfdestroy(){ + terraform apply "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy +} tfrun(){ export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" @@ -62,6 +68,13 @@ tfrun(){ tfplan tfapply set +e + elif [[ "$TF_STAGE" = "destroy" ]] + then + set -e + tfinit + tfplandestroy + tfdestroy + set +e else echo "Invalid Terraform stage: TF_STAGE = $TF_STAGE" exit 1 From ea49db12f9a5d3765355ee53545ba78f9ee7fcbb Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 4 May 2023 22:22:14 +0200 Subject: [PATCH 175/179] added terraform destroy buildspec file to samples --- samples/sample-terraform/tf_destroy.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 samples/sample-terraform/tf_destroy.yml diff --git a/samples/sample-terraform/tf_destroy.yml b/samples/sample-terraform/tf_destroy.yml new file mode 100644 index 000000000..bd0b07c2e --- /dev/null +++ b/samples/sample-terraform/tf_destroy.yml @@ -0,0 +1,17 @@ +version: 0.2 + +env: + variables: + TF_VAR_TARGET_ACCOUNT_ROLE: adf-terraform-role # The IAM Role Terraform will assume to deploy resources + TF_IN_AUTOMATION: true + TF_STAGE: "destroy" + TF_CLI_ARGS: "-no-color" + +phases: + install: + runtime-versions: + python: 3.9 + build: + commands: + - python adf-build/helpers/terraform/get_accounts.py + - bash adf-build/helpers/terraform/adf_terraform.sh From 3c22d11bc005934185eae9ac0b3294a4547eac52 Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 4 May 2023 22:25:19 +0200 Subject: [PATCH 176/179] added terraform destroy capability to user documentation --- docs/user-guide.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/user-guide.md b/docs/user-guide.md index 822291596..634214b2d 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1077,6 +1077,8 @@ stages defined in the following CodeBuild build specification: run a Terraform plan. - `tf_apply.yml`: get the list of accounts from the organization and run a Terraform plan and apply. +- `tf_destroy.yml`: get the list of accounts from the organization and + run a Terraform plan and destroy. An optional approval step could be added between plan and apply as shown in the pipeline definition below. @@ -1132,6 +1134,10 @@ pipelines: - name: terraform-apply properties: spec_filename: tf_apply.yml # Terraform apply + # (optional) + - name: terraform-destroy + properties: + spec_filename: tf_destroy.yml # Terraform destroy ``` 1. Add a sample-terraform pipeline in ADF `deployment-map.yml` as shown above. From b9669206057e09c82b8347a066b01f9c684684dc Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Thu, 4 May 2023 22:32:35 +0200 Subject: [PATCH 177/179] removed trailing whitespace --- .../adf-build/shared/helpers/terraform/adf_terraform.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 9df861b4e..99d6fccf9 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -42,7 +42,7 @@ tfapply(){ terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" } tfplandestroy(){ - terraform plan -destroy -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy + terraform plan -destroy -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy } tfdestroy(){ terraform apply "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy From c09c3cfc629fb39c1eed350b1529324a14ff691d Mon Sep 17 00:00:00 2001 From: Stefano Montanelli Date: Fri, 5 May 2023 09:22:11 +0200 Subject: [PATCH 178/179] removed syntax error --- .../adf-build/shared/helpers/terraform/get_accounts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py index 2e7b02bd4..b8fda6e3f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/get_accounts.py @@ -96,7 +96,7 @@ def get_accounts_from_ous(): 'organizations', ( f'arn:{PARTITION}:sts::{MANAGEMENT_ACCOUNT_ID}:role/' - f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly', + f'{CROSS_ACCOUNT_ACCESS_ROLE}-readonly' ), 'getRootAccountIDs', ) From afaa146ae40b63a239539ab5031589530bc6bbef Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Fri, 12 May 2023 22:54:30 +0200 Subject: [PATCH 179/179] Fix linting / style issues --- docs/user-guide.md | 3 +-- linters/custom-adf-dict.txt | 2 ++ samples/sample-terraform/tf_destroy.yml | 1 + .../adf-build/shared/helpers/terraform/adf_terraform.sh | 4 ++-- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/user-guide.md b/docs/user-guide.md index 634214b2d..59f778bab 100644 --- a/docs/user-guide.md +++ b/docs/user-guide.md @@ -1134,8 +1134,7 @@ pipelines: - name: terraform-apply properties: spec_filename: tf_apply.yml # Terraform apply - # (optional) - - name: terraform-destroy + - name: terraform-destroy # (optional stage) properties: spec_filename: tf_destroy.yml # Terraform destroy ``` diff --git a/linters/custom-adf-dict.txt b/linters/custom-adf-dict.txt index 8fd5ddbe5..8df486765 100644 --- a/linters/custom-adf-dict.txt +++ b/linters/custom-adf-dict.txt @@ -48,9 +48,11 @@ sdkman stefanzweifel stubber tfapply +tfdestroy tfinit tflint tflocktable +tfplandestroy tfrun tfstate tfvars diff --git a/samples/sample-terraform/tf_destroy.yml b/samples/sample-terraform/tf_destroy.yml index bd0b07c2e..5dc54f55d 100644 --- a/samples/sample-terraform/tf_destroy.yml +++ b/samples/sample-terraform/tf_destroy.yml @@ -11,6 +11,7 @@ phases: install: runtime-versions: python: 3.9 + build: commands: - python adf-build/helpers/terraform/get_accounts.py diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index 99d6fccf9..5ca974a1f 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -42,10 +42,10 @@ tfapply(){ terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" } tfplandestroy(){ - terraform plan -destroy -out "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy + terraform plan -destroy -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-destroy" } tfdestroy(){ - terraform apply "$ADF_PROJECT_NAME"-"$TF_VAR_TARGET_ACCOUNT_ID"-destroy + terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-destroy" } tfrun(){ export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID