From 5d3905ce89b09cb426d261c2da6a606610245935 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Wed, 8 Dec 2021 11:18:33 +0100 Subject: [PATCH 1/3] Set Log Level as part of the SAR deployment process **Why?** When you run into errors deploying the latest version of ADF, it was not easy to enable the debug level, as those used to be hard coded and non-configurable. **What?** This change introduces a Log Level parameter to the SAR deployment process. Allowing the administrators to deploy ADF with a specific Log Level such as DEBUG directly from the SAR. --- src/template.yml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/template.yml b/src/template.yml index 4278a13d5..e19ff7f5a 100644 --- a/src/template.yml +++ b/src/template.yml @@ -57,6 +57,16 @@ Parameters: Description: "(Optional) Example -> ou-123,ou-234" Type: CommaDelimitedList Default: "" + LogLevel: + Description: "At what Log Level the ADF should operate, default is INFO. Valid options are: DEBUG, INFO, WARN, ERROR, and CRITICAL." + Type: String + Default: "INFO" + AllowedValues: + - DEBUG + - INFO + - WARN + - ERROR + - CRITICAL Resources: BootstrapTemplatesBucketPolicy: Type: AWS::S3::BucketPolicy @@ -207,7 +217,7 @@ Resources: MASTER_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] - ADF_LOG_LEVEL: INFO + ADF_LOG_LEVEL: !Ref LogLevel FunctionName: StackWaiter Role: !GetAtt LambdaRole.Arn Runtime: python3.8 @@ -228,7 +238,7 @@ Resources: MASTER_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] - ADF_LOG_LEVEL: INFO + ADF_LOG_LEVEL: !Ref LogLevel FunctionName: DetermineEventFunction Role: !GetAtt LambdaRole.Arn Runtime: python3.8 @@ -249,7 +259,7 @@ Resources: MASTER_ACCOUNT_ID: !Ref AWS::AccountId ORGANIZATION_ID: !GetAtt Organization.OrganizationId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] - ADF_LOG_LEVEL: INFO + ADF_LOG_LEVEL: !Ref LogLevel FunctionName: CrossAccountExecuteFunction Role: !GetAtt LambdaRole.Arn Runtime: python3.8 @@ -268,7 +278,7 @@ Resources: TERMINATION_PROTECTION: false MASTER_ACCOUNT_ID: !Ref AWS::AccountId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] - ADF_LOG_LEVEL: INFO + ADF_LOG_LEVEL: !Ref LogLevel FunctionName: RoleStackDeploymentFunction Role: !GetAtt LambdaRole.Arn Runtime: python3.8 @@ -287,7 +297,7 @@ Resources: TERMINATION_PROTECTION: false MASTER_ACCOUNT_ID: !Ref AWS::AccountId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] - ADF_LOG_LEVEL: INFO + ADF_LOG_LEVEL: !Ref LogLevel FunctionName: MovedToRootActionFunction Role: !GetAtt LambdaRole.Arn Runtime: python3.8 @@ -306,7 +316,7 @@ Resources: TERMINATION_PROTECTION: false MASTER_ACCOUNT_ID: !Ref AWS::AccountId ADF_VERSION: !FindInMap ["Metadata", "ADF", "Version"] - ADF_LOG_LEVEL: INFO + ADF_LOG_LEVEL: !Ref LogLevel FunctionName: UpdateResourcePoliciesFunction Role: !GetAtt LambdaRole.Arn Runtime: python3.8 @@ -529,7 +539,7 @@ Resources: - Name: ORGANIZATION_ID Value: !GetAtt Organization.OrganizationId - Name: ADF_LOG_LEVEL - Value: INFO + Value: !Ref LogLevel Type: LINUX_CONTAINER Name: "aws-deployment-framework-base-templates" ServiceRole: !Ref BootstrapCodeBuildRole @@ -906,7 +916,7 @@ Resources: Description: DO NOT EDIT - Used by The AWS Deployment Framework Name: adf_log_level Type: String - Value: INFO + Value: !Ref LogLevel CrossRegionBucketHandler: Type: AWS::Serverless::Function Properties: From 2ad76963b5f40c1326e17c887eb4c6f3b8ae21aa Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Wed, 8 Dec 2021 18:32:14 +0100 Subject: [PATCH 2/3] Add documentation to troubleshoot ADF issues --- docs/admin-guide.md | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 90f90024d..6e0432a35 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -29,6 +29,7 @@ - [Check Current Version](#check-current-version) - [Updating Between Versions](#updating-between-versions) - [Removing ADF](#removing-adf) +- [Troubleshooting](#troubleshooting) ## Src Folder @@ -505,3 +506,63 @@ If you wish to remove ADF you can delete the CloudFormation stack named *serverl One thing to keep in mind if you are planning to re-install ADF is that you will want to clean up the parameter from SSM Parameter Store named *deployment_account_id* within us-east-1 on the master account. AWS Step Functions uses this parameter to determine if ADF has already got a deployment account setup, if you re-install ADF with this parameter set with a value, ADF will attempt an assume role to the account to do some work, which will fail since that role will not be on the account at that point. There is also a CloudFormation stack named *adf-global-base-adf-build* which lives on the master account in your main deployment region. This stack creates two roles on the master account after the deployment account has been setup. These roles allow the deployment accounts CodeBuild role to assume a role back to the master account in order to query Organizations for AWS Accounts. This stack must be deleted manually also, if you do not remove this stack and then perform a fresh install of ADF, AWS CodeBuild on the deployment account will not be able to assume a role to the master account to query AWS Organizations. This is because this specific stack creates IAM roles with a strict trust relationship to the CodeBuild role on the deployment account, if that role gets deleted *(Which is will when you delete adf-global-base-deployment)* then this stack references invalid IAM roles that no longer exist. If you forget to remove this stack and notice the trust relationship of the IAM roles referenced in the stack are no longer valid, you can delete the stack and re-run the main bootstrap pipeline which will recreate it with valid roles and links to the correct roles. + +## Troubleshooting + +If you are experiencing an issue with ADF, please follow the [guide on Updating +Between Versions](#updating-between-versions) to check if your latest +installation was installed successfully before you continue. + +When you need to troubleshoot the installation or upgrade of ADF, please set +set the `Log Level` parameter of the ADF Stack to `DEBUG. + +There are two ways to enable this: + +1. If you installed/upgraded to the latest version and that failed, you can + follow the [installation docs](./installation-guide.md). When you are about + to deploy the latest version again, set the `Log Level` to `DEBUG` to get + extra logging information about the issue you are experiencing. +1. If you are running an older version of ADF, please navigate to the + CloudFormation Console in `us-east-1` of the AWS Management account. + 1. Update the stack. + 1. For any ADF deployment of v3.2.0 and later, please change the `Log Level` + parameter and set it to `DEBUG`. Deploy those changes and revert them + after you gathered the information required to report or fix the issue. + 1. If you are running a version prior to v3.2.0, you will need to update the + template using the CloudFormation Designer. Search for `INFO` and replace + that with `DEBUG`. Deploy the updated version and reverse this process + after you found the logging information you needed to report the issue + or resolve it. + +Please trace the failed component and dive into/report the debug information. + +The main components to look at are: + +1. In the AWS Management Account in `us-east-1`: + 1. the [CloudFormation aws-deployment-framework stack](https://console.aws.amazon.com/cloudformation/home?region=us-east-1#/stacks?filteringStatus=active&filteringText=aws-deployment-framework&viewNested=true&hideStacks=false). + 1. the [CloudWatch Logs for the Lambda functions deployed by ADF](https://console.aws.amazon.com/lambda/home?region=us-east-1#/functions?f0=true&n0=false&op=and&v0=ADF). + 1. check if the [CodeCommit pull request](https://console.aws.amazon.com/codesuite/codecommit/repositories/aws-deployment-framework-bootstrap/pull-requests?region=us-east-1&status=OPEN) to install the latest version changes of ADF has been merged into your main branch for the `aws-deployment-framework-bootstrap` (ADF Bootstrap) repository. + 1. the [CodePipeline execution of the AWS Bootstrap pipeline](https://console.aws.amazon.com/codesuite/codepipeline/pipelines/aws-deployment-framework-bootstrap-pipeline/view?region=us-east-1). + 1. the [ADF Bootstrapping Step Function State Machine](https://console.aws.amazon.com/states/home?region=us-east-1#/statemachines). + * Look at the previous executions of the State Machine. + * When you find one that has a failed execution, check the components that are marked orange/red in the diagram. +1. In the AWS Deployment Account in the deployment region: + 1. the [CodePipeline execution of the `aws-deployment-framework-pipelines` (ADF pipelines) repository](https://eu-west-1.console.aws.amazon.com/codesuite/codepipeline/pipelines/aws-deployment-framework-pipelines/view?region=eu-west-1) <- link points to `eu-west-1`, please change that to your own deployment region. + +### How to share debug information + +**Important**: If you are about to share any debug information through an +issue on the [ADF Github repository](https://github.com/awslabs/aws-deployment-framework/issues), +please replace: + +* the account ids with simple account ids like: `111111111111`, `222222222222`, etc. +* the organization id with a simple one, `o-theorgid`. +* the organization unit identifiers and names. +* the email addresses by hiding them behind `--some-notifcation-email-address--`. +* the slack channel identifier and SNS topics configured with simplified ones. +* the cross account access role with the default `OrganizationAccountAccessRole`. +* the S3 buckets using a simplified bucket name, like `example-bucket-1`. +* the Amazon Resource Names (ARNs) could also expose information. + +Always read what you are about to share carefully to make sure any identifiable +or sensitive information is removed. From 6d6cbbd7457fd04e30a9d81ff06b4c2f898e0b60 Mon Sep 17 00:00:00 2001 From: Simon Kok Date: Thu, 9 Dec 2021 15:56:23 +0100 Subject: [PATCH 3/3] Update docs/admin-guide.md - add missing tick Co-authored-by: Joris Conijn --- docs/admin-guide.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/admin-guide.md b/docs/admin-guide.md index 6e0432a35..3b22702b7 100644 --- a/docs/admin-guide.md +++ b/docs/admin-guide.md @@ -514,7 +514,7 @@ Between Versions](#updating-between-versions) to check if your latest installation was installed successfully before you continue. When you need to troubleshoot the installation or upgrade of ADF, please set -set the `Log Level` parameter of the ADF Stack to `DEBUG. +set the `Log Level` parameter of the ADF Stack to `DEBUG`. There are two ways to enable this: