diff --git a/.editorconfig b/.editorconfig index c7a938d82..9ace96ac9 100644 --- a/.editorconfig +++ b/.editorconfig @@ -56,3 +56,6 @@ max_line_length = 120 indent_style = tab indent_size = 4 tab_width = 4 + +[*.sh] +space_redirects = true diff --git a/.mega-linter.yml b/.mega-linter.yml index c89e80a57..29fb905a3 100644 --- a/.mega-linter.yml +++ b/.mega-linter.yml @@ -10,6 +10,7 @@ APPLY_FIXES: none # If you use ENABLE_LINTERS variable, all other linters will be disabled by default ENABLE_LINTERS: - BASH_EXEC + - BASH_SHFMT - CLOUDFORMATION_CFN_LINT - DOCKERFILE_HADOLINT - EDITORCONFIG_EDITORCONFIG_CHECKER @@ -30,16 +31,20 @@ FILEIO_REPORTER: false # Install plugin for list handling. JSON_PRETTIER_PRE_COMMANDS: - - command: "npm install prettier-plugin-multiline-arrays@3.0.0" + - command: "npm install prettier-plugin-multiline-arrays@3.0.4" cwd: "workspace" CLOUDFORMATION_CFN_LINT_CONFIG_FILE: '.cfnlintrc' CLOUDFORMATION_CFN_LINT_FILE_EXTENSIONS: [".yml", ".yaml"] + EDITORCONFIG_EDITORCONFIG_CHECKER_CONFIG_FILE: '.ecrc.json' + MARKDOWN_MARKDOWN_LINK_CHECK_ARGUMENTS: '-q' MARKDOWN_MARKDOWNLINT_DISABLE_ERRORS: false + SPELL_CSPELL_ARGUMENTS: '--gitignore --no-progress --show-suggestions' SPELL_CSPELL_FILE_EXTENSIONS: ["*"] + TERRAFORM_TFLINT_UNSECURED_ENV_VARIABLES: - GITHUB_TOKEN diff --git a/samples/sample-ec2-java-app-codedeploy/scripts/validate.sh b/samples/sample-ec2-java-app-codedeploy/scripts/validate.sh index 30b9c8668..01e2c5ebd 100755 --- a/samples/sample-ec2-java-app-codedeploy/scripts/validate.sh +++ b/samples/sample-ec2-java-app-codedeploy/scripts/validate.sh @@ -4,7 +4,7 @@ echo "Waiting for 15 seconds before checking health.." sleep 15 status_code=$(curl --write-out %{http_code} --silent --output /dev/null http://localhost:80) -if [[ "$status_code" -ne 200 ]] ; then +if [[ "$status_code" -ne 200 ]]; then echo "App is not healthy - $status_code" exit 1 else diff --git a/samples/sample-ec2-with-codedeploy/scripts/install-codedeploy.sh b/samples/sample-ec2-with-codedeploy/scripts/install-codedeploy.sh index 96a9d1af5..81127e951 100755 --- a/samples/sample-ec2-with-codedeploy/scripts/install-codedeploy.sh +++ b/samples/sample-ec2-with-codedeploy/scripts/install-codedeploy.sh @@ -4,10 +4,10 @@ set -xe ## Code Deploy Agent Bootstrap Script ## -exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +exec > >(tee /var/log/user-data.log | logger -t user-data -s 2> /dev/console) 2>&1 AUTOUPDATE=false -function installdep(){ +function installdep() { if [ ${PLAT} = "ubuntu" ]; then apt-get -y update # Satisfying even Ubuntu older versions. @@ -18,7 +18,7 @@ function installdep(){ fi } -function platformize(){ +function platformize() { # Linux OS detection if hash lsb_release; then echo "Ubuntu server OS detected" @@ -32,8 +32,7 @@ function platformize(){ fi } - -function execute(){ +function execute() { if [ ${PLAT} = "ubuntu" ]; then cd /tmp/ wget https://aws-codedeploy-${REGION}.s3.${REGION}.amazonaws.com/latest/install @@ -41,12 +40,12 @@ function execute(){ if ./install auto; then echo "Installation completed" - if ! ${AUTOUPDATE}; then - echo "Disabling Auto Update" - sed -i '/@reboot/d' /etc/cron.d/codedeploy-agent-update - chattr +i /etc/cron.d/codedeploy-agent-update - rm -f /tmp/install - fi + if ! ${AUTOUPDATE}; then + echo "Disabling Auto Update" + sed -i '/@reboot/d' /etc/cron.d/codedeploy-agent-update + chattr +i /etc/cron.d/codedeploy-agent-update + rm -f /tmp/install + fi exit 0 else echo "Installation script failed, please investigate" @@ -61,12 +60,12 @@ function execute(){ if ./install auto; then echo "Installation completed" - if ! ${AUTOUPDATE}; then - echo "Disabling auto update" - sed -i '/@reboot/d' /etc/cron.d/codedeploy-agent-update - chattr +i /etc/cron.d/codedeploy-agent-update - rm -f /tmp/install - fi + if ! ${AUTOUPDATE}; then + echo "Disabling auto update" + sed -i '/@reboot/d' /etc/cron.d/codedeploy-agent-update + chattr +i /etc/cron.d/codedeploy-agent-update + rm -f /tmp/install + fi exit 0 else echo "Installation script failed, please investigate" diff --git a/samples/sample-ec2-with-codedeploy/scripts/install-deps.sh b/samples/sample-ec2-with-codedeploy/scripts/install-deps.sh index 874e4f050..91858d13e 100755 --- a/samples/sample-ec2-with-codedeploy/scripts/install-deps.sh +++ b/samples/sample-ec2-with-codedeploy/scripts/install-deps.sh @@ -30,11 +30,13 @@ sudo useradd springboot sudo chsh -s /sbin/nologin springboot # forward port 80 to 8080 -echo " +echo " + ProxyRequests Off ProxyPass / http://localhost:8080/ ProxyPassReverse / http://localhost:8080/ -" >> sudo /etc/httpd/conf/httpd.conf + +" | sudo tee -a /etc/httpd/conf/httpd.conf > /dev/null # start the httpd service now and stop it until userdata sudo service httpd start diff --git a/samples/sample-fargate-node-app/build/docker.sh b/samples/sample-fargate-node-app/build/docker.sh index 2e628a446..a79887076 100755 --- a/samples/sample-fargate-node-app/build/docker.sh +++ b/samples/sample-fargate-node-app/build/docker.sh @@ -11,4 +11,5 @@ docker tag $REPOSITORY_URI:latest $REPOSITORY_URI:$IMAGE_TAG docker push $REPOSITORY_URI:latest docker push $REPOSITORY_URI:$IMAGE_TAG -tmp=$(mktemp); jq --arg REPOSITORY_URI "$REPOSITORY_URI" --arg IMAGE_TAG "$IMAGE_TAG" '.Parameters.Image = $REPOSITORY_URI+":"+$IMAGE_TAG' params/global.json > "$tmp" && mv "$tmp" params/global.json +tmp=$(mktemp) +jq --arg REPOSITORY_URI "$REPOSITORY_URI" --arg IMAGE_TAG "$IMAGE_TAG" '.Parameters.Image = $REPOSITORY_URI+":"+$IMAGE_TAG' params/global.json > "$tmp" && mv "$tmp" params/global.json diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh index c325be39e..044560146 100755 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/package_transform.sh @@ -11,16 +11,15 @@ set -e SKIP_BUILD=0 # Walk through the options passed to this script -for i in "$@" -do +for i in "$@"; do case $i in - --no-build) - SKIP_BUILD=1 - ;; - *) - echo "Unknown option: $i" - exit 1 - ;; + --no-build) + SKIP_BUILD=1 + ;; + *) + echo "Unknown option: $i" + exit 1 + ;; esac done @@ -34,16 +33,15 @@ fi # Get list of regions supported by this application echo "Determine which regions need to be prepared" -app_regions=`aws ssm get-parameters --names /adf/deployment/$ADF_DEPLOYMENT_MAP_SOURCE/$ADF_PROJECT_NAME/regions --with-decryption --output=text --query='Parameters[0].Value'` +app_regions=$(aws ssm get-parameters --names /adf/deployment/$ADF_DEPLOYMENT_MAP_SOURCE/$ADF_PROJECT_NAME/regions --with-decryption --output=text --query='Parameters[0].Value') # Convert json list to bash list (space delimited regions) -regions="`echo $app_regions | sed -e 's/\[\([^]]*\)\]/\1/g' | sed 's/,/ /g' | sed "s/'//g"`" +regions="$(echo $app_regions | sed -e 's/\[\([^]]*\)\]/\1/g' | sed 's/,/ /g' | sed "s/'//g")" -for region in $regions -do +for region in $regions; do if [ $CONTAINS_TRANSFORM ]; then echo "Packaging templates for region $region" ssm_bucket_name="/adf/cross_region/s3_regional_bucket/$region" - bucket=`aws ssm get-parameters --names $ssm_bucket_name --with-decryption --output=text --query='Parameters[0].Value'` + bucket=$(aws ssm get-parameters --names $ssm_bucket_name --with-decryption --output=text --query='Parameters[0].Value') sam package --s3-bucket $bucket --output-template-file $CODEBUILD_SRC_DIR/template_$region.yml --region $region else # If package is not needed, just copy the file for each region diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh index d9635ab73..a6d073e9b 100755 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-build/shared/helpers/terraform/adf_terraform.sh @@ -5,123 +5,111 @@ CURRENT=$(pwd) terraform --version echo "Terraform stage: $TF_STAGE" -tfinit(){ - # retrieve regional S3 bucket name from parameter store - S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/adf/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) - mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit - cp -R "${CURRENT}"/tf/. "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - # if account related variables exist copy the folder in the work directory - if [ -d "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}" ]; then - cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/." "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - fi - if [ -d "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/${AWS_REGION}" ]; then - cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/${AWS_REGION}"/. "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - fi - if [ -f "${CURRENT}/tfvars/global.auto.tfvars" ]; then - cp -R "${CURRENT}/tfvars/global.auto.tfvars" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" - fi - terraform init \ - -backend-config "bucket=$S3_BUCKET_REGION_NAME" \ - -backend-config "region=$AWS_REGION" \ - -backend-config "key=$ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" \ - -backend-config "dynamodb_table=adf-tflocktable" +tfinit() { + # retrieve regional S3 bucket name from parameter store + S3_BUCKET_REGION_NAME=$(aws ssm get-parameter --name "/adf/cross_region/s3_regional_bucket/$AWS_REGION" --region "$AWS_DEFAULT_REGION" | jq .Parameter.Value | sed s/\"//g) + mkdir -p "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + cd "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" || exit + cp -R "${CURRENT}"/tf/. "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + # if account related variables exist copy the folder in the work directory + if [ -d "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}" ]; then + cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/." "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + fi + if [ -d "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/${AWS_REGION}" ]; then + cp -R "${CURRENT}/tfvars/${TF_VAR_TARGET_ACCOUNT_ID}/${AWS_REGION}"/. "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + fi + if [ -f "${CURRENT}/tfvars/global.auto.tfvars" ]; then + cp -R "${CURRENT}/tfvars/global.auto.tfvars" "${CURRENT}/tmp/${TF_VAR_TARGET_ACCOUNT_ID}-${AWS_REGION}" + fi + terraform init \ + -backend-config "bucket=$S3_BUCKET_REGION_NAME" \ + -backend-config "region=$AWS_REGION" \ + -backend-config "key=$ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" \ + -backend-config "dynamodb_table=adf-tflocktable" - echo "Bucket: $S3_BUCKET_REGION_NAME" - echo "Region: $AWS_REGION" - echo "Key: $ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" - echo "DynamoDB table: adf-tflocktable" + echo "Bucket: $S3_BUCKET_REGION_NAME" + echo "Region: $AWS_REGION" + echo "Key: $ADF_PROJECT_NAME/$ACCOUNT_ID.tfstate" + echo "DynamoDB table: adf-tflocktable" } -tfplan(){ - DATE=$(date +%Y-%m-%d) - TS=$(date +%Y%m%d%H%M%S) - bash "${CURRENT}/adf-build/helpers/sts.sh" "${TF_VAR_TARGET_ACCOUNT_ID}" "${TF_VAR_TARGET_ACCOUNT_ROLE}" - set -o pipefail - terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" - set +o pipefail - # Save Terraform plan results to the S3 bucket - aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" - echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" +tfplan() { + DATE=$(date +%Y-%m-%d) + TS=$(date +%Y%m%d%H%M%S) + bash "${CURRENT}/adf-build/helpers/sts.sh" "${TF_VAR_TARGET_ACCOUNT_ID}" "${TF_VAR_TARGET_ACCOUNT_ROLE}" + set -o pipefail + terraform plan -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" 2>&1 | tee -a "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" + set +o pipefail + # Save Terraform plan results to the S3 bucket + aws s3 cp "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" "s3://${S3_BUCKET_REGION_NAME}/${ADF_PROJECT_NAME}/tf-plan/${DATE}/${TF_VAR_TARGET_ACCOUNT_ID}/${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-${TS}.log" + echo "Path to terraform plan s3://$S3_BUCKET_REGION_NAME/$ADF_PROJECT_NAME/tf-plan/$DATE/$TF_VAR_TARGET_ACCOUNT_ID/$ADF_PROJECT_NAME-$TF_VAR_TARGET_ACCOUNT_ID-$TS.log" } -tfapply(){ - terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" +tfapply() { + terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}" } -tfplandestroy(){ - terraform plan -destroy -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-destroy" +tfplandestroy() { + terraform plan -destroy -out "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-destroy" } -tfdestroy(){ - terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-destroy" +tfdestroy() { + terraform apply "${ADF_PROJECT_NAME}-${TF_VAR_TARGET_ACCOUNT_ID}-destroy" } -tfrun(){ - export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID - echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" - if [[ "$TF_STAGE" = "init" ]] - then - set -e - tfinit - set +e - elif [[ "$TF_STAGE" = "plan" ]] - then - set -e - tfinit - tfplan - set +e - elif [[ "$TF_STAGE" = "apply" ]] - then - set -e - tfinit - tfplan - tfapply - set +e - elif [[ "$TF_STAGE" = "destroy" ]] - then - set -e - tfinit - tfplandestroy - tfdestroy - set +e - else - echo "Invalid Terraform stage: TF_STAGE = $TF_STAGE" - exit 1 - fi +tfrun() { + export TF_VAR_TARGET_ACCOUNT_ID=$ACCOUNT_ID + echo "Running terraform $TF_STAGE on account $ACCOUNT_ID and region $REGION" + if [[ "$TF_STAGE" = "init" ]]; then + set -e + tfinit + set +e + elif [[ "$TF_STAGE" = "plan" ]]; then + set -e + tfinit + tfplan + set +e + elif [[ "$TF_STAGE" = "apply" ]]; then + set -e + tfinit + tfplan + tfapply + set +e + elif [[ "$TF_STAGE" = "destroy" ]]; then + set -e + tfinit + tfplandestroy + tfdestroy + set +e + else + echo "Invalid Terraform stage: TF_STAGE = $TF_STAGE" + exit 1 + fi } # if REGIONS is not defined as pipeline parameters use default region -if [[ -z "$REGIONS" ]] -then - REGIONS=$AWS_DEFAULT_REGION +if [[ -z "$REGIONS" ]]; then + REGIONS=$AWS_DEFAULT_REGION fi echo "List of target regions: $REGIONS" -for REGION in $(echo "$REGIONS" | sed "s/,/ /g") -do - AWS_REGION=$(echo -n "$REGION" | sed 's/^[ \t]*//;s/[ \t]*$//') # sed trims whitespaces - export TF_VAR_TARGET_REGION=$AWS_REGION - # if TARGET_ACCOUNTS and TARGET_OUS are not defined apply to all accounts - if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]] - then - echo "Apply to all accounts" - for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts.json" | sed 's/"//g' ) - do - tfrun - done - fi +for REGION in $(echo "$REGIONS" | sed "s/,/ /g"); do + AWS_REGION=$(echo -n "$REGION" | sed 's/^[ \t]*//;s/[ \t]*$//') # sed trims whitespaces + export TF_VAR_TARGET_REGION=$AWS_REGION + # if TARGET_ACCOUNTS and TARGET_OUS are not defined apply to all accounts + if [[ -z "$TARGET_ACCOUNTS" ]] && [[ -z "$TARGET_OUS" ]]; then + echo "Apply to all accounts" + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts.json" | sed 's/"//g'); do + tfrun + done + fi - if ! [[ -z "$TARGET_ACCOUNTS" ]] - then - # apply only on a subset of accounts (TARGET_ACCOUNTS) - echo "List of target account: $TARGET_ACCOUNTS" - for ACCOUNT_ID in $(echo "$TARGET_ACCOUNTS" | sed "s/,/ /g") - do - tfrun - done - fi + if ! [[ -z "$TARGET_ACCOUNTS" ]]; then + # apply only on a subset of accounts (TARGET_ACCOUNTS) + echo "List of target account: $TARGET_ACCOUNTS" + for ACCOUNT_ID in $(echo "$TARGET_ACCOUNTS" | sed "s/,/ /g"); do + tfrun + done + fi - if ! [[ -z "$TARGET_OUS" ]] - then - echo "List target OUs: $TARGET_OUS" - for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g' ) - do - tfrun - done - fi + if ! [[ -z "$TARGET_OUS" ]]; then + echo "List target OUs: $TARGET_OUS" + for ACCOUNT_ID in $(jq '.[].AccountId' "${CURRENT}/accounts_from_ous.json" | sed 's/"//g'); do + tfrun + done + fi done