v1.9.0 - Lock files, lock tables, S3 upgrade, kms key alias

Author: Zordrak

CHANGELOG.md

@@ -1,3 +1,24 @@
+## 1.9.0 (06/09/2024)
+
+FEATURES:
+
+ * New options:
+   * -l/--lockfile: Local lock file
+   * -t/--lock-table: Lock with DynamoDB Table
+ * Bootstrap:
+   * New multi-resource approach to S3 Bucket
+   * KMS Key Alias for S3 Bucket KMS Key, for easy data-sourcing elsewhere
+   * New tfscaffold: prefix for tfscaffold tags
+ * Updated the warning for duplicate input variables
+
+BUG FIXES:
+
+ * Some alignment and ordering
+
+## 1.8.0 (Unreleased)
+
+ * Merged into 1.9.0
+
 ## 1.7.1 (14/07/2023)
 
  * Explicitly declare and clear out, refresh and destroy in case someone exports them

README.md

@@ -111,9 +111,11 @@ bin/terraform.sh \
   -e/--environment       `environment` \
   -g/--group             `group` (optional) \
   -i/--build-id          `build_id` (optional) \
+  -l/--lockfile          `mode` (optional) \
   -p/--project           `project` \
   -r/--region            `region` \
   -d/--detailed-exitcode (optional) \
+  -t/--lock-table        (optional) \
   -n/--no-color          (optional) \
   -w/--compact-warnings  (optional) \
   -- \
@@ -134,9 +136,11 @@ Where:
 * `component_name`: The name of the terraform component in the components directory to run the `action` against.
 * `environment`: The name of the environment the component is to be actioned against, therefore implying the variables file(s) to be included
 * `group` (optional): The name of the group to which the environment belongs, permitting the use of a group tfvars file as a "meta-environment" shared by more than one environment
+* `lockfile` (optional): Passes the given lockfile mode to terraform.
 * `project`: The name of the project being deployed, as per the default bucket-prefix and state file keyspace
 * `region` (optional): The AWS region name unique to all components and terraform processes. Defaults to the value of the _AWS_DEFAULT_REGION_ environment variable.
 * `detailed-exitcode` (optional): Passes detailed exit code flag to terraform.
+* `lock-table` (optional): Tells tfscaffold to use a DynamoDB Table with the same name as the S3 Bucket for state file locking
 * `no-color` (optional): Passes no-color flag to terraform.
 * `compact-warnings` (optional): Passes compact-warnings flag to terraform.
 * `additional arguments`: Any arguments provided after "--" will be passed directly to terraform as its own arguments, e.g. allowing the provision of a 'target=value' parameter.

bin/terraform.sh

@@ -8,7 +8,7 @@
 ##
 # Set Script Version
 ##
-readonly script_ver="1.7.1";
+readonly script_ver="1.9.0";
 
 ##
 # Standardised failure function
@@ -38,10 +38,12 @@ Usage: ${0} \\
   -e/--environment   [environment] \\
   -g/--group         [group]
   -i/--build-id      [build_id] (optional) \\
+  -l/--lockfile      [mode] \\
   -p/--project       [project] \\
   -r/--region        [region] \\
   -d/--detailed-exitcode \\
   -n/--no-color \\
+  -t/--lock-table \\
   -w/--compact-warnings \\
   -- \\
   <additional arguments to forward to the terraform binary call>
@@ -92,12 +94,21 @@ detailed-exitcode (optional):
  Changes the plan operation to exit 0 only when there are no changes.
  Will be ignored for actions other than plan.
 
+lock-table (optional):
+  When not provided, false.
+  Adds a dynamodb_table statement to the S3 backend configuration
+  to use a DynamoDB table with the same name as the S3 bucket
+  for terraform state locking.
+
 no-color (optional):
  Append -no-color to all terraform calls
 
 compact-warnings (optional):
  Append -compact-warnings to all terraform calls
 
+lockfile:
+ Append -lockfile=MODE to calls to terraform init
+
 additional arguments:
  Any arguments provided after "--" will be passed directly to terraform as its own arguments
 EOF
@@ -116,8 +127,8 @@ fi
 ##
 readonly raw_arguments="${*}";
 ARGS=$(getopt \
-         -o dhnvwa:b:c:e:g:i:p:r: \
-         -l "help,version,bootstrap,action:,bucket-prefix:,build-id:,component:,environment:,group:,project:,region:,detailed-exitcode,no-color,compact-warnings" \
+         -o dhntvwa:b:c:e:g:i:l:p:r: \
+         -l "help,version,bootstrap,action:,bucket-prefix:,build-id:,component:,environment:,group:,project:,region:,lockfile:,detailed-exitcode,lock-table,no-color,compact-warnings" \
          -n "${0}" \
          -- \
          "$@");
@@ -138,8 +149,10 @@ declare group;
 declare action;
 declare bucket_prefix;
 declare build_id;
+declare lockfile;
 declare project;
 declare detailed_exitcode;
+declare lock_table;
 declare no_color;
 declare compact_warnings;
 declare out="";
@@ -205,6 +218,13 @@ while true; do
         shift;
       fi;
       ;;
+    -l|--lockfile)
+      shift;
+      if [ -n "${1}" ]; then
+        lockfile="-lockfile=${1}";
+        shift;
+      fi;
+      ;;
     -p|--project)
       shift;
       if [ -n "${1}" ]; then
@@ -220,6 +240,10 @@ while true; do
       shift;
       detailed_exitcode="true";
       ;;
+    -t|--lock-table)
+      shift;
+      lock_table="true";
+      ;;
     -n|--no-color)
       shift;
       no_color="-no-color";
@@ -272,6 +296,7 @@ if [ "${bootstrap}" == "true" ]; then
     && error_and_die "The --bootstrap parameter and the -c/--component parameter are mutually exclusive";
   [ -n "${build_id}" ] \
     && error_and_die "The --bootstrap parameter and the -i/--build-id parameter are mutually exclusive. We do not currently support plan files for bootstrap";
+  [ -n "${environment_arg}" ] && readonly environment="${environment_arg}";
 else
   # Validate component to work with
   [ -n "${component_arg}" ] \
@@ -536,13 +561,8 @@ The following input variables appear to be duplicated:
 
 ${duplicate_variables}
 
-This could lead to unexpected behaviour. Overriding of variables
-has previously been unpredictable and is not currently supported,
-but it may work.
-
-Recent changes to terraform might give you useful overriding and
-map-merging functionality, please use with caution and report back
-on your successes & failures.
+Ensure this is intentional behaviour, and that the order of
+precedence for variable values is as you expect.
 ###################################################################";
 
 # Build up the tfvars arguments for terraform command line
@@ -579,13 +599,25 @@ else
 fi;
 
 readonly backend_key="${backend_prefix}/${backend_filename}";
-readonly backend_config="terraform {
+declare backend_config
+if [ "${lock_table}" == "true" ]; then
+  backend_config="terraform {
+  backend \"s3\" {
+    region         = \"${region}\"
+    bucket         = \"${bucket}\"
+    key            = \"${backend_key}\"
+    dynamodb_table = \"${bucket}\"
+  }
+}";
+else
+  backend_config="terraform {
   backend \"s3\" {
     region = \"${region}\"
     bucket = \"${bucket}\"
     key    = \"${backend_key}\"
   }
 }";
+fi;
 
 # We're now all ready to go. All that's left is to:
 #   * Write the backend config
@@ -621,16 +653,21 @@ if [ "${bootstrapped}" == "true" ]; then
   # Nix the horrible hack on exit
   trap "rm -f $(pwd)/backend_tfscaffold.tf" EXIT;
 
+  declare lockfile_or_upgrade;
+  [ -n ${lockfile} ] && lockfile_or_upgrade='-upgrade' || lockfile_or_upgrade="${lockfile}";
+
   # Configure remote state storage
   echo "Setting up S3 remote state from s3://${bucket}/${backend_key}";
-  terraform init -upgrade ${no_color} ${compact_warnings} \
+  [ "${lock_table}" == "true" ] && echo "Using DynamoDB Table for state locking: ${bucket}";
+  terraform init ${no_color} ${compact_warnings} ${lockfile_or_upgrade} \
     || error_and_die "Terraform init failed";
 else
   # We are bootstrapping. Download the providers, skip the backend config.
   terraform init \
     -backend=false \
     ${no_color} \
     ${compact_warnings} \
+    ${lockfile} \
     || error_and_die "Terraform init failed";
 fi;
 
@@ -739,7 +776,7 @@ case "${action}" in
 
         # Push Terraform Remote State to S3
         # TODO: Add -upgrade to init when we drop support for <0.10
-        echo "yes" | terraform init || error_and_die "Terraform init failed";
+        echo "yes" | terraform init ${lockfile} || error_and_die "Terraform init failed";
 
         # Hard cleanup
         rm -f backend_tfscaffold.tf;