From e0db937b468b42ed32dd6540599e213038b29767 Mon Sep 17 00:00:00 2001 From: Andreas Falkenberg Date: Sun, 13 Aug 2023 10:56:49 +0200 Subject: [PATCH 1/6] fix: avoid overwritten truncated pipeline definitions --- .../process_deployment_map.py | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 0f538f113..3b3146ac2 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -12,6 +12,7 @@ from typing import Any, TypedDict import hashlib import yaml +import uuid from yaml.error import YAMLError import boto3 @@ -168,18 +169,34 @@ def start_executions( PIPELINE_MANAGEMENT_STATEMACHINE, run_id ) + truncated_pipeline_name_list = [] for pipeline in deployment_map.get("pipelines"): LOGGER.debug("Payload: %s", pipeline) full_pipeline_name = pipeline.get('name', 'no-pipeline-name') - # AWS Step Functions supports max 80 characters. - # Since the run_id equals 49 characters plus the dash, we have 30 - # characters available. To ensure we don't run over, lets use a - # truncated version concatenated with an hash generated from - # the pipeline name - truncated_pipeline_name = full_pipeline_name[:24] - name_bytes_to_hash = bytes(full_pipeline_name + run_id, 'utf-8') - execution_unique_hash = hashlib.md5(name_bytes_to_hash).hexdigest()[:5] - sfn_execution_name = f"{truncated_pipeline_name}-{execution_unique_hash}-{run_id}" + # Creation of unique StepFunction execution Ids: + # - AWS Step Functions execution id supports max 80 characters. + # - However, CodePipeline names can be up to 100 characters. + # That means CodePipeline names might need to be truncted. Truncating CodePipeline names + # can lead to pipelines having the same name. In that case only the first instance of the + # pipeline with the identical truncated name gets created. The other pipelines doesn't get created + # without any error or warning. + # To ensure all defined CodePipelines get created, we check that the truncated pipeline name + # is unique across all deployment maps. If duplicates are detected, we stop using the + # codepipeline_execution_id and generate a fresh one just for this pipeline. + # Truncation happens as follows: + # - Total max length: stepfunction execution id 80 characters + # - full_pipeline_name - Nativly, can be up to 100 characters. + # Gets truncated to 60 chars. + # - run_id (uuid) equals 49 characters nativly plus the dash. + # Gets truncated to 29 chars. + truncated_pipeline_name = full_pipeline_name[:60] + if len(full_pipeline_name) > 60 and truncated_pipeline_name in truncated_pipeline_name_list: + # Stop using codepipeline_execution_id and generarte fresh uuid to avoid collisions + run_id_new = str(uuid.uuid4()) + sfn_execution_name = f"{truncated_pipeline_name}-{run_id_new}"[:80] + else: + sfn_execution_name = f"{truncated_pipeline_name}-{run_id}"[:80] + truncated_pipeline_name_list.append(truncated_pipeline_name) sfn_client.start_execution( stateMachineArn=PIPELINE_MANAGEMENT_STATEMACHINE, name=sfn_execution_name, From bf98ba0c7345881a163852132217527159b5cbac Mon Sep 17 00:00:00 2001 From: Andreas Falkenberg Date: Sun, 13 Aug 2023 11:05:34 +0200 Subject: [PATCH 2/6] fix: typo --- .../pipeline_management/process_deployment_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 3b3146ac2..02ebca1b6 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -191,7 +191,7 @@ def start_executions( # Gets truncated to 29 chars. truncated_pipeline_name = full_pipeline_name[:60] if len(full_pipeline_name) > 60 and truncated_pipeline_name in truncated_pipeline_name_list: - # Stop using codepipeline_execution_id and generarte fresh uuid to avoid collisions + # Stop using codepipeline_execution_id and generate fresh uuid to avoid collisions run_id_new = str(uuid.uuid4()) sfn_execution_name = f"{truncated_pipeline_name}-{run_id_new}"[:80] else: From e4ee8709d76773f34ccf7987a2d37a14befa3814 Mon Sep 17 00:00:00 2001 From: Andreas Falkenberg Date: Fri, 15 Sep 2023 08:30:54 +0200 Subject: [PATCH 3/6] fix: refactor truncation method based on feedback --- .../process_deployment_map.py | 38 +++++++------------ 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 02ebca1b6..2b95c0938 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -12,7 +12,6 @@ from typing import Any, TypedDict import hashlib import yaml -import uuid from yaml.error import YAMLError import boto3 @@ -169,34 +168,23 @@ def start_executions( PIPELINE_MANAGEMENT_STATEMACHINE, run_id ) - truncated_pipeline_name_list = [] for pipeline in deployment_map.get("pipelines"): LOGGER.debug("Payload: %s", pipeline) full_pipeline_name = pipeline.get('name', 'no-pipeline-name') - # Creation of unique StepFunction execution Ids: - # - AWS Step Functions execution id supports max 80 characters. - # - However, CodePipeline names can be up to 100 characters. - # That means CodePipeline names might need to be truncted. Truncating CodePipeline names - # can lead to pipelines having the same name. In that case only the first instance of the - # pipeline with the identical truncated name gets created. The other pipelines doesn't get created - # without any error or warning. - # To ensure all defined CodePipelines get created, we check that the truncated pipeline name - # is unique across all deployment maps. If duplicates are detected, we stop using the - # codepipeline_execution_id and generate a fresh one just for this pipeline. - # Truncation happens as follows: - # - Total max length: stepfunction execution id 80 characters - # - full_pipeline_name - Nativly, can be up to 100 characters. - # Gets truncated to 60 chars. - # - run_id (uuid) equals 49 characters nativly plus the dash. - # Gets truncated to 29 chars. - truncated_pipeline_name = full_pipeline_name[:60] - if len(full_pipeline_name) > 60 and truncated_pipeline_name in truncated_pipeline_name_list: - # Stop using codepipeline_execution_id and generate fresh uuid to avoid collisions - run_id_new = str(uuid.uuid4()) - sfn_execution_name = f"{truncated_pipeline_name}-{run_id_new}"[:80] + # AWS Step Functions supports max 80 characters. + # Since the run_id equals 49 characters plus the dash, we have 30 + # characters available. To ensure we don't run over in case of 80+ + # characters, lets use a truncated version concatenated with an + # hash generated from the pipeline name. If below 80 characters, the + # full_pipeline_name + run_id is used. + sfn_execution_name_raw = f"{full_pipeline_name}-{run_id}" + if len(sfn_execution_name_raw) > 80: + truncated_pipeline_name = full_pipeline_name[:60] + name_bytes_to_hash = bytes(full_pipeline_name + run_id, 'utf-8') + execution_unique_hash = hashlib.md5(name_bytes_to_hash).hexdigest()[:5] + sfn_execution_name = f"{truncated_pipeline_name}-{execution_unique_hash}-{run_id}"[:80] else: - sfn_execution_name = f"{truncated_pipeline_name}-{run_id}"[:80] - truncated_pipeline_name_list.append(truncated_pipeline_name) + sfn_execution_name = f"{full_pipeline_name}-{run_id}" sfn_client.start_execution( stateMachineArn=PIPELINE_MANAGEMENT_STATEMACHINE, name=sfn_execution_name, From 9ace2b5ee7e759b9e80bba2118386211af35b629 Mon Sep 17 00:00:00 2001 From: AndyEfaa <73257849+AndyEfaa@users.noreply.github.com> Date: Wed, 4 Oct 2023 09:25:41 +0200 Subject: [PATCH 4/6] Update process_deployment_map.py Fix trailing space --- .../pipeline_management/process_deployment_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 2b95c0938..798106f09 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -174,7 +174,7 @@ def start_executions( # AWS Step Functions supports max 80 characters. # Since the run_id equals 49 characters plus the dash, we have 30 # characters available. To ensure we don't run over in case of 80+ - # characters, lets use a truncated version concatenated with an + # characters, lets use a truncated version concatenated with an # hash generated from the pipeline name. If below 80 characters, the # full_pipeline_name + run_id is used. sfn_execution_name_raw = f"{full_pipeline_name}-{run_id}" From 0825f673ce5de5b24c333067a46c9b063cb96055 Mon Sep 17 00:00:00 2001 From: AndyEfaa <73257849+AndyEfaa@users.noreply.github.com> Date: Thu, 5 Oct 2023 07:43:52 +0200 Subject: [PATCH 5/6] Update src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py Thank you. Makes sense Co-authored-by: Stewart Wallace --- .../pipeline_management/process_deployment_map.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 798106f09..04aa48985 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -177,14 +177,12 @@ def start_executions( # characters, lets use a truncated version concatenated with an # hash generated from the pipeline name. If below 80 characters, the # full_pipeline_name + run_id is used. - sfn_execution_name_raw = f"{full_pipeline_name}-{run_id}" - if len(sfn_execution_name_raw) > 80: + sfn_execution_name = f"{full_pipeline_name}-{run_id}" + if len(sfn_execution_name) > 80: truncated_pipeline_name = full_pipeline_name[:60] name_bytes_to_hash = bytes(full_pipeline_name + run_id, 'utf-8') execution_unique_hash = hashlib.md5(name_bytes_to_hash).hexdigest()[:5] sfn_execution_name = f"{truncated_pipeline_name}-{execution_unique_hash}-{run_id}"[:80] - else: - sfn_execution_name = f"{full_pipeline_name}-{run_id}" sfn_client.start_execution( stateMachineArn=PIPELINE_MANAGEMENT_STATEMACHINE, name=sfn_execution_name, From f76dd3174a1743a3608f380d424f6812621818d9 Mon Sep 17 00:00:00 2001 From: Andreas Falkenberg Date: Tue, 31 Oct 2023 16:57:18 +0100 Subject: [PATCH 6/6] fix: consider comment execution_unique_hash --- .../pipeline_management/process_deployment_map.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py index 04aa48985..7cdf94251 100644 --- a/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py +++ b/src/lambda_codebase/initial_commit/bootstrap_repository/adf-bootstrap/deployment/lambda_codebase/pipeline_management/process_deployment_map.py @@ -180,7 +180,7 @@ def start_executions( sfn_execution_name = f"{full_pipeline_name}-{run_id}" if len(sfn_execution_name) > 80: truncated_pipeline_name = full_pipeline_name[:60] - name_bytes_to_hash = bytes(full_pipeline_name + run_id, 'utf-8') + name_bytes_to_hash = bytes(full_pipeline_name, 'utf-8') execution_unique_hash = hashlib.md5(name_bytes_to_hash).hexdigest()[:5] sfn_execution_name = f"{truncated_pipeline_name}-{execution_unique_hash}-{run_id}"[:80] sfn_client.start_execution(