diff --git a/cloudformation/ds_admin_detective.yaml b/cloudformation/ds_admin_detective.yaml index bf03cb5..df0cee9 100644 --- a/cloudformation/ds_admin_detective.yaml +++ b/cloudformation/ds_admin_detective.yaml @@ -17,7 +17,6 @@ Resources: Principal: Service: lambda.amazonaws.com Action: 'sts:AssumeRole' - RoleName: !Sub '${StackSetName}-SageMaker-DetectiveControl-Role' Policies: - PolicyName: LambdaInlineForSageMaker PolicyDocument: diff --git a/cloudformation/ds_admin_principals.yaml b/cloudformation/ds_admin_principals.yaml index b9dae72..70c2b71 100644 --- a/cloudformation/ds_admin_principals.yaml +++ b/cloudformation/ds_admin_principals.yaml @@ -67,6 +67,7 @@ Resources: Effect: Allow Action: - 'cloudformation:CreateStack' + - 'cloudformation:CreateChangeSet' - 'cloudformation:DeleteStack' - 'cloudformation:DescribeStacks' - 'cloudformation:DescribeStackEvents' @@ -130,6 +131,7 @@ Resources: - 'kms:UpdateAlias' - 'kms:UpdateCustomKeyStore' - 'kms:UpdateKeyDescription' + - 'lambda:InvokeFunction' - 'resource-groups:CreateGroup' - 'resource-groups:DeleteGroup' - 'resource-groups:Tag' diff --git a/cloudformation/ds_administration.yaml b/cloudformation/ds_administration.yaml index 3048ead..99add70 100644 --- a/cloudformation/ds_administration.yaml +++ b/cloudformation/ds_administration.yaml @@ -73,6 +73,11 @@ Resources: ProductId: !Ref DSEnvironmentProduct RoleArn: !GetAtt DataSciencePrincipals.Outputs.SCLaunchRoleArn + CfnPyPlateMacro: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: pyplate_macro.yaml + DataSciencePrincipals: Type: AWS::CloudFormation::Stack Properties: diff --git a/cloudformation/ds_env_backing_store.yaml b/cloudformation/ds_env_backing_store.yaml index 54a9a21..fa3b6eb 100644 --- a/cloudformation/ds_env_backing_store.yaml +++ b/cloudformation/ds_env_backing_store.yaml @@ -5,10 +5,14 @@ Description: Data Science Environment S3 data storage Parameters: + NaturalProjectName: + Type: String + Description: Natural name of the project team. + ProjectName: Type: String - AllowedPattern: '[A-Za-z0-9\-]*' - Description: Please specify your project name. Used as a suffix for project resource names. + AllowedPattern: '[a-z0-9\-]*' + Description: Project team name with no spaces or uppercase letters. EnvType: Description: System Environment @@ -93,7 +97,7 @@ Resources: !Sub "ds-s3-endpoint-${ProjectName}-${EnvType}-id" Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -116,7 +120,7 @@ Resources: DataBucket: Type: 'AWS::S3::Bucket' Properties: - BucketName: + BucketName: !Join - '' - @@ -146,7 +150,7 @@ Resources: KMSMasterKeyID: !Ref KMSCMK Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -204,7 +208,7 @@ Resources: KMSMasterKeyID: !Ref KMSCMK Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -240,6 +244,6 @@ Resources: Key: '< S3_CFN_STAGING_PATH >/project_template.zip' Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType diff --git a/cloudformation/ds_env_catalog.yaml b/cloudformation/ds_env_catalog.yaml index be7198d..ffa5936 100644 --- a/cloudformation/ds_env_catalog.yaml +++ b/cloudformation/ds_env_catalog.yaml @@ -2,10 +2,14 @@ Description: | Template to create a service catalog product to launch a notebook Parameters: + NaturalProjectName: + Type: String + Description: Natural name of the project team. + ProjectName: Type: String - AllowedPattern: '[A-Za-z0-9\-]*' - Description: Please specify your project name. Used as a suffix for project resource names. + AllowedPattern: '[a-z0-9\-]*' + Description: Project team name with no spaces or uppercase letters. EnvType: Description: System Environment @@ -29,15 +33,15 @@ Resources: ProviderName: !Sub '${ProjectName} Administration' Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType DSUserNotebookProduct: Type: AWS::ServiceCatalog::CloudFormationProduct Properties: - Description: !Sub 'SageMaker ${EnvType} notebook for the ${ProjectName} project' - Name: !Sub '${ProjectName} SageMaker ${EnvType} notebook' + Description: !Sub 'SageMaker ${EnvType} notebook for the ${NaturalProjectName} project' + Name: !Sub '${NaturalProjectName} SageMaker ${EnvType} notebook' Owner: 'Data Science CoE' ProvisioningArtifactParameters: - Name: 'DS User Notebook v1' @@ -155,7 +159,7 @@ Resources: - 'arn:aws:iam::aws:policy/AWSLambdaFullAccess' Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType diff --git a/cloudformation/ds_env_network.yaml b/cloudformation/ds_env_network.yaml index f03b61b..08cb5fa 100644 --- a/cloudformation/ds_env_network.yaml +++ b/cloudformation/ds_env_network.yaml @@ -2,10 +2,14 @@ Description: | Create a secure VPC designed to host a data science project team. Parameters: + NaturalProjectName: + Type: String + Description: Natural name of the project team. + ProjectName: Type: String - AllowedPattern: '[A-Za-z0-9\-]*' - Description: Please specify your project name. Used as a suffix for project resource names. + AllowedPattern: '[a-z0-9\-]*' + Description: Project team name with no spaces or uppercase letters. EnvType: Description: System Environment @@ -93,7 +97,7 @@ Resources: - Key: Name Value: !Sub "ds-vpc-${ProjectName}-${EnvType}" - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -113,7 +117,7 @@ Resources: AvailabilityZone: !Sub "${AWS::Region}a" Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -133,7 +137,7 @@ Resources: AvailabilityZone: !Sub "${AWS::Region}b" Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -153,7 +157,7 @@ Resources: AvailabilityZone: !Sub "${AWS::Region}c" Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -177,7 +181,7 @@ Resources: VpcId: !Ref SageMakerVPC Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -217,7 +221,7 @@ Resources: SourceSecurityGroupId: !GetAtt SageMakerSecurityGroup.GroupId Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -228,7 +232,7 @@ Resources: VpcId: !Ref SageMakerVPC Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -252,7 +256,7 @@ Resources: SourceSecurityGroupId: !GetAtt SageMakerSecurityGroup.GroupId Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType diff --git a/cloudformation/ds_env_principals.yaml b/cloudformation/ds_env_principals.yaml index 27cf2f0..cd75784 100644 --- a/cloudformation/ds_env_principals.yaml +++ b/cloudformation/ds_env_principals.yaml @@ -2,10 +2,14 @@ Description: | Template to create IAM principals for operation within the data science environment. Parameters: + NaturalProjectName: + Type: String + Description: Natural name of the project team. + ProjectName: Type: String - AllowedPattern: '[A-Za-z0-9\-]*' - Description: Please specify your project name. Used as a suffix for project resource names. + AllowedPattern: '[a-z0-9\-]*' + Description: Project team name with no spaces or uppercase letters. EnvType: Description: System Environment @@ -67,7 +71,7 @@ Resources: - 'arn:aws:iam::aws:policy/AWSCodeCommitFullAccess' Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType @@ -148,6 +152,7 @@ Resources: - 'sagemaker:DescribeCodeRepository' - 'sagemaker:ListEndpoints' - 'sagemaker:ListCodeRepositories' + - 'sagemaker:ListProcessingJobs' - 'codecommit:BatchGetRepositories' - 'codecommit:GitPull' - 'codecommit:GitPush' @@ -176,7 +181,7 @@ Resources: - 'arn:aws:iam::aws:policy/AWSCodeCommitReadOnly' Tags: - Key: ProjectName - Value: !Ref ProjectName + Value: !Ref NaturalProjectName - Key: EnvironmentType Value: !Ref EnvType diff --git a/cloudformation/ds_env_sagemaker.yaml b/cloudformation/ds_env_sagemaker.yaml index 2aadbc4..4550a75 100644 --- a/cloudformation/ds_env_sagemaker.yaml +++ b/cloudformation/ds_env_sagemaker.yaml @@ -5,10 +5,14 @@ Description: SageMaker specific resources for Data Science Environment Parameters: + NaturalProjectName: + Type: String + Description: Natural name of the project team. + ProjectName: Type: String - AllowedPattern: '[A-Za-z0-9\-]*' - Description: Please specify your Team Name. Used as a suffix for team resource names + AllowedPattern: '[a-z0-9\-]*' + Description: Project team name with no spaces or uppercase letters. EnvType: Description: >- diff --git a/cloudformation/ds_environment.yaml b/cloudformation/ds_environment.yaml index 4d17c70..5af6b7e 100644 --- a/cloudformation/ds_environment.yaml +++ b/cloudformation/ds_environment.yaml @@ -4,10 +4,11 @@ # Lastly the template stores outputs into Parameter Store so they can be referenced later by SC products. Description: Data Science Environment +Transform: [PyPlate] + Parameters: ProjectName: Type: String - AllowedPattern: '[A-Za-z0-9\-]*' Description: Please specify your Team Name. Used as a suffix for team resource names EnvType: @@ -55,8 +56,12 @@ Resources: DSEnvironmentNetwork: Type: AWS::CloudFormation::Stack Properties: + TemplateURL: ds_env_network.yaml Parameters: - ProjectName: !Ref ProjectName + NaturalProjectName: !Ref ProjectName + ProjectName: | + #!PyPlate + output = params['ProjectName'][:26].lower().replace(' ', '-') EnvType: !Ref EnvType VpcCIDR: !Ref VpcCIDR Subnet1CIDR: !Ref Subnet1CIDR @@ -68,7 +73,6 @@ Resources: - ENDPOINT_ID: Fn::ImportValue: !Sub 'srv-endsrv-pypimirror-${SharedServiceStackSetName}' - TemplateURL: ds_env_network.yaml Tags: - Key: ProjectName Value: !Ref ProjectName @@ -78,10 +82,13 @@ Resources: DSEnvironmentPrincipals: Type: AWS::CloudFormation::Stack Properties: + TemplateURL: ds_env_principals.yaml Parameters: - ProjectName: !Ref ProjectName + NaturalProjectName: !Ref ProjectName + ProjectName: | + #!PyPlate + output = params['ProjectName'][:26].lower().replace(' ', '-') EnvType: !Ref EnvType - TemplateURL: ds_env_principals.yaml Tags: - Key: ProjectName Value: !Ref ProjectName @@ -94,10 +101,13 @@ Resources: - DSEnvironmentNetwork - DSEnvironmentPrincipals Properties: + TemplateURL: ds_env_backing_store.yaml Parameters: - ProjectName: !Ref ProjectName + NaturalProjectName: !Ref ProjectName + ProjectName: | + #!PyPlate + output = params['ProjectName'][:26].lower().replace(' ', '-') EnvType: !Ref EnvType - TemplateURL: ds_env_backing_store.yaml Tags: - Key: ProjectName Value: !Ref ProjectName @@ -108,14 +118,16 @@ Resources: Type: AWS::CloudFormation::Stack DependsOn: DSEnvironmentPrincipals Properties: + TemplateURL: ds_env_catalog.yaml Parameters: - ProjectName: !Ref ProjectName + NaturalProjectName: !Ref ProjectName + ProjectName: | + #!PyPlate + output = params['ProjectName'][:26].lower().replace(' ', '-') EnvType: !Ref EnvType QuickstartMode: Fn::ImportValue: !Sub '${SharedServiceStackSetName}-QuickstartMode' - - TemplateURL: ds_env_catalog.yaml Tags: - Key: ProjectName Value: !Ref ProjectName @@ -126,10 +138,13 @@ Resources: Type: AWS::CloudFormation::Stack DependsOn: DSEnvironmentBackingStore Properties: + TemplateURL: ds_env_sagemaker.yaml Parameters: - ProjectName: !Ref ProjectName + NaturalProjectName: !Ref ProjectName + ProjectName: | + #!PyPlate + output = params['ProjectName'][:26].lower().replace(' ', '-') EnvType: !Ref EnvType - TemplateURL: ds_env_sagemaker.yaml Tags: - Key: ProjectName Value: !Ref ProjectName @@ -139,7 +154,17 @@ Resources: DSEnvironmentResourceGroup: Type: "AWS::ResourceGroups::Group" Properties: - Name: !Sub 'ds-${ProjectName}-${EnvType}-resource-group' + Name: + 'Fn::Join': + - "" + - + - "ds-" + - | + #!PyPlate + output = params['ProjectName'][:26].lower().replace(' ', '-') + - "-" + - !Ref EnvType + - "-resource-group" Description: !Sub 'AWS Resources belonging to ${ProjectName} in its ${EnvType} environment.' ResourceQuery: Type: "TAG_FILTERS_1_0" diff --git a/cloudformation/pyplate_macro.yaml b/cloudformation/pyplate_macro.yaml new file mode 100644 index 0000000..5e59a05 --- /dev/null +++ b/cloudformation/pyplate_macro.yaml @@ -0,0 +1,84 @@ +AWSTemplateFormatVersion: 2010-09-09 +Resources: + TransformExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: [lambda.amazonaws.com] + Action: ['sts:AssumeRole'] + Path: / + Policies: + - PolicyName: root + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: ['logs:*'] + Resource: 'arn:aws:logs:*:*:*' + TransformFunction: + Type: AWS::Lambda::Function + Properties: + Code: + ZipFile: | + import traceback + import json + + + def obj_iterate(obj, params): + if isinstance(obj, dict): + for k in obj: + obj[k] = obj_iterate(obj[k], params) + elif isinstance(obj, list): + for i, v in enumerate(obj): + obj[i] = obj_iterate(v, params) + elif isinstance(obj, str): + if obj.startswith("#!PyPlate"): + params['output'] = None + exec(obj, params) + obj = params['output'] + return obj + + + def handler(event, context): + + print(json.dumps(event)) + + macro_response = { + "requestId": event["requestId"], + "status": "success" + } + try: + params = { + "params": event["templateParameterValues"], + "template": event["fragment"], + "account_id": event["accountId"], + "region": event["region"] + } + response = event["fragment"] + macro_response["fragment"] = obj_iterate(response, params) + except Exception as e: + traceback.print_exc() + macro_response["status"] = "failure" + macro_response["errorMessage"] = str(e) + return macro_response + + Handler: index.handler + Runtime: python3.6 + Role: !GetAtt TransformExecutionRole.Arn + TransformFunctionPermissions: + Type: AWS::Lambda::Permission + Properties: + Action: 'lambda:InvokeFunction' + FunctionName: !GetAtt TransformFunction.Arn + Principal: 'cloudformation.amazonaws.com' + Transform: + Type: AWS::CloudFormation::Macro + Properties: + Name: !Sub 'PyPlate' + Description: Processes inline python in templates + FunctionName: !GetAtt TransformFunction.Arn + diff --git a/codebuild_local_readme.md b/codebuild_local_readme.md index 216745c..fe370bf 100644 --- a/codebuild_local_readme.md +++ b/codebuild_local_readme.md @@ -25,7 +25,7 @@ docker pull ubuntu To run CodeBuild locally use the shell script `codebuild_local.sh` to kick things off: ```bash -./codebuild_local.sh -i 'ubuntu:latest' -c -a /tmp -s . +./codebuild_build.sh -i 'ubuntu:latest' -c -a /tmp -s . ``` This will install any dependencies such as zip, Python pip, and the AWS CLI. It will then execute the `package_cloudformation.sh` diff --git a/package_cloudformation.sh b/package_cloudformation.sh index c7676c0..f467f4f 100755 --- a/package_cloudformation.sh +++ b/package_cloudformation.sh @@ -101,6 +101,8 @@ echo "Packaging Cloudformation templates..." for fname in ${AWS_PACKAGE_LIST}; do pushd ${TMP_OUTPUT_DIR} + echo "Packaging ${fname}..." + aws cloudformation validate-template --template-body file://${fname} aws cloudformation package \ --template-file ${fname} \ --s3-bucket ${CFN_BUCKET_NAME} \