diff --git a/sdk/python/test/compilation/pipeline_compilation_test.py b/sdk/python/test/compilation/pipeline_compilation_test.py index d7779637387..4b0377a82ce 100644 --- a/sdk/python/test/compilation/pipeline_compilation_test.py +++ b/sdk/python/test/compilation/pipeline_compilation_test.py @@ -66,6 +66,8 @@ flipcoin_pipeline as flip_coin from test_data.sdk_compiled_pipelines.valid.critical.loop_consume_upstream import \ loop_consume_upstream +from test_data.sdk_compiled_pipelines.valid.critical.missing_kubernetes_optional_inputs import \ + missing_kubernetes_optional_inputs_pipeline from test_data.sdk_compiled_pipelines.valid.critical.mixed_parameters import \ crust as mixed_parameters_pipeline from test_data.sdk_compiled_pipelines.valid.critical.modelcar.modelcar import \ @@ -353,6 +355,13 @@ def __repr__(self) -> str: compiled_file_name='parameter_oneof.yaml', expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/critical/parameter_oneof.yaml' ), + TestData( + pipeline_name='missing-kubernetes-optional-inputs', + pipeline_func=missing_kubernetes_optional_inputs_pipeline, + pipline_func_args=None, + compiled_file_name='missing_kubernetes_optional_inputs.yaml', + expected_compiled_file_path=f'{_VALID_PIPELINE_FILES}/critical/missing_kubernetes_optional_inputs.yaml' + ), TestData( pipeline_name='pipeline-with-env', pipeline_func=env_pipeline, diff --git a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml index de64ad7156e..74e6976c9af 100644 --- a/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml +++ b/test_data/compiled-workflows/missing_kubernetes_optional_inputs.yaml @@ -6,52 +6,24 @@ metadata: spec: arguments: parameters: - - name: kubernetes-comp-log-message-2 - value: '{"nodeAffinity":[{"nodeAffinityJson":{"componentInputParameter":"optional_affinity"}}]}' - - name: components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8 - value: '{"executorLabel":"exec-log-message-2","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' - - name: implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8 + - name: kubernetes-comp-log-message + value: '{"configMapAsEnv":[{"configMapNameParameter":{"componentInputParameter":"optional_config_map_name"},"keyToEnv":[{"configMapKey":"setting","envVar":"SETTING"}],"optional":true}],"configMapAsVolume":[{"configMapNameParameter":{"componentInputParameter":"optional_config_map_name"},"mountPath":"/mnt/config","optional":true}],"imagePullSecret":[{"secretNameParameter":{"componentInputParameter":"optional_image_pull_secret_name"}}],"nodeAffinity":[{"nodeAffinityJson":{"componentInputParameter":"optional_affinity"}}],"nodeSelector":{"nodeSelectorJson":{"componentInputParameter":"optional_selector"}},"secretAsEnv":[{"keyToEnv":[{"envVar":"PASSWORD","secretKey":"password"}],"optional":true,"secretNameParameter":{"componentInputParameter":"optional_secret_name"}}],"secretAsVolume":[{"mountPath":"/mnt/secret","optional":true,"secretNameParameter":{"componentInputParameter":"optional_secret_name"}}],"tolerations":[{"tolerationJson":{"componentInputParameter":"optional_tolerations"}}]}' + - name: components-96322856eafd7d43227888c8781b0302ec66e8a5befa15af46a259fdf0571c35 + value: '{"executorLabel":"exec-log-message","inputDefinitions":{"parameters":{"message":{"parameterType":"STRING"}}}}' + - name: implementations-96322856eafd7d43227888c8781b0302ec66e8a5befa15af46a259fdf0571c35 value: '{"args":["--executor_input","{{$}}","--function_to_execute","log_message"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 - python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.4'' ''--no-deps'' + python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.6'' ''--no-deps'' ''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef - log_message(message: str) -\u003e None:\n print(message)\n\n"],"image":"python:3.9"}' - - name: components-comp-condition-1 - value: '{"dag":{"tasks":{"log-message-2":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node - affinity applied"}}}},"taskInfo":{"name":"log-message-2"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_affinity":{"parameterType":"BOOLEAN"}}}}' - - name: kubernetes-comp-log-message-3 - value: '{"nodeSelector":{"nodeSelectorJson":{"componentInputParameter":"optional_selector"}}}' - - name: components-comp-condition-2 - value: '{"dag":{"tasks":{"log-message-3":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-3"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node - selector applied"}}}},"taskInfo":{"name":"log-message-3"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_selector":{"parameterType":"BOOLEAN"}}}}' - - name: kubernetes-comp-log-message-4 - value: '{"secretAsEnv":[{"keyToEnv":[{"envVar":"PASSWORD","secretKey":"password"}],"secretNameParameter":{"componentInputParameter":"optional_secret_name"}}]}' - - name: components-comp-condition-3 - value: '{"dag":{"tasks":{"log-message-4":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-4"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"secret - env applied"}}}},"taskInfo":{"name":"log-message-4"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_secret_env":{"parameterType":"BOOLEAN"}}}}' - - name: kubernetes-comp-log-message-5 - value: '{"configMapAsEnv":[{"configMapNameParameter":{"componentInputParameter":"optional_config_map_name"},"keyToEnv":[{"configMapKey":"setting","envVar":"SETTING"}]}]}' - - name: components-comp-condition-4 - value: '{"dag":{"tasks":{"log-message-5":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-5"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"configmap - env applied"}}}},"taskInfo":{"name":"log-message-5"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_config_map_env":{"parameterType":"BOOLEAN"}}}}' - - name: kubernetes-comp-log-message-6 - value: '{"tolerations":[{"tolerationJson":{"componentInputParameter":"optional_tolerations"}}]}' - - name: components-comp-condition-5 - value: '{"dag":{"tasks":{"log-message-6":{"cachingOptions":{},"componentRef":{"name":"comp-log-message-6"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"tolerations - applied"}}}},"taskInfo":{"name":"log-message-6"}}}},"inputDefinitions":{"parameters":{"pipelinechannel--enable_tolerations":{"parameterType":"BOOLEAN"}}}}' + log_message(message: str) -\u003e None:\n print(message)\n\n"],"image":"python:3.11"}' - name: components-root - value: '{"dag":{"tasks":{"condition-1":{"componentRef":{"name":"comp-condition-1"},"inputs":{"parameters":{"pipelinechannel--enable_affinity":{"componentInputParameter":"enable_affinity"}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_affinity''] - == true"}},"condition-2":{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--enable_selector":{"componentInputParameter":"enable_selector"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_selector''] - == true"}},"condition-3":{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--enable_secret_env":{"componentInputParameter":"enable_secret_env"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_secret_env''] - == true"}},"condition-4":{"componentRef":{"name":"comp-condition-4"},"inputs":{"parameters":{"pipelinechannel--enable_config_map_env":{"componentInputParameter":"enable_config_map_env"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_config_map_env''] - == true"}},"condition-5":{"componentRef":{"name":"comp-condition-5"},"inputs":{"parameters":{"pipelinechannel--enable_tolerations":{"componentInputParameter":"enable_tolerations"}}},"taskInfo":{"name":"condition-5"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_tolerations''] - == true"}},"log-message":{"cachingOptions":{},"componentRef":{"name":"comp-log-message"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baseline - task"}}}},"taskInfo":{"name":"log-message"}}}},"inputDefinitions":{"parameters":{"enable_affinity":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_config_map_env":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_secret_env":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_selector":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"enable_tolerations":{"defaultValue":false,"isOptional":true,"parameterType":"BOOLEAN"},"optional_affinity":{"isOptional":true,"parameterType":"STRUCT"},"optional_config_map_name":{"isOptional":true,"parameterType":"STRING"},"optional_secret_name":{"isOptional":true,"parameterType":"STRING"},"optional_selector":{"isOptional":true,"parameterType":"STRUCT"},"optional_tolerations":{"isOptional":true,"parameterType":"LIST"}}}}' + value: '{"dag":{"tasks":{"log-message":{"cachingOptions":{},"componentRef":{"name":"comp-log-message"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baseline + task"}}}},"taskInfo":{"name":"log-message"}}}},"inputDefinitions":{"parameters":{"optional_affinity":{"isOptional":true,"parameterType":"STRUCT"},"optional_config_map_name":{"isOptional":true,"parameterType":"STRING"},"optional_image_pull_secret_name":{"isOptional":true,"parameterType":"STRING"},"optional_secret_name":{"isOptional":true,"parameterType":"STRING"},"optional_selector":{"isOptional":true,"parameterType":"STRUCT"},"optional_tolerations":{"isOptional":true,"parameterType":"LIST"}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -232,175 +204,35 @@ spec: - arguments: parameters: - name: component - value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + value: '{{workflow.parameters.components-96322856eafd7d43227888c8781b0302ec66e8a5befa15af46a259fdf0571c35}}' - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-2"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node - affinity applied"}}}},"taskInfo":{"name":"log-message-2"}}' - - name: container - value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task-name - value: log-message-2 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-log-message-2}}' - name: log-message-2-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.log-message-2-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.log-message-2-driver.outputs.parameters.cached-decision}}' - depends: log-message-2-driver.Succeeded - name: log-message-2 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-condition-1 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-3"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"node - selector applied"}}}},"taskInfo":{"name":"log-message-3"}}' - - name: container - value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task-name - value: log-message-3 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-log-message-3}}' - name: log-message-3-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.log-message-3-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.log-message-3-driver.outputs.parameters.cached-decision}}' - depends: log-message-3-driver.Succeeded - name: log-message-3 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-condition-2 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-4"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"secret - env applied"}}}},"taskInfo":{"name":"log-message-4"}}' - - name: container - value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task-name - value: log-message-4 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-log-message-4}}' - name: log-message-4-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.log-message-4-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.log-message-4-driver.outputs.parameters.cached-decision}}' - depends: log-message-4-driver.Succeeded - name: log-message-4 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-condition-3 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-5"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"configmap - env applied"}}}},"taskInfo":{"name":"log-message-5"}}' - - name: container - value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task-name - value: log-message-5 - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-log-message-5}}' - name: log-message-5-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.log-message-5-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.log-message-5-driver.outputs.parameters.cached-decision}}' - depends: log-message-5-driver.Succeeded - name: log-message-5 - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: comp-condition-4 - outputs: {} - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message-6"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"tolerations - applied"}}}},"taskInfo":{"name":"log-message-6"}}' + value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baseline + task"}}}},"taskInfo":{"name":"log-message"}}' - name: container - value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' + value: '{{workflow.parameters.implementations-96322856eafd7d43227888c8781b0302ec66e8a5befa15af46a259fdf0571c35}}' - name: task-name - value: log-message-6 + value: log-message - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - name: kubernetes-config - value: '{{workflow.parameters.kubernetes-comp-log-message-6}}' - name: log-message-6-driver + value: '{{workflow.parameters.kubernetes-comp-log-message}}' + name: log-message-driver template: system-container-driver - arguments: parameters: - name: pod-spec-patch - value: '{{tasks.log-message-6-driver.outputs.parameters.pod-spec-patch}}' + value: '{{tasks.log-message-driver.outputs.parameters.pod-spec-patch}}' - default: "false" name: cached-decision - value: '{{tasks.log-message-6-driver.outputs.parameters.cached-decision}}' - depends: log-message-6-driver.Succeeded - name: log-message-6 + value: '{{tasks.log-message-driver.outputs.parameters.cached-decision}}' + depends: log-message-driver.Succeeded + name: log-message template: system-container-executor inputs: parameters: - name: parent-dag-id metadata: {} - name: comp-condition-5 + name: root outputs: {} - container: args: @@ -479,154 +311,6 @@ spec: valueFrom: default: "true" path: /tmp/outputs/condition - - dag: - tasks: - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-condition-1}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-1"},"inputs":{"parameters":{"pipelinechannel--enable_affinity":{"componentInputParameter":"enable_affinity"}}},"taskInfo":{"name":"condition-1"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_affinity''] - == true"}}' - - name: task-name - value: condition-1 - name: condition-1-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.condition-1-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.condition-1-driver.outputs.parameters.condition}}' - depends: condition-1-driver.Succeeded - name: condition-1 - template: comp-condition-1 - when: '{{tasks.condition-1-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-condition-2}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-2"},"inputs":{"parameters":{"pipelinechannel--enable_selector":{"componentInputParameter":"enable_selector"}}},"taskInfo":{"name":"condition-2"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_selector''] - == true"}}' - - name: task-name - value: condition-2 - name: condition-2-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.condition-2-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.condition-2-driver.outputs.parameters.condition}}' - depends: condition-2-driver.Succeeded - name: condition-2 - template: comp-condition-2 - when: '{{tasks.condition-2-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-condition-3}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-3"},"inputs":{"parameters":{"pipelinechannel--enable_secret_env":{"componentInputParameter":"enable_secret_env"}}},"taskInfo":{"name":"condition-3"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_secret_env''] - == true"}}' - - name: task-name - value: condition-3 - name: condition-3-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.condition-3-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.condition-3-driver.outputs.parameters.condition}}' - depends: condition-3-driver.Succeeded - name: condition-3 - template: comp-condition-3 - when: '{{tasks.condition-3-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-condition-4}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-4"},"inputs":{"parameters":{"pipelinechannel--enable_config_map_env":{"componentInputParameter":"enable_config_map_env"}}},"taskInfo":{"name":"condition-4"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_config_map_env''] - == true"}}' - - name: task-name - value: condition-4 - name: condition-4-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.condition-4-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.condition-4-driver.outputs.parameters.condition}}' - depends: condition-4-driver.Succeeded - name: condition-4 - template: comp-condition-4 - when: '{{tasks.condition-4-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-comp-condition-5}}' - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - - name: task - value: '{"componentRef":{"name":"comp-condition-5"},"inputs":{"parameters":{"pipelinechannel--enable_tolerations":{"componentInputParameter":"enable_tolerations"}}},"taskInfo":{"name":"condition-5"},"triggerPolicy":{"condition":"inputs.parameter_values[''pipelinechannel--enable_tolerations''] - == true"}}' - - name: task-name - value: condition-5 - name: condition-5-driver - template: system-dag-driver - - arguments: - parameters: - - name: parent-dag-id - value: '{{tasks.condition-5-driver.outputs.parameters.execution-id}}' - - name: condition - value: '{{tasks.condition-5-driver.outputs.parameters.condition}}' - depends: condition-5-driver.Succeeded - name: condition-5 - template: comp-condition-5 - when: '{{tasks.condition-5-driver.outputs.parameters.condition}} != false' - - arguments: - parameters: - - name: component - value: '{{workflow.parameters.components-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task - value: '{"cachingOptions":{},"componentRef":{"name":"comp-log-message"},"inputs":{"parameters":{"message":{"runtimeValue":{"constant":"baseline - task"}}}},"taskInfo":{"name":"log-message"}}' - - name: container - value: '{{workflow.parameters.implementations-c8c67f3fc917440660bebc9bdeebf74582aea2a34af0448b655b50197afff8f8}}' - - name: task-name - value: log-message - - name: parent-dag-id - value: '{{inputs.parameters.parent-dag-id}}' - name: log-message-driver - template: system-container-driver - - arguments: - parameters: - - name: pod-spec-patch - value: '{{tasks.log-message-driver.outputs.parameters.pod-spec-patch}}' - - default: "false" - name: cached-decision - value: '{{tasks.log-message-driver.outputs.parameters.cached-decision}}' - depends: log-message-driver.Succeeded - name: log-message - template: system-container-executor - inputs: - parameters: - - name: parent-dag-id - metadata: {} - name: root - outputs: {} - dag: tasks: - arguments: @@ -634,7 +318,7 @@ spec: - name: component value: '{{workflow.parameters.components-root}}' - name: runtime-config - value: '{"parameterValues":{"enable_affinity":false,"enable_config_map_env":false,"enable_secret_env":false,"enable_selector":false,"enable_tolerations":false,"optional_affinity":null,"optional_config_map_name":null,"optional_secret_name":null,"optional_selector":null,"optional_tolerations":null}}' + value: '{"parameterValues":{"optional_affinity":null,"optional_config_map_name":null,"optional_image_pull_secret_name":null,"optional_secret_name":null,"optional_selector":null,"optional_tolerations":null}}' - name: driver-type value: ROOT_DAG name: root-driver diff --git a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml index 5efc8f18da4..6c85dbbf31c 100644 --- a/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml +++ b/test_data/compiled-workflows/pipeline_with_string_machine_fields_task_output.yaml @@ -72,7 +72,7 @@ spec: kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sum_numbers(a: int, b: int) -\u003e int:\n return a + b\n\n"],"image":"python:3.11","resources":{"accelerator":{"resourceCount":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}","resourceType":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"},"resourceCpuLimit":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}","resourceMemoryLimit":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}}' - name: components-root - value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","accelerator-type","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' + value: '{"dag":{"tasks":{"accelerator-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-limit"},"taskInfo":{"name":"accelerator-limit"}},"accelerator-type":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-accelerator-type"},"taskInfo":{"name":"accelerator-type"}},"cpu-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-cpu-limit"},"taskInfo":{"name":"cpu-limit"}},"memory-limit":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-memory-limit"},"taskInfo":{"name":"memory-limit"}},"sum-numbers":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["memory-limit","accelerator-type","cpu-limit","accelerator-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}}}}' entrypoint: entrypoint podMetadata: annotations: @@ -351,15 +351,15 @@ spec: - name: component value: '{{workflow.parameters.components-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task - value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["accelerator-limit","cpu-limit","accelerator-type","memory-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' + value: '{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sum-numbers"},"dependentTasks":["memory-limit","accelerator-type","cpu-limit","accelerator-limit"],"inputs":{"parameters":{"a":{"runtimeValue":{"constant":1}},"accelerator_count":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-limit-Output'']}}"}},"accelerator_type":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--accelerator-type-Output'']}}"}},"b":{"runtimeValue":{"constant":2}},"cpu_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--cpu-limit-Output'']}}"}},"memory_limit":{"runtimeValue":{"constant":"{{$.inputs.parameters[''pipelinechannel--memory-limit-Output'']}}"}},"pipelinechannel--accelerator-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-limit"}},"pipelinechannel--accelerator-type-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"accelerator-type"}},"pipelinechannel--cpu-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"cpu-limit"}},"pipelinechannel--memory-limit-Output":{"taskOutputParameter":{"outputParameterKey":"Output","producerTask":"memory-limit"}}}},"taskInfo":{"name":"sum-numbers"}}' - name: container value: '{{workflow.parameters.implementations-49f9a898b718a077f30b7fd8c02d39767cff91ff0bbda4379daf866a91dbdb1b}}' - name: task-name value: sum-numbers - name: parent-dag-id value: '{{inputs.parameters.parent-dag-id}}' - depends: accelerator-limit.Succeeded && cpu-limit.Succeeded && accelerator-type.Succeeded - && memory-limit.Succeeded + depends: memory-limit.Succeeded && accelerator-type.Succeeded && cpu-limit.Succeeded + && accelerator-limit.Succeeded name: sum-numbers-driver template: system-container-driver - arguments: diff --git a/test_data/compiled-workflows/pipeline_with_workspace.yaml b/test_data/compiled-workflows/pipeline_with_workspace.yaml index 0f521fc9da8..77d1a90319e 100644 --- a/test_data/compiled-workflows/pipeline_with_workspace.yaml +++ b/test_data/compiled-workflows/pipeline_with_workspace.yaml @@ -384,11 +384,11 @@ spec: creationTimestamp: null name: kfp-workspace spec: + accessModes: + - ReadWriteOnce resources: requests: storage: 1Gi - accessModes: - - ReadWriteOnce storageClassName: standard status: {} status: diff --git a/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py index 82c2653fbcf..52605f64d43 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py +++ b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.py @@ -11,60 +11,60 @@ def log_message(message: str) -> None: @dsl.pipeline def missing_kubernetes_optional_inputs_pipeline( optional_affinity: Optional[Dict[str, Any]] = None, - enable_affinity: bool = False, optional_selector: Optional[Dict[str, Any]] = None, - enable_selector: bool = False, optional_secret_name: Optional[str] = None, - enable_secret_env: bool = False, optional_config_map_name: Optional[str] = None, - enable_config_map_env: bool = False, + optional_image_pull_secret_name: Optional[str] = None, optional_tolerations: Optional[List[Dict[str, Any]]] = None, - enable_tolerations: bool = False, ): - base_task = log_message(message="baseline task") - base_task.set_caching_options(enable_caching=False) + task = log_message(message="baseline task") + task.set_caching_options(enable_caching=False) - with dsl.If(enable_affinity == True): - affinity_task = log_message(message="node affinity applied") - affinity_task.set_caching_options(enable_caching=False) - kubernetes.add_node_affinity_json( - task=affinity_task, - node_affinity_json=optional_affinity, - ) + kubernetes.add_node_affinity_json( + task=task, + node_affinity_json=optional_affinity, + ) - with dsl.If(enable_selector == True): - selector_task = log_message(message="node selector applied") - selector_task.set_caching_options(enable_caching=False) - kubernetes.add_node_selector_json( - task=selector_task, - node_selector_json=optional_selector, - ) + kubernetes.add_node_selector_json( + task=task, + node_selector_json=optional_selector, + ) - with dsl.If(enable_secret_env == True): - secret_task = log_message(message="secret env applied") - secret_task.set_caching_options(enable_caching=False) - kubernetes.use_secret_as_env( - task=secret_task, - secret_name=optional_secret_name, - secret_key_to_env={"password": "PASSWORD"}, - ) + kubernetes.use_secret_as_env( + task=task, + secret_name=optional_secret_name, + secret_key_to_env={"password": "PASSWORD"}, + optional=True, + ) + kubernetes.use_secret_as_volume( + task=task, + secret_name=optional_secret_name, + mount_path="/mnt/secret", + optional=True, + ) - with dsl.If(enable_config_map_env == True): - config_map_task = log_message(message="configmap env applied") - config_map_task.set_caching_options(enable_caching=False) - kubernetes.use_config_map_as_env( - task=config_map_task, - config_map_name=optional_config_map_name, - config_map_key_to_env={"setting": "SETTING"}, - ) + kubernetes.use_config_map_as_env( + task=task, + config_map_name=optional_config_map_name, + config_map_key_to_env={"setting": "SETTING"}, + optional=True, + ) + kubernetes.use_config_map_as_volume( + task=task, + config_map_name=optional_config_map_name, + mount_path="/mnt/config", + optional=True, + ) + + kubernetes.set_image_pull_secrets( + task=task, + secret_names=[optional_image_pull_secret_name], + ) - with dsl.If(enable_tolerations == True): - toleration_task = log_message(message="tolerations applied") - toleration_task.set_caching_options(enable_caching=False) - kubernetes.add_toleration_json( - task=toleration_task, - toleration_json=optional_tolerations, - ) + kubernetes.add_toleration_json( + task=task, + toleration_json=optional_tolerations, + ) if __name__ == "__main__": diff --git a/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml index 30a61bf9a31..ad9813a7339 100644 --- a/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml +++ b/test_data/sdk_compiled_pipelines/valid/critical/missing_kubernetes_optional_inputs.yaml @@ -1,143 +1,19 @@ # PIPELINE DEFINITION # Name: missing-kubernetes-optional-inputs-pipeline # Inputs: -# enable_affinity: bool [Default: False] -# enable_config_map_env: bool [Default: False] -# enable_secret_env: bool [Default: False] -# enable_selector: bool [Default: False] -# enable_tolerations: bool [Default: False] # optional_affinity: dict # optional_config_map_name: str +# optional_image_pull_secret_name: str # optional_secret_name: str # optional_selector: dict # optional_tolerations: list components: - comp-condition-1: - dag: - tasks: - log-message-2: - cachingOptions: {} - componentRef: - name: comp-log-message-2 - inputs: - parameters: - message: - runtimeValue: - constant: node affinity applied - taskInfo: - name: log-message-2 - inputDefinitions: - parameters: - pipelinechannel--enable_affinity: - parameterType: BOOLEAN - comp-condition-2: - dag: - tasks: - log-message-3: - cachingOptions: {} - componentRef: - name: comp-log-message-3 - inputs: - parameters: - message: - runtimeValue: - constant: node selector applied - taskInfo: - name: log-message-3 - inputDefinitions: - parameters: - pipelinechannel--enable_selector: - parameterType: BOOLEAN - comp-condition-3: - dag: - tasks: - log-message-4: - cachingOptions: {} - componentRef: - name: comp-log-message-4 - inputs: - parameters: - message: - runtimeValue: - constant: secret env applied - taskInfo: - name: log-message-4 - inputDefinitions: - parameters: - pipelinechannel--enable_secret_env: - parameterType: BOOLEAN - comp-condition-4: - dag: - tasks: - log-message-5: - cachingOptions: {} - componentRef: - name: comp-log-message-5 - inputs: - parameters: - message: - runtimeValue: - constant: configmap env applied - taskInfo: - name: log-message-5 - inputDefinitions: - parameters: - pipelinechannel--enable_config_map_env: - parameterType: BOOLEAN - comp-condition-5: - dag: - tasks: - log-message-6: - cachingOptions: {} - componentRef: - name: comp-log-message-6 - inputs: - parameters: - message: - runtimeValue: - constant: tolerations applied - taskInfo: - name: log-message-6 - inputDefinitions: - parameters: - pipelinechannel--enable_tolerations: - parameterType: BOOLEAN comp-log-message: executorLabel: exec-log-message inputDefinitions: parameters: message: parameterType: STRING - comp-log-message-2: - executorLabel: exec-log-message-2 - inputDefinitions: - parameters: - message: - parameterType: STRING - comp-log-message-3: - executorLabel: exec-log-message-3 - inputDefinitions: - parameters: - message: - parameterType: STRING - comp-log-message-4: - executorLabel: exec-log-message-4 - inputDefinitions: - parameters: - message: - parameterType: STRING - comp-log-message-5: - executorLabel: exec-log-message-5 - inputDefinitions: - parameters: - message: - parameterType: STRING - comp-log-message-6: - executorLabel: exec-log-message-6 - inputDefinitions: - parameters: - message: - parameterType: STRING deploymentSpec: executors: exec-log-message: @@ -152,119 +28,7 @@ deploymentSpec: - -c - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" - image: python:3.9 - exec-log-message-2: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - log_message - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" - image: python:3.9 - exec-log-message-3: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - log_message - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" - image: python:3.9 - exec-log-message-4: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - log_message - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" - image: python:3.9 - exec-log-message-5: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - log_message - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ + \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.6'\ \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ $0\" \"$@\"\n" - sh @@ -279,100 +43,12 @@ deploymentSpec: ' - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" - image: python:3.9 - exec-log-message-6: - container: - args: - - --executor_input - - '{{$}}' - - --function_to_execute - - log_message - command: - - sh - - -c - - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ - \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ - \ python3 -m pip install --quiet --no-warn-script-location 'kfp==2.14.4'\ - \ '--no-deps' 'typing-extensions>=3.7.4,<5; python_version<\"3.9\"' && \"\ - $0\" \"$@\"\n" - - sh - - -ec - - 'program_path=$(mktemp -d) - - - printf "%s" "$0" > "$program_path/ephemeral_component.py" - - _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" - - ' - - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ - \ *\n\ndef log_message(message: str) -> None:\n print(message)\n\n" - image: python:3.9 + image: python:3.11 pipelineInfo: name: missing-kubernetes-optional-inputs-pipeline root: dag: tasks: - condition-1: - componentRef: - name: comp-condition-1 - inputs: - parameters: - pipelinechannel--enable_affinity: - componentInputParameter: enable_affinity - taskInfo: - name: condition-1 - triggerPolicy: - condition: inputs.parameter_values['pipelinechannel--enable_affinity'] == - true - condition-2: - componentRef: - name: comp-condition-2 - inputs: - parameters: - pipelinechannel--enable_selector: - componentInputParameter: enable_selector - taskInfo: - name: condition-2 - triggerPolicy: - condition: inputs.parameter_values['pipelinechannel--enable_selector'] == - true - condition-3: - componentRef: - name: comp-condition-3 - inputs: - parameters: - pipelinechannel--enable_secret_env: - componentInputParameter: enable_secret_env - taskInfo: - name: condition-3 - triggerPolicy: - condition: inputs.parameter_values['pipelinechannel--enable_secret_env'] - == true - condition-4: - componentRef: - name: comp-condition-4 - inputs: - parameters: - pipelinechannel--enable_config_map_env: - componentInputParameter: enable_config_map_env - taskInfo: - name: condition-4 - triggerPolicy: - condition: inputs.parameter_values['pipelinechannel--enable_config_map_env'] - == true - condition-5: - componentRef: - name: comp-condition-5 - inputs: - parameters: - pipelinechannel--enable_tolerations: - componentInputParameter: enable_tolerations - taskInfo: - name: condition-5 - triggerPolicy: - condition: inputs.parameter_values['pipelinechannel--enable_tolerations'] - == true log-message: cachingOptions: {} componentRef: @@ -386,32 +62,15 @@ root: name: log-message inputDefinitions: parameters: - enable_affinity: - defaultValue: false - isOptional: true - parameterType: BOOLEAN - enable_config_map_env: - defaultValue: false - isOptional: true - parameterType: BOOLEAN - enable_secret_env: - defaultValue: false - isOptional: true - parameterType: BOOLEAN - enable_selector: - defaultValue: false - isOptional: true - parameterType: BOOLEAN - enable_tolerations: - defaultValue: false - isOptional: true - parameterType: BOOLEAN optional_affinity: isOptional: true parameterType: STRUCT optional_config_map_name: isOptional: true parameterType: STRING + optional_image_pull_secret_name: + isOptional: true + parameterType: STRING optional_secret_name: isOptional: true parameterType: STRING @@ -422,35 +81,46 @@ root: isOptional: true parameterType: LIST schemaVersion: 2.1.0 -sdkVersion: kfp-2.14.4 +sdkVersion: kfp-2.14.6 --- platforms: kubernetes: deploymentSpec: executors: - exec-log-message-2: + exec-log-message: + configMapAsEnv: + - configMapNameParameter: + componentInputParameter: optional_config_map_name + keyToEnv: + - configMapKey: setting + envVar: SETTING + optional: true + configMapAsVolume: + - configMapNameParameter: + componentInputParameter: optional_config_map_name + mountPath: /mnt/config + optional: true + imagePullSecret: + - secretNameParameter: + componentInputParameter: optional_image_pull_secret_name nodeAffinity: - nodeAffinityJson: componentInputParameter: optional_affinity - exec-log-message-3: nodeSelector: nodeSelectorJson: componentInputParameter: optional_selector - exec-log-message-4: secretAsEnv: - keyToEnv: - envVar: PASSWORD secretKey: password + optional: true + secretNameParameter: + componentInputParameter: optional_secret_name + secretAsVolume: + - mountPath: /mnt/secret + optional: true secretNameParameter: componentInputParameter: optional_secret_name - exec-log-message-5: - configMapAsEnv: - - configMapNameParameter: - componentInputParameter: optional_config_map_name - keyToEnv: - - configMapKey: setting - envVar: SETTING - exec-log-message-6: tolerations: - tolerationJson: componentInputParameter: optional_tolerations