Skip to content

Commit c6042ed

Browse files
alyssacgoinsalyssacgoins
committed
Correct Runtime Artifact custom path logic/testing.
Signed-off-by: agoins <[email protected]>
1 parent fc69198 commit c6042ed

File tree

5 files changed

+448
-22
lines changed

5 files changed

+448
-22
lines changed

‎backend/src/v2/component/launcher_v2.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -945,6 +945,10 @@ func mergeRuntimeArtifacts(src, dst *pipelinespec.RuntimeArtifact) {
945945
}
946946
}
947947
}
948+
949+
if src.CustomPath != nil && *src.CustomPath != "" {
950+
dst.CustomPath = src.CustomPath
951+
}
948952
}
949953

950954
func getExecutorOutputFile(path string) (*pipelinespec.ExecutorOutput, error) {

‎sdk/python/kfp/dsl/types/artifact_types.py‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@ def path(self) -> str:
9191

9292
@path.setter
9393
def path(self, path: str) -> None:
94-
self._set_custom_path(path)
94+
self._set_path(path)
9595

9696
def _get_path(self) -> Optional[str]:
9797
if self.custom_path is not '':
@@ -112,23 +112,28 @@ def _get_path(self) -> Optional[str]:
112112
# uri == path for local execution
113113
return self.uri
114114

115+
def _set_path(self, path: str) -> None:
116+
self.uri = convert_local_path_to_remote_path(path)
117+
115118
@property
116119
def custom_path(self) -> str:
120+
return self._get_custom_path()
121+
122+
@custom_path.getter
123+
def custom_path(self):
117124
return self._custom_path
118125

119126
def _get_custom_path(self) -> str:
120127
return self._custom_path
121128

122-
def _set_path(self, path: str) -> None:
123-
self.uri = convert_local_path_to_remote_path(path)
124-
125-
def _set_custom_path(self, value: str) -> None:
126-
self._custom_path = value
127-
128129
@custom_path.setter
129130
def custom_path(self, value: str):
130131
self._custom_path = value
131132

133+
def set_path(self, path: str) -> None:
134+
# If user specified a custom path, use it instead of the default path.
135+
self._custom_path = path
136+
132137

133138
def convert_local_path_to_remote_path(path: str) -> str:
134139
if path.startswith(_GCS_LOCAL_MOUNT_PREFIX):

‎test_data/compiled-workflows/pipeline_with_artifact_custom_path.yaml‎

Lines changed: 340 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,340 @@
1+
apiVersion: argoproj.io/v1alpha1
2+
kind: Workflow
3+
metadata:
4+
creationTimestamp: null
5+
generateName: pipeline-with-custom-path-artifact-
6+
spec:
7+
arguments:
8+
parameters:
9+
- name: components-768671644bcdb5cb9cc126b44bff9d26fa3c3d25cecf19ba1c687dc403e6d386
10+
value: '{"executorLabel":"exec-validate-custom-artifact-path","inputDefinitions":{"parameters":{"num":{"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"out_dataset":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}}'
11+
- name: implementations-768671644bcdb5cb9cc126b44bff9d26fa3c3d25cecf19ba1c687dc403e6d386
12+
value: '{"args":["--executor_input","{{$}}","--function_to_execute","validate_custom_artifact_path"],"command":["sh","-c","\nif
13+
! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3
14+
-m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1
15+
python3 -m pip install --quiet --no-warn-script-location ''kfp==2.14.6'' ''--no-deps''
16+
''typing-extensions\u003e=3.7.4,\u003c5; python_version\u003c\"3.9\"'' \u0026\u0026
17+
\"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\"
18+
\u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3
19+
-m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport
20+
kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef
21+
validate_custom_artifact_path(num: int, out_dataset: Output[Dataset]):\n with
22+
open(out_dataset.path, ''w'') as f:\n f.write(str(2 * num))\n out_dataset.set_path(''/etc/test/file/path'')\n\n if
23+
out_dataset.path != ''/etc/test/file/path'':\n raise ValueError(f\"File
24+
path is {out_dataset.path} but should be ''/etc/test/file/path''.\")\n if
25+
out_dataset.custom_path != ''/etc/test/file/path'':\n raise ValueError(f\"File
26+
uri is {out_dataset.custom_path} but should be ''/etc/test/file/path''.\")\n\n"],"image":"python:3.11"}'
27+
- name: components-root
28+
value: '{"dag":{"tasks":{"validate-custom-artifact-path":{"cachingOptions":{},"componentRef":{"name":"comp-validate-custom-artifact-path"},"inputs":{"parameters":{"num":{"runtimeValue":{"constant":1}}}},"taskInfo":{"name":"validate-custom-artifact-path"}}}}}'
29+
entrypoint: entrypoint
30+
podMetadata:
31+
annotations:
32+
pipelines.kubeflow.org/v2_component: "true"
33+
labels:
34+
pipelines.kubeflow.org/v2_component: "true"
35+
serviceAccountName: pipeline-runner
36+
templates:
37+
- container:
38+
args:
39+
- --type
40+
- CONTAINER
41+
- --pipeline_name
42+
- pipeline-with-custom-path-artifact
43+
- --run_id
44+
- '{{workflow.uid}}'
45+
- --run_name
46+
- '{{workflow.name}}'
47+
- --run_display_name
48+
- ""
49+
- --dag_execution_id
50+
- '{{inputs.parameters.parent-dag-id}}'
51+
- --component
52+
- '{{inputs.parameters.component}}'
53+
- --task
54+
- '{{inputs.parameters.task}}'
55+
- --task_name
56+
- '{{inputs.parameters.task-name}}'
57+
- --container
58+
- '{{inputs.parameters.container}}'
59+
- --iteration_index
60+
- '{{inputs.parameters.iteration-index}}'
61+
- --cached_decision_path
62+
- '{{outputs.parameters.cached-decision.path}}'
63+
- --pod_spec_patch_path
64+
- '{{outputs.parameters.pod-spec-patch.path}}'
65+
- --condition_path
66+
- '{{outputs.parameters.condition.path}}'
67+
- --kubernetes_config
68+
- '{{inputs.parameters.kubernetes-config}}'
69+
- --http_proxy
70+
- ""
71+
- --https_proxy
72+
- ""
73+
- --no_proxy
74+
- ""
75+
command:
76+
- driver
77+
image: ghcr.io/kubeflow/kfp-driver:latest
78+
name: ""
79+
resources:
80+
limits:
81+
cpu: 500m
82+
memory: 512Mi
83+
requests:
84+
cpu: 100m
85+
memory: 64Mi
86+
inputs:
87+
parameters:
88+
- name: component
89+
- name: task
90+
- name: container
91+
- name: task-name
92+
- name: parent-dag-id
93+
- default: "-1"
94+
name: iteration-index
95+
- default: ""
96+
name: kubernetes-config
97+
metadata: {}
98+
name: system-container-driver
99+
outputs:
100+
parameters:
101+
- name: pod-spec-patch
102+
valueFrom:
103+
default: ""
104+
path: /tmp/outputs/pod-spec-patch
105+
- default: "false"
106+
name: cached-decision
107+
valueFrom:
108+
default: "false"
109+
path: /tmp/outputs/cached-decision
110+
- name: condition
111+
valueFrom:
112+
default: "true"
113+
path: /tmp/outputs/condition
114+
- dag:
115+
tasks:
116+
- arguments:
117+
parameters:
118+
- name: pod-spec-patch
119+
value: '{{inputs.parameters.pod-spec-patch}}'
120+
name: executor
121+
template: system-container-impl
122+
when: '{{inputs.parameters.cached-decision}} != true'
123+
inputs:
124+
parameters:
125+
- name: pod-spec-patch
126+
- default: "false"
127+
name: cached-decision
128+
metadata: {}
129+
name: system-container-executor
130+
outputs: {}
131+
- container:
132+
command:
133+
- should-be-overridden-during-runtime
134+
env:
135+
- name: KFP_POD_NAME
136+
valueFrom:
137+
fieldRef:
138+
fieldPath: metadata.name
139+
- name: KFP_POD_UID
140+
valueFrom:
141+
fieldRef:
142+
fieldPath: metadata.uid
143+
envFrom:
144+
- configMapRef:
145+
name: metadata-grpc-configmap
146+
optional: true
147+
image: gcr.io/ml-pipeline/should-be-overridden-during-runtime
148+
name: ""
149+
resources: {}
150+
volumeMounts:
151+
- mountPath: /kfp-launcher
152+
name: kfp-launcher
153+
- mountPath: /gcs
154+
name: gcs-scratch
155+
- mountPath: /s3
156+
name: s3-scratch
157+
- mountPath: /minio
158+
name: minio-scratch
159+
- mountPath: /.local
160+
name: dot-local-scratch
161+
- mountPath: /.cache
162+
name: dot-cache-scratch
163+
- mountPath: /.config
164+
name: dot-config-scratch
165+
initContainers:
166+
- args:
167+
- --copy
168+
- /kfp-launcher/launch
169+
command:
170+
- launcher-v2
171+
image: ghcr.io/kubeflow/kfp-launcher:latest
172+
name: kfp-launcher
173+
resources:
174+
limits:
175+
cpu: 500m
176+
memory: 128Mi
177+
requests:
178+
cpu: 100m
179+
volumeMounts:
180+
- mountPath: /kfp-launcher
181+
name: kfp-launcher
182+
inputs:
183+
parameters:
184+
- name: pod-spec-patch
185+
metadata: {}
186+
name: system-container-impl
187+
outputs: {}
188+
podSpecPatch: '{{inputs.parameters.pod-spec-patch}}'
189+
volumes:
190+
- emptyDir: {}
191+
name: kfp-launcher
192+
- emptyDir: {}
193+
name: gcs-scratch
194+
- emptyDir: {}
195+
name: s3-scratch
196+
- emptyDir: {}
197+
name: minio-scratch
198+
- emptyDir: {}
199+
name: dot-local-scratch
200+
- emptyDir: {}
201+
name: dot-cache-scratch
202+
- emptyDir: {}
203+
name: dot-config-scratch
204+
- dag:
205+
tasks:
206+
- arguments:
207+
parameters:
208+
- name: component
209+
value: '{{workflow.parameters.components-768671644bcdb5cb9cc126b44bff9d26fa3c3d25cecf19ba1c687dc403e6d386}}'
210+
- name: task
211+
value: '{"cachingOptions":{},"componentRef":{"name":"comp-validate-custom-artifact-path"},"inputs":{"parameters":{"num":{"runtimeValue":{"constant":1}}}},"taskInfo":{"name":"validate-custom-artifact-path"}}'
212+
- name: container
213+
value: '{{workflow.parameters.implementations-768671644bcdb5cb9cc126b44bff9d26fa3c3d25cecf19ba1c687dc403e6d386}}'
214+
- name: task-name
215+
value: validate-custom-artifact-path
216+
- name: parent-dag-id
217+
value: '{{inputs.parameters.parent-dag-id}}'
218+
name: validate-custom-artifact-path-driver
219+
template: system-container-driver
220+
- arguments:
221+
parameters:
222+
- name: pod-spec-patch
223+
value: '{{tasks.validate-custom-artifact-path-driver.outputs.parameters.pod-spec-patch}}'
224+
- default: "false"
225+
name: cached-decision
226+
value: '{{tasks.validate-custom-artifact-path-driver.outputs.parameters.cached-decision}}'
227+
depends: validate-custom-artifact-path-driver.Succeeded
228+
name: validate-custom-artifact-path
229+
template: system-container-executor
230+
inputs:
231+
parameters:
232+
- name: parent-dag-id
233+
metadata: {}
234+
name: root
235+
outputs: {}
236+
- container:
237+
args:
238+
- --type
239+
- '{{inputs.parameters.driver-type}}'
240+
- --pipeline_name
241+
- pipeline-with-custom-path-artifact
242+
- --run_id
243+
- '{{workflow.uid}}'
244+
- --run_name
245+
- '{{workflow.name}}'
246+
- --run_display_name
247+
- ""
248+
- --dag_execution_id
249+
- '{{inputs.parameters.parent-dag-id}}'
250+
- --component
251+
- '{{inputs.parameters.component}}'
252+
- --task
253+
- '{{inputs.parameters.task}}'
254+
- --task_name
255+
- '{{inputs.parameters.task-name}}'
256+
- --runtime_config
257+
- '{{inputs.parameters.runtime-config}}'
258+
- --iteration_index
259+
- '{{inputs.parameters.iteration-index}}'
260+
- --execution_id_path
261+
- '{{outputs.parameters.execution-id.path}}'
262+
- --iteration_count_path
263+
- '{{outputs.parameters.iteration-count.path}}'
264+
- --condition_path
265+
- '{{outputs.parameters.condition.path}}'
266+
- --http_proxy
267+
- ""
268+
- --https_proxy
269+
- ""
270+
- --no_proxy
271+
- ""
272+
command:
273+
- driver
274+
image: ghcr.io/kubeflow/kfp-driver:latest
275+
name: ""
276+
resources:
277+
limits:
278+
cpu: 500m
279+
memory: 512Mi
280+
requests:
281+
cpu: 100m
282+
memory: 64Mi
283+
inputs:
284+
parameters:
285+
- name: component
286+
- default: ""
287+
name: runtime-config
288+
- default: ""
289+
name: task
290+
- default: ""
291+
name: task-name
292+
- default: "0"
293+
name: parent-dag-id
294+
- default: "-1"
295+
name: iteration-index
296+
- default: DAG
297+
name: driver-type
298+
metadata: {}
299+
name: system-dag-driver
300+
outputs:
301+
parameters:
302+
- name: execution-id
303+
valueFrom:
304+
path: /tmp/outputs/execution-id
305+
- name: iteration-count
306+
valueFrom:
307+
default: "0"
308+
path: /tmp/outputs/iteration-count
309+
- name: condition
310+
valueFrom:
311+
default: "true"
312+
path: /tmp/outputs/condition
313+
- dag:
314+
tasks:
315+
- arguments:
316+
parameters:
317+
- name: component
318+
value: '{{workflow.parameters.components-root}}'
319+
- name: runtime-config
320+
value: '{}'
321+
- name: driver-type
322+
value: ROOT_DAG
323+
name: root-driver
324+
template: system-dag-driver
325+
- arguments:
326+
parameters:
327+
- name: parent-dag-id
328+
value: '{{tasks.root-driver.outputs.parameters.execution-id}}'
329+
- name: condition
330+
value: ""
331+
depends: root-driver.Succeeded
332+
name: root
333+
template: root
334+
inputs: {}
335+
metadata: {}
336+
name: entrypoint
337+
outputs: {}
338+
status:
339+
finishedAt: null
340+
startedAt: null

0 commit comments

Comments
 (0)