Cherry-picks for 4.2.1 release (opendatahub-io#109)

caponetto · web-flow · commit d5ee1b85b7c7 · 2025-05-07T10:23:37.000-03:00
* Extract common code to dedicated GitHub actions and fail the build if the lock file is not up-to-date Signed-off-by: Guilherme Caponetto <638737+caponetto@users.noreply.github.com> * Improve tag resolution on the Build notebook images step (opendatahub-io#83) * Suppress flake8 F824 warning to fix the CI Signed-off-by: Guilherme Caponetto <638737+caponetto@users.noreply.github.com> * Update JL version compatibility (opendatahub-io#108) * Fix disable cache mechanism (opendatahub-io#106) * Fix disable cache mechanism * Add tests for disable_node_caching * Add a check to run conda env remove only if the env exists * Remove unneeded file after series of cherry-picks * Reuse common actions in the release and update-version workflows --------- Signed-off-by: Guilherme Caponetto <638737+caponetto@users.noreply.github.com>
diff --git a/.github/actions/install-ui-dependencies/action.yml b/.github/actions/install-ui-dependencies/action.yml
@@ -0,0 +1,28 @@
+name: "Install UI dependencies"
+description: "Install UI dependencies; fail the build if the lock file is not up-to-date"
+
+inputs:
+  working_dir:
+    description: "Elyra path"
+    required: false
+    default: "."
+
+runs:
+  using: "composite"
+  steps:
+    - name: Go to working directory
+      shell: bash
+      run: |
+        echo "Step: Go to working directory"
+        cd ${{ inputs.working_dir }}
+    - name: Install dependencies
+      shell: bash
+      run: |
+        echo "Step: Install dependencies"
+        make yarn-install
+    - name: "Check yarn.lock"
+      shell: bash
+      run: |
+        echo "Step: Check yarn.lock"
+        git diff yarn.lock;
+        [ "0" == "$(git diff yarn.lock | wc -l | tr -d ' ')" ]
diff --git a/.github/actions/install-yarn/action.yaml b/.github/actions/install-yarn/action.yaml
@@ -0,0 +1,27 @@
+name: "Install Yarn"
+description: "Install Yarn"
+
+inputs:
+  working_dir:
+    description: "Elyra path"
+    required: false
+    default: "."
+  version:
+    description: "Yarn version"
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Go to working directory
+      shell: bash
+      run: |
+        echo "Step: Go to working directory"
+        cd ${{ inputs.working_dir }}
+    - name: Install
+      shell: bash
+      run: |
+        echo "Step: Install"
+        corepack prepare yarn@${{ inputs.version }} --activate
+        yarn set version ${{ inputs.version }}
+        echo "Yarn version: $(yarn --version)"
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -46,13 +46,11 @@ jobs:
             /home/runner/.cache/Cypress
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          yarn --version
-      - name: Install dependencies
-        run: |
-          yarn install
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
   lint-server:
     name: Lint Server
     runs-on: ubuntu-latest
@@ -81,13 +79,11 @@ jobs:
             */*/node_modules
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          yarn --version
-      - name: Install dependencies
-        run: |
-          make yarn-install
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
       - name: Lint
         run: make eslint-check-ui
       - name: Check format
@@ -141,14 +137,14 @@ jobs:
             */*/node_modules
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          yarn --version
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
       - name: Build
         run: |
           make build-dependencies
-          make yarn-install
           make build-ui-prod
       - name: Install
         run: make install-server
@@ -175,14 +171,14 @@ jobs:
             /home/runner/.cache/Cypress
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          yarn --version
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
       - name: Build
         run: |
           make build-dependencies
-          make yarn-install
           make build-ui-prod
       - name: Install
         run: |
@@ -254,10 +250,9 @@ jobs:
             /home/runner/.cache/Cypress
           key: ${{ runner.os }}-yarn-${{ hashFiles('**/yarn.lock') }}
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          yarn --version
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
       - name: Check whether to build container images
         id: should-build-images
         run: |
@@ -266,6 +261,8 @@ jobs:
         if: steps.should-build-images.outputs.result == 'true'
         run: |
           ./.github/workflows/scripts/free_gha_space.sh
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
       - name: Build
         run: |
           make install-prod
@@ -289,9 +286,7 @@ jobs:
 
           URLS=""
           for tag_prefix in $(echo "${{ env.QUAY_WORKBENCH_IMAGES_TAG_PREFIX_CSV }}" | tr ',' '\n'); do
-            LATEST_TAG=$(curl -s "https://quay.io/api/v1/repository/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}/tag/?onlyActiveTags=true" | \
-              jq -r ".tags | map(select(.name | startswith(\"${tag_prefix}\"))) | .[].name" | \
-              sort -r | head -n 1)
+            LATEST_TAG=$(curl -s "https://quay.io/api/v1/repository/${{ env.QUAY_WORKBENCH_IMAGES_REPOSITORY}}/tag/?onlyActiveTags=true&filter_tag_name=like:${tag_prefix}&limit=1" | jq -r ".tags[].name")
             echo "Latest tag: $LATEST_TAG"
 
             CONTAINER_IMAGE=ghcr.io/${{ github.repository }}/workbench-images:${LATEST_TAG}${TAG_SUFFIX}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -64,10 +64,11 @@ jobs:
         with:
           node-version: ${{ env.NODE_VERSION }}
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          yarn --version
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
       - name: Check version
         if: ${{ github.repository_owner == 'opendatahub-io' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && !inputs.dry_run)) }}
         run: |
diff --git a/.github/workflows/update-version-through-pr.yml b/.github/workflows/update-version-through-pr.yml
@@ -41,15 +41,12 @@ jobs:
           node-version: ${{ env.NODE_VERSION }}
 
       - name: Install Yarn ${{ env.YARN_VERSION }}
-        run: |
-          corepack prepare yarn@${{ env.YARN_VERSION }} --activate
-          yarn set version ${{ env.YARN_VERSION }}
-          git checkout package.json
-          yarn --version
+        uses: ./.github/actions/install-yarn
+        with:
+          version: ${{ env.YARN_VERSION }}
 
-      - name: Install dependencies
-        run: |
-          make yarn-install
+      - name: Install UI dependencies
+        uses: ./.github/actions/install-ui-dependencies
 
       - name: Update version to ${{ github.event.inputs.version }}
         id: update
diff --git a/Makefile b/Makefile
@@ -198,7 +198,9 @@ release: yarn-install build-ui-prod build-server ## Build wheel file for release
 
 
 elyra-image-env: ## Creates a conda env consisting of the dependencies used in images
-	conda env remove -y -n $(ELYRA_IMAGE_ENV)
+	@conda env list | grep -q "$(ELYRA_IMAGE_ENV)" && \
+		conda env remove -y -n $(ELYRA_IMAGE_ENV) || \
+		echo "Environment $(ELYRA_IMAGE_ENV) does not exist, skipping removal"
 	conda create -y -n $(ELYRA_IMAGE_ENV) python=$(PYTHON_VERSION) --channel conda-forge
 	$(CONDA_ACTIVATE) $(ELYRA_IMAGE_ENV) && \
 	$(PYTHON_PIP) install -r etc/generic/requirements-elyra.txt && \
diff --git a/elyra/kfp/bootstrapper.py b/elyra/kfp/bootstrapper.py
@@ -735,7 +735,7 @@ def log_operation_info(cls, action_clause: str, duration_secs: Optional[float] =
         :param action_clause: str representing the action that is being logged
         :param duration_secs: optional float value representing the duration of the action being logged
         """
-        global pipeline_name, operation_name
+        global pipeline_name, operation_name  # noqa: F824
         if enable_pipeline_info:
             duration_clause = f"({duration_secs:.3f} secs)" if duration_secs else ""
             logger.info(f"'{pipeline_name}':'{operation_name}' - {action_clause} {duration_clause}")
diff --git a/elyra/pipeline/kfp/processor_kfp.py b/elyra/pipeline/kfp/processor_kfp.py
@@ -886,6 +886,10 @@ def _generate_workflow_tasks(
                     # Use Kubeflow Pipelines provided RUN_ID_PLACEHOLDER as run name
                     workflow_task["task_modifiers"]["set_run_name"] = RUN_ID_PLACEHOLDER
 
+                disable_node_caching = pipeline.pipeline_properties.get(pipeline_constants.DISABLE_NODE_CACHING)
+                if disable_node_caching:
+                    workflow_task["task_modifiers"]["disable_node_caching"] = disable_node_caching.selection
+
                 # Upload dependencies to cloud storage
                 self._upload_dependencies_to_object_store(
                     runtime_configuration, pipeline_name, operation, prefix=artifact_object_prefix
diff --git a/elyra/templates/kubeflow/v2/python_dsl_template.jinja2 b/elyra/templates/kubeflow/v2/python_dsl_template.jinja2
@@ -91,7 +91,7 @@ def generated_pipeline(
     {{ task_name }}.set_env_variable(name="ELYRA_RUN_NAME", value="{{ workflow_task.task_modifiers.set_run_name }}")
 {%  endif %}
 {%  if workflow_task.task_modifiers.disable_node_caching %}
-    {{ task_name }}.execution_options.caching_strategy.max_cache_staleness = "P0D"
+    {{ task_name }}.set_caching_options(enable_caching={{ workflow_task.task_modifiers.disable_node_caching == false }})
 {%  endif %}
 {%  if workflow_task.task_modifiers.pod_labels %}
 {%    for pod_label_key, pod_label_value in workflow_task.task_modifiers.pod_labels.items() %}
diff --git a/elyra/tests/pipeline/kfp/conftest.py b/elyra/tests/pipeline/kfp/conftest.py
@@ -32,7 +32,7 @@
 from elyra.pipeline.kfp.processor_kfp import WorkflowEngineType
 from elyra.pipeline.parser import PipelineParser
 from elyra.pipeline.pipeline import Pipeline
-from elyra.pipeline.pipeline_constants import COS_OBJECT_PREFIX
+from elyra.pipeline.pipeline_constants import COS_OBJECT_PREFIX, DISABLE_NODE_CACHING
 
 
 @pytest.fixture()
@@ -158,6 +158,7 @@ def metadata_dependencies(metadata_managers, request):
      - "pipeline_file": existing pipeline filename
     Optional inputs:
         - "with_cos_object_prefix": bool (default: False)
+        - "with_disable_node_caching": bool (default: False)
         - "workflow_engine": WorkflowEngineType.ARGO or WorkflowEngineType.TEKTON
         - "use_cos_credentials_secret": bool (default: False)
         - "require_pull_secret": bool (default: False)
@@ -179,6 +180,7 @@ def metadata_dependencies(metadata_managers, request):
     customization_options = {}
     for supported_option in [
         "with_cos_object_prefix",
+        "with_disable_node_caching",
         "resources_cpu",
         "resources_cpu_limit",
         "resources_gpu",
@@ -365,6 +367,7 @@ def get_pipeline_object(
     Creates a KFP Pipeline instance from pipeline_filename, taking into account the
     following optional customization options:
       - "with_cos_object_prefix" (True/False)
+      - "with_disable_node_caching" (True/False)
       - "resources_cpu" (number, applied to all generic nodes)
       - "resources_gpu" (number, applied to all generic nodes)
       - "resources_memory" (number, applied to all generic nodes)
@@ -410,6 +413,9 @@ def get_pipeline_object(
             if app_data["properties"].get("pipeline_defaults") is not None:
                 app_data["properties"]["pipeline_defaults"].pop(COS_OBJECT_PREFIX, None)
 
+        if customization_options.get("with_disable_node_caching"):
+            app_data["properties"]["pipeline_defaults"][DISABLE_NODE_CACHING] = True
+
         #
         # Add resource customizations to every generic node
         for node in primary_pipeline["nodes"]:
diff --git a/elyra/tests/pipeline/kfp/test_processor_kfp.py b/elyra/tests/pipeline/kfp/test_processor_kfp.py
@@ -567,6 +567,101 @@ def test_generate_pipeline_dsl_compile_pipeline_dsl_workflow_engine_test(
     assert "platforms" in workflow_spec_docs[1]
 
 
+@pytest.mark.parametrize(
+    "metadata_dependencies",
+    [
+        {
+            "pipeline_file": Path(__file__).parent
+            / ".."
+            / "resources"
+            / "test_pipelines"
+            / "kfp"
+            / "kfp-one-node-generic.pipeline",
+            "workflow_engine": WorkflowEngineType.ARGO,
+            "with_disable_node_caching": True,
+        },
+        {
+            "pipeline_file": Path(__file__).parent
+            / ".."
+            / "resources"
+            / "test_pipelines"
+            / "kfp"
+            / "kfp-one-node-generic.pipeline",
+            "workflow_engine": WorkflowEngineType.ARGO,
+            "with_disable_node_caching": False,
+        },
+    ],
+    indirect=True,
+)
+def test_disable_node_caching_on_compiled_pipeline(
+    monkeypatch, processor: KfpPipelineProcessor, metadata_dependencies: Dict[str, Any], tmpdir
+):
+    """
+    This test validates the disable_node_caching option all the way to the compiled pipeline.
+    """
+
+    # Obtain artifacts from metadata_dependencies fixture
+    test_pipeline_file = metadata_dependencies["pipeline_file"]
+    pipeline = metadata_dependencies["pipeline_object"]
+    assert pipeline is not None
+    runtime_config = metadata_dependencies["runtime_config"]
+    assert runtime_config is not None
+    assert runtime_config.name == pipeline.runtime_config
+
+    disable_node_caching = metadata_dependencies["fixture_parameters"]["with_disable_node_caching"]
+
+    workflow_engine = WorkflowEngineType.get_instance_by_value(runtime_config.metadata["engine"])
+
+    # Mock calls that require access to object storage, because their side effects
+    # have no bearing on the outcome of this test.
+    monkeypatch.setattr(processor, "_upload_dependencies_to_object_store", lambda w, x, y, prefix: True)
+    monkeypatch.setattr(processor, "_verify_cos_connectivity", lambda x: True)
+
+    compiled_output_file = Path(tmpdir) / test_pipeline_file.with_suffix(".yaml").name
+    compiled_output_file_name = str(compiled_output_file.absolute())
+
+    # generate Python DSL for the specified workflow engine
+    pipeline_version = f"{pipeline.name}-test-0"
+    pipeline_instance_id = f"{pipeline.name}-{datetime.now().strftime('%m%d%H%M%S')}"
+    experiment_name = f"{pipeline.name}-test-0"
+    generated_dsl = processor._generate_pipeline_dsl(
+        pipeline=pipeline,
+        pipeline_name=pipeline.name,
+        workflow_engine=workflow_engine,
+        pipeline_version=pipeline_version,
+        pipeline_instance_id=pipeline_instance_id,
+        experiment_name=experiment_name,
+    )
+
+    if disable_node_caching:
+        assert "set_caching_options(enable_caching=False)" in generated_dsl
+    else:
+        # Default behavior is to enable node caching
+        assert "set_caching_options" not in generated_dsl
+
+    # Compile the generated Python DSL
+    processor._compile_pipeline_dsl(
+        dsl=generated_dsl,
+        workflow_engine=workflow_engine,
+        output_file=compiled_output_file_name,
+        pipeline_conf=None,
+    )
+
+    # Load compiled workflow
+    with open(compiled_output_file_name) as f:
+        workflow_spec_docs = list(yaml.safe_load_all(f.read()))
+
+    assert len(workflow_spec_docs) == 2
+    assert "components" in workflow_spec_docs[0]
+    assert "platforms" in workflow_spec_docs[1]
+
+    caching_options = workflow_spec_docs[0]["root"]["dag"]["tasks"]["run-a-file"]["cachingOptions"]
+    if disable_node_caching:
+        assert caching_options == {}
+    else:
+        assert caching_options["enableCache"] is True
+
+
 @pytest.mark.parametrize(
     "metadata_dependencies",
     [
diff --git a/pyproject.toml b/pyproject.toml
