Add deprecation notice to eval code (#14316)

* Add deprecation notice to eval code

Signed-off-by: Abhishree <[email protected]>

* Update nemo/collections/llm/api.py

Signed-off-by: oliver könig <[email protected]>

* Update nemo/collections/llm/api.py

Signed-off-by: oliver könig <[email protected]>

* Update scripts/llm/evaluation.py

Signed-off-by: oliver könig <[email protected]>

* Update scripts/llm/evaluation.py

Signed-off-by: oliver könig <[email protected]>

* deprecate export-deploy as well

Signed-off-by: oliver könig <[email protected]>

* Apply isort and black reformatting

Signed-off-by: ko3n1g <[email protected]>

* f

Signed-off-by: oliver könig <[email protected]>

* Apply isort and black reformatting

Signed-off-by: ko3n1g <[email protected]>

---------

Signed-off-by: Abhishree <[email protected]>
Signed-off-by: oliver könig <[email protected]>
Signed-off-by: ko3n1g <[email protected]>
Co-authored-by: oliver könig <[email protected]>
Co-authored-by: ko3n1g <[email protected]>

Author: athitten

.github/workflows/cicd-main-export-deploy.yml

@@ -253,27 +253,6 @@ jobs:
     with:
       test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
 
-  cicd-main-export-deploy:
-    needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
-    uses: ./.github/workflows/cicd-main-export-deploy.yml
-    if: |
-      (
-        needs.pre-flight.outputs.test_to_run != '[]'
-        && (
-          contains(fromJson(needs.pre-flight.outputs.components_to_run), 'export-deploy')
-        )
-      )
-      && (
-        success()
-        || (
-          needs.cicd-wait-in-queue.result == 'skipped'
-          && needs.pre-flight.outputs.is_ci_workload == 'true'
-        )
-      )
-      && !cancelled()
-    with:
-      test_to_run: ${{ needs.pre-flight.outputs.test_to_run }}
-
   cicd-main-speech:
     needs: [pre-flight, cicd-test-container-build, cicd-main-unit-tests]
     uses: ./.github/workflows/cicd-main-speech.yml
@@ -346,7 +325,6 @@ jobs:
       - L0_Setup_Test_Data_And_Models
       - cicd-main-unit-tests
       - cicd-main-nemo2
-      - cicd-main-export-deploy
       - cicd-main-automodel
       - cicd-main-speech
     if: always()

.github/workflows/cicd-main.yml

@@ -603,6 +603,12 @@ def deploy(
     enable_flash_decode: bool = True,
     legacy_ckpt: bool = False,
 ):
+    warnings.warn(
+        "The 'deploy' function is deprecated and will be removed in NeMo FW 25.09 container release. "
+        "For evaluation functionality, please use the new Eval repository: https://github.com/NVIDIA-NeMo/Eval",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     """
     Deploys nemo model on a PyTriton server either "in-framework" or by converting to trtllm depending on the backend.
     This deploy method is intended to be used for evaluation.
@@ -791,6 +797,12 @@ def evaluate(
     eval_cfg: EvaluationConfig = EvaluationConfig(type="gsm8k"),
     adapter_cfg: AdapterConfig | None = None,
 ) -> dict:
+    warnings.warn(
+        "The 'evaluate' function is deprecated and will be removed in NeMo FW 25.09 container release. "
+        "For evaluation functionality, please use the new Eval repository: https://github.com/NVIDIA-NeMo/Eval",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     """
     Evaluates nemo model deployed on PyTriton server using nvidia-lm-eval
 

nemo/collections/llm/api.py

@@ -13,6 +13,15 @@
 # limitations under the License.
 
 
-from nemo.deploy.deploy_base import DeployBase
-from nemo.deploy.deploy_pytriton import DeployPyTriton
-from nemo.deploy.triton_deployable import ITritonDeployable
+import warnings
+
+from nemo.deploy.deploy_base import DeployBase  # noqa: F401
+from nemo.deploy.deploy_pytriton import DeployPyTriton  # noqa: F401
+from nemo.deploy.triton_deployable import ITritonDeployable  # noqa: F401
+
+warnings.warn(
+    "The 'nemo.deploy' is deprecated and will be removed in NeMo FW 25.09 container release. "
+    "For evaluation functionality, please use the new Eval repository: https://github.com/NVIDIA-NeMo/Export-Deploy",
+    DeprecationWarning,
+    stacklevel=2,
+)

nemo/deploy/__init__.py

@@ -19,3 +19,12 @@
     __all__ = ["io"]
 except (ImportError, ModuleNotFoundError):
     pass
+
+import warnings
+
+warnings.warn(
+    "The 'nemo.export' is deprecated and will be removed in NeMo FW 25.09 container release. "
+    "For evaluation functionality, please use the new Eval repository: https://github.com/NVIDIA-NeMo/Export-Deploy",
+    DeprecationWarning,
+    stacklevel=2,
+)

nemo/export/__init__.py

@@ -12,11 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import warnings
+
+warnings.warn(
+    "This evaluation script is deprecated and will be removed in NeMo FW 25.09 container release. "
+    "For evaluation functionality, please use the new Eval repository: https://github.com/NVIDIA-NeMo/Eval",
+    DeprecationWarning,
+    stacklevel=1,
+)
+
 # NOTE: This script is only an example of using NeMo with NeMo-Run's APIs and is subject to change without notice.
 # This script is used for evaluation on local and slurm executors using NeMo-Run.
 # It uses deploy method from nemo/llm/collections/api.py to deploy nemo2.0 ckpt on PyTriton server and uses evaluate
 # method from nemo/llm/collections/api.py to run evaluation on it.
 # (https://github.com/NVIDIA/NeMo-Run) to configure and execute the runs.
+# DEPRECATED: This script and the underlying deploy/evaluate functions will be removed in NeMo FW 25.09.
+# Please use the new Eval repository: https://github.com/NVIDIA-NeMo/Eval
 
 import argparse
 from typing import Optional