Merge pull request #1355 from data-prep-kit/pending-release/1.1.2

- kfpv2 tests are failing, and will be addressed on ops repo. 
- tokenization2arrow is failing due to pulling for pypi, and will be fixed once this release if pushed to pypy.

Author: swith005

.make.versions

@@ -18,14 +18,14 @@ REQUIRED_PYTHON_VERSIONS=">=3.10,<3.13"
 # for integration testing or other
 
 DPK_NEXT_RELEASE=1.0.0
-DPK_VERSION_SUFFIX=.dev1
+DPK_VERSION_SUFFIX=
 
 
 DPK_CONNECTOR_NEXT_RELEASE=0.2.4
 DPK_CONNECTOR_SUFFIX=.dev0
 
 TRANSFORM_NEXT_RELEASE=1.1.2
-TRANSFORM_VERSION_SUFFIX=.dev0
+TRANSFORM_VERSION_SUFFIX=
 
 ################################################################################
 # Begin versions that the repo depends on. 
@@ -75,11 +75,11 @@ endif
 docker-tag::
 	@echo $(DOCKER_IMAGE_VERSION)
 
-# Printout the current releast tag for the DPK wheel
+# Printout the current release tag for the DPK wheel
 dpk-tag::
 	@echo $(DPK_VERSION)
 
-# Printout the current releast tag for the trsnsdfotmd wheel
+# Printout the current release tag for the transform wheel
 transforms-tag::
 	@echo $(TRANSFORMS_PKG_VERSION)
 

data-processing-lib/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "1.0.0.dev1"
+version = "1.0.0"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Ray and Python"

data-processing-lib/python/requirements.txt

@@ -1,10 +1,8 @@
 numpy < 1.29.0
 pyarrow==16.1.0
-# Need to revisit boto3 version
 # Later version 1.38.7 is breaking authentication. 
-# other suggestion:  boto3==1.34.69 and botocore==1.34.154
-# Go with a DMF compatible version for now
-boto3>=1.35.74,<1.35.94
+# check compatibility with other applications that depend on botocore(DMF, DSift, etc.)
+boto3>=1.35.74,<=1.38.18
 mmh3
 psutil
 polars>=1.9.0

data-processing-lib/python/src/data_processing/data_access/data_access_local.py

@@ -88,14 +88,21 @@ def __init__(
                          files_to_use=files_to_use, files_to_checkpoint=files_to_checkpoint)
 
         ######
+        ## data_config = {'input_folder': str= 'path to input folder',
+        ##               'output_folder': str='path to output folder',
+        ##              'cache' : bool = True | False}
         ## Calling DataAccessLocal.validate_config should have caught this in a production setting
         ## but we still allow the class to be created with no configuration defined. Why ?
+        self.tables = {}
+
         if config is None:
             self.input_folder = None
             self.output_folder = None
+            self.cache = False
         else:
             self.input_folder = os.path.abspath(config["input_folder"])
             self.output_folder = os.path.abspath(config["output_folder"])
+            self.cache = config.get('cache', False)
         ######
 
         logger.debug(f"Local input folder: {self.input_folder}")
@@ -162,6 +169,10 @@ def get_table(self, path: str) -> tuple[pa.table, int]:
         Returns:
             pyarrow.Table: PyArrow table if read successfully, None otherwise.
         """
+        # if the table exists in memory, use it for faster access
+        if self.tables.get(path):
+            logger.debug('Table found in memory') 
+            return self.tables[path], 0
 
         try:
             table = pq.read_table(path)
@@ -186,6 +197,10 @@ def save_table(self, path: str, table: pa.Table) -> tuple[int, dict[str, Any], i
                     - size (int): The size of the file (bytes).
                 If saving fails, file_info will be None.
         """
+        #save the table in memory for faster access
+        if self.cache:
+           self.tables[path] = table
+
         # Get table size in memory
         size_in_memory = table.nbytes
         try:

data-processing-lib/python/test/data_processing_tests/data_access/data_access_local_test.py

@@ -30,6 +30,7 @@ class TestInit:
     path_dict = {
         "input_folder": os.path.join(os.sep, "tmp", "input_guf"),
         "output_folder": os.path.join(os.sep, "tmp", "output_guf"),
+        "cache": True,
     }
     dal = DataAccessLocal(path_dict, d_sets=["dset1", "dset2"], checkpoint=True, m_files=-1)
     size_stat_dict_empty = {"max_file_size": 0.0, "min_file_size": float(GB), "total_file_size": 0.0}
@@ -464,9 +465,11 @@ def test_successful_save(self):
         with patch("os.path.getsize") as mock_getsize, patch("os.path.basename") as mock_basename:
             mock_getsize.return_value = 1024
             mock_basename.return_value = "test_file.parquet"
+            assert len(self.dal.tables) == 0
             size_in_memory, file_info, _ = self.dal.save_table(self.pq_file_path, self.table)
             os.remove(self.pq_file_path)
             # Assertions about return values
+            assert len(self.dal.tables) > 0
             assert size_in_memory == self.table.nbytes
             assert file_info == {"name": "test_file.parquet", "size": 1024}
 

kfp/kfp_ray_components/createRayClusterComponent.yaml

@@ -12,7 +12,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:1.1.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/deleteRayClusterComponent.yaml

@@ -9,7 +9,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:1.1.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/executeRayJobComponent.yaml

@@ -12,7 +12,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:1.1.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml

@@ -13,7 +13,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:1.1.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

kfp/kfp_ray_components/executeSubWorkflowComponent.yaml

@@ -27,7 +27,7 @@ outputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:1.1.2"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists, and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.