Merge pull request #885 from IBM/pending-version-change/0.2.4

Start of a new release cycle with 1.0.0
IBM · Dec 17, 2024 · 9e1b281 · 9e1b281
2 parents 4953e82 + ed6daf5
commit 9e1b281
Show file tree

Hide file tree

Showing 120 changed files with 191 additions and 175 deletions.
diff --git a/.make.versions b/.make.versions
@@ -19,7 +19,7 @@ DPK_MINOR_VERSION=2
 DPK_MICRO_VERSION=3
 # The suffix is generally always set in the main/development branch and only nulled out when creating release branches.
 # It can be manually incremented, for example, to allow publishing a new intermediate version wheel to pypi. 
-DPK_VERSION_SUFFIX=.dev2
+DPK_VERSION_SUFFIX=
 
 DPK_VERSION=$(DPK_MAJOR_VERSION).$(DPK_MINOR_VERSION).$(DPK_MICRO_VERSION)$(DPK_VERSION_SUFFIX)
 
@@ -66,4 +66,4 @@ endif
 #
 # If you change the versions numbers, be sure to run "make set-versions" to 
 # update version numbers across the transform (e.g., pyproject.toml).
-TRANSFORMS_PKG_VERSION=0.2.3.dev3
+TRANSFORMS_PKG_VERSION=0.2.3
diff --git a/README.md b/README.md
@@ -79,8 +79,8 @@ conda install gxx_linux-64
 Next, install the data prep toolkit library. This library installs both the python and ray versions of the transforms. For better management of dependencies, it is recommended to install the same tagged version of both the library and the transform. 
 
 ```bash
-pip3 install  'data-prep-toolkit[ray]==0.2.3.dev0'
-pip3 install  'data-prep-toolkit-transforms[ray,all]==0.2.3.dev1'
+pip3 install  'data-prep-toolkit[ray]==0.2.3'
+pip3 install  'data-prep-toolkit-transforms[all]==0.2.3'
 pip3 install jupyterlab   ipykernel  ipywidgets
 
 ## install custom kernel

diff --git a/data-processing-lib/pyproject.toml b/data-processing-lib/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit"
-version = "0.2.3.dev2"
+version = "0.2.3"
 keywords = ["data", "data preprocessing", "data preparation", "llm", "generative", "ai", "fine-tuning", "llmapps" ]
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Toolkit Library for Ray and Python"

diff --git a/kfp/kfp_ray_components/createRayClusterComponent.yaml b/kfp/kfp_ray_components/createRayClusterComponent.yaml
@@ -11,7 +11,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/deleteRayClusterComponent.yaml b/kfp/kfp_ray_components/deleteRayClusterComponent.yaml
@@ -9,7 +9,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeRayJobComponent.yaml b/kfp/kfp_ray_components/executeRayJobComponent.yaml
@@ -12,7 +12,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml b/kfp/kfp_ray_components/executeRayJobComponent_multi_s3.yaml
@@ -13,7 +13,7 @@ inputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_ray_components/executeSubWorkflowComponent.yaml b/kfp/kfp_ray_components/executeSubWorkflowComponent.yaml
@@ -27,7 +27,7 @@ outputs:
 
 implementation:
     container:
-        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+        image: "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
         # command is a list of strings (command-line arguments).
         # The YAML language has two syntaxes for lists, and you can use either of them.
         # Here we use the "flow syntax" - comma-separated strings inside square brackets.

diff --git a/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v1_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_v1"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -13,7 +13,7 @@ authors = [
 ]
 dependencies = [
     "kfp==1.8.22",
-    "data-prep-toolkit-kfp-shared==0.2.3.dev2",
+    "data-prep-toolkit-kfp-shared==0.2.3",
 ]
 
 [build-system]

diff --git a/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml b/kfp/kfp_support_lib/kfp_v2_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_v2"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -14,7 +14,7 @@ authors = [
 dependencies = [
     "kfp==2.8.0",
     "kfp-kubernetes==1.2.0",
-    "data-prep-toolkit-kfp-shared==0.2.3.dev2",
+    "data-prep-toolkit-kfp-shared==0.2.3",
 ]
 
 [build-system]

diff --git a/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml b/kfp/kfp_support_lib/shared_workflow_support/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "data_prep_toolkit_kfp_shared"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Data Preparation Kit Library. KFP support"
 license = {text = "Apache-2.0"}
@@ -14,7 +14,7 @@ authors = [
 dependencies = [
     "requests",
     "kubernetes",
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
+    "data-prep-toolkit[ray]>=0.2.3",
 ]
 
 [build-system]

diff --git a/release-notes.md b/release-notes.md
@@ -1,5 +1,21 @@
 # Data Prep Kit Release notes
 
+## Release 0.2.3 - 12/15/2024
+
+## General
+
+New algorithm for Fuzzy dedup transform
+Sample notebooks for some of the language transforms
+Integrate Semantic profiler and report generation for code profiler transform
+
+### data-prep-toolkit libraries (python, ray, spark) 
+
+1. Increase ray agent limit to 10,000 (default was 100) 
+
+### Transforms
+
+1. Fuzzy dedup new algorithm for Python, Ray and Spark
+
 ## Release 0.2.2 - 11/25/2024
 
 ### General 

diff --git a/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py b/transforms/code/code2parquet/kfp_ray/code2parquet_wf.py
@@ -25,7 +25,7 @@
 
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/code2parquet/python/pyproject.toml b/transforms/code/code2parquet/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code2parquet_transform_python"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "code2parquet Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/code2parquet/python/requirements.txt b/transforms/code/code2parquet/python/requirements.txt
@@ -1,3 +1,3 @@
-data-prep-toolkit>=0.2.3.dev2
+data-prep-toolkit>=0.2.3
 parameterized
 pandas
diff --git a/transforms/code/code2parquet/ray/pyproject.toml b/transforms/code/code2parquet/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code2parquet_transform_ray"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "code2parquet Ray Transform"
 license = {text = "Apache-2.0"}
@@ -10,8 +10,8 @@ authors = [
     { name = "Boris Lublinsky", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
-    "dpk-code2parquet-transform-python==0.2.3.dev2",
+    "data-prep-toolkit[ray]>=0.2.3",
+    "dpk-code2parquet-transform-python==0.2.3",
     "parameterized",
     "pandas",
 ]

diff --git a/transforms/code/code_profiler/python/pyproject.toml b/transforms/code/code_profiler/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_profiler_transform_python"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Code Profiler Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/code_profiler/python/requirements.txt b/transforms/code/code_profiler/python/requirements.txt
@@ -1,4 +1,4 @@
-data-prep-toolkit>=0.2.3.dev2
+data-prep-toolkit>=0.2.3
 parameterized
 pandas
 aiolimiter==1.1.0

diff --git a/transforms/code/code_profiler/ray/pyproject.toml b/transforms/code/code_profiler/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_profiler_transform_ray"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Code Profiler Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Pankaj Thorat", email = "[email protected]" },
 ]
 dependencies = [
-	"dpk-code-profiler-transform-python==0.2.3.dev2",
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
+	"dpk-code-profiler-transform-python==0.2.3",
+    "data-prep-toolkit[ray]>=0.2.3",
 	]
 
 [build-system]

diff --git a/transforms/code/code_quality/kfp_ray/code_quality_wf.py b/transforms/code/code_quality/kfp_ray/code_quality_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/code_quality-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/code_quality/python/pyproject.toml b/transforms/code/code_quality/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_quality_transform_python"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Code Quality Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/code_quality/python/requirements.txt b/transforms/code/code_quality/python/requirements.txt
@@ -1,3 +1,3 @@
-data-prep-toolkit>=0.2.3.dev2
+data-prep-toolkit>=0.2.3
 bs4==0.0.2
 transformers==4.38.2
diff --git a/transforms/code/code_quality/ray/pyproject.toml b/transforms/code/code_quality/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_code_quality_transform_ray"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Code Quality Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Shivdeep Singh", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-code-quality-transform-python==0.2.3.dev2",
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
+    "dpk-code-quality-transform-python==0.2.3",
+    "data-prep-toolkit[ray]>=0.2.3",
 ]
 
 [build-system]

diff --git a/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py b/transforms/code/header_cleanser/kfp_ray/header_cleanser_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/header_cleanser-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/header_cleanser/python/pyproject.toml b/transforms/code/header_cleanser/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_header_cleanser_transform_python"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "License and Copyright Removal Transform for Python"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/header_cleanser/python/requirements.txt b/transforms/code/header_cleanser/python/requirements.txt
@@ -1,3 +1,3 @@
-data-prep-toolkit>=0.2.3.dev2
+data-prep-toolkit>=0.2.3
 scancode-toolkit==32.1.0 ; platform_system != 'Darwin'
 
diff --git a/transforms/code/header_cleanser/ray/pyproject.toml b/transforms/code/header_cleanser/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_header_cleanser_transform_ray"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "License and copyright removal Transform for Ray"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Yash kalathiya", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-header-cleanser-transform-python==0.2.3.dev2",
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
+    "dpk-header-cleanser-transform-python==0.2.3",
+    "data-prep-toolkit[ray]>=0.2.3",
     "scancode-toolkit==32.1.0",
 ]
 

diff --git a/transforms/code/license_select/kfp_ray/license_select_wf.py b/transforms/code/license_select/kfp_ray/license_select_wf.py
@@ -25,7 +25,7 @@
 
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/license_select/python/pyproject.toml b/transforms/code/license_select/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_license_select_transform_python"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "License Select Python Transform"
 license = {text = "Apache-2.0"}

diff --git a/transforms/code/license_select/python/requirements.txt b/transforms/code/license_select/python/requirements.txt
@@ -1 +1 @@
-data-prep-toolkit>=0.2.3.dev2
+data-prep-toolkit>=0.2.3
diff --git a/transforms/code/license_select/ray/pyproject.toml b/transforms/code/license_select/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_license_select_transform_ray"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "License Select Transform"
 license = {text = "Apache-2.0"}
@@ -10,8 +10,8 @@ authors = [
     { name = "Mark Lewis", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-license-select-transform-python==0.2.3.dev2",
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
+    "dpk-license-select-transform-python==0.2.3",
+    "data-prep-toolkit[ray]>=0.2.3",
 ]
 
 [build-system]

diff --git a/transforms/code/malware/kfp_ray/malware_wf.py b/transforms/code/malware/kfp_ray/malware_wf.py
@@ -24,7 +24,7 @@
 task_image = "quay.io/dataprep1/data-prep-kit/malware-ray:latest"
 
 # components
-base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:latest"
+base_kfp_image = "quay.io/dataprep1/data-prep-kit/kfp-data-processing:0.2.3"
 
 # path to kfp component specifications files
 component_spec_path = "../../../../kfp/kfp_ray_components/"

diff --git a/transforms/code/malware/python/pyproject.toml b/transforms/code/malware/python/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_malware_transform_python"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Malware Python Transform"
 license = {text = "Apache-2.0"}
@@ -9,7 +9,7 @@ authors = [
     { name = "Takuya Goto", email = "[email protected]" },
 ]
 dependencies = [
-    "data-prep-toolkit>=0.2.3.dev2",
+    "data-prep-toolkit>=0.2.3",
     "clamd==1.0.2",
 ]
 

diff --git a/transforms/code/malware/ray/pyproject.toml b/transforms/code/malware/ray/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "dpk_malware_transform_ray"
-version = "0.2.3.dev2"
+version = "0.2.3"
 requires-python = ">=3.10,<3.13"
 description = "Malware Ray Transform"
 license = {text = "Apache-2.0"}
@@ -9,8 +9,8 @@ authors = [
     { name = "Takuya Goto", email = "[email protected]" },
 ]
 dependencies = [
-    "dpk-malware-transform-python==0.2.3.dev2",
-    "data-prep-toolkit[ray]>=0.2.3.dev2",
+    "dpk-malware-transform-python==0.2.3",
+    "data-prep-toolkit[ray]>=0.2.3",
 ]
 
 [build-system]
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		data-prep-toolkit>=0.2.3.dev2
		data-prep-toolkit>=0.2.3