diff --git a/classifier-e2e/README.md b/classifier-e2e/README.md index e932141a..17aeadb9 100644 --- a/classifier-e2e/README.md +++ b/classifier-e2e/README.md @@ -11,58 +11,76 @@ pinned: false license: apache-2.0 --- -Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference +# ZenML MLOps Breast Cancer Classification Demo -# ๐Ÿ“œ ZenML Stack Show Case +## ๐ŸŒ Project Overview -This project aims to demonstrate the power of stacks. The code in this -project assumes that you have quite a few stacks registered already. +This is a minimalistic MLOps project demonstrating how to put machine learning +workflows into production using ZenML. The project focuses on building a breast +cancer classification model with end-to-end ML pipeline management. -## default - * `default` Orchestrator - * `default` Artifact Store +### Key Features -```commandline -zenml stack set default -python run.py --training-pipeline +- ๐Ÿ”ฌ Feature engineering pipeline +- ๐Ÿค– Model training pipeline +- ๐Ÿงช Batch inference pipeline +- ๐Ÿ“Š Artifact and model lineage tracking +- ๐Ÿ”— Integration with Weights & Biases for experiment tracking + +## ๐Ÿš€ Installation + +1. Clone the repository +2. Install requirements: + ```bash + pip install -r requirements.txt + ``` +3. Install ZenML integrations: + ```bash + zenml integration install sklearn xgboost wandb -y + zenml login + zenml init + ``` +4. You need to register a stack with a [Weights & Biases Experiment Tracker](https://docs.zenml.io/stack-components/experiment-trackers/wandb). + +## ๐Ÿง  Project Structure + +- `steps/`: Contains individual pipeline steps +- `pipelines/`: Pipeline definitions +- `run.py`: Main script to execute pipelines + +## ๐Ÿ” Workflow and Execution + +First, you need to set your stack: + +```bash +zenml stack set stack-with-wandb ``` -## local-sagemaker-step-operator-stack - * `default` Orchestrator - * `s3` Artifact Store - * `local` Image Builder - * `aws` Container Registry - * `Sagemaker` Step Operator +### 1. Data Loading and Feature Engineering -```commandline -zenml stack set local-sagemaker-step-operator-stack -zenml integration install aws -y -python run.py --training-pipeline +- Uses the Breast Cancer dataset from scikit-learn +- Splits data into training and inference sets +- Preprocesses data for model training + +```bash +python run.py --feature-pipeline ``` -## sagemaker-airflow-stack - * `Airflow` Orchestrator - * `s3` Artifact Store - * `local` Image Builder - * `aws` Container Registry - * `Sagemaker` Step Operator - -```commandline -zenml stack set sagemaker-airflow-stack -zenml integration install airflow -y -pip install apache-airflow-providers-docker apache-airflow~=2.5.0 -zenml stack up +### 2. Model Training + +- Supports multiple model types (SGD, XGBoost) +- Evaluates and compares model performance +- Tracks model metrics with Weights & Biases + +```bash python run.py --training-pipeline ``` -## sagemaker-stack - * `Sagemaker` Orchestrator - * `s3` Artifact Store - * `local` Image Builder - * `aws` Container Registry - * `Sagemaker` Step Operator +### 3. Batch Inference -```commandline -zenml stack set sagemaker-stack -python run.py --training-pipeline +- Loads production model +- Generates predictions on new data + +```bash +python run.py --inference-pipeline ``` diff --git a/classifier-e2e/requirements.txt b/classifier-e2e/requirements.txt index c3fd2c34..499700e7 100644 --- a/classifier-e2e/requirements.txt +++ b/classifier-e2e/requirements.txt @@ -1,4 +1,4 @@ -zenml[server]>=0.55.2 +zenml[server]>=0.70.0 notebook scikit-learn<1.3 s3fs>2022.3.0,<=2023.4.0 diff --git a/classifier-e2e/run_full.ipynb b/classifier-e2e/run_full.ipynb index afaa6001..26f1fb90 100644 --- a/classifier-e2e/run_full.ipynb +++ b/classifier-e2e/run_full.ipynb @@ -38,7 +38,7 @@ "source": [ "! pip3 install -r requirements.txt\n", "! zenml integration install sklearn xgboost -y\n", - "! zenml connect --url https://1cf18d95-zenml.cloudinfra.zenml.io \n", + "! zenml login https://1cf18d95-zenml.cloudinfra.zenml.io \n", "\n", "import IPython\n", "IPython.Application.instance().kernel.do_shutdown(restart=True)" @@ -941,10 +941,17 @@ " .ravel()\n", " .tolist(),\n", " }\n", - " log_model_metadata(metadata={\"wandb_url\": wandb.run.url})\n", - " log_artifact_metadata(\n", + "\n", + " try:\n", + " if get_step_context().model:\n", + " log_metadata(metadata=metadata, infer_model=True)\n", + " except StepContextError:\n", + " # If a model is not configured, it is not able to log metadata\n", + " pass\n", + "\n", + " log_metadata(\n", " metadata=metadata,\n", - " artifact_name=\"breast_cancer_classifier\",\n", + " artifact_version_id=get_step_context().inputs[\"model\"].id,\n", " )\n", "\n", " wandb.log({\"train_accuracy\": metadata[\"train_accuracy\"]})\n", @@ -1073,6 +1080,7 @@ { "cell_type": "code", "execution_count": null, + "id": "1e2130b9", "metadata": {}, "outputs": [], "source": [ @@ -1083,6 +1091,7 @@ { "cell_type": "code", "execution_count": null, + "id": "476cbf5c", "metadata": {}, "outputs": [], "source": [ @@ -1091,6 +1100,7 @@ }, { "cell_type": "markdown", + "id": "75df10e7", "metadata": {}, "source": [ "Now full run executed on local stack and experiment is tracked using Model Control Plane and Weights&Biases.\n", @@ -1103,6 +1113,7 @@ { "cell_type": "code", "execution_count": null, + "id": "bfd6345f", "metadata": {}, "outputs": [], "source": [ @@ -1113,6 +1124,7 @@ { "cell_type": "code", "execution_count": null, + "id": "24358031", "metadata": {}, "outputs": [], "source": [ @@ -1136,7 +1148,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/classifier-e2e/run_skip_basics.ipynb b/classifier-e2e/run_skip_basics.ipynb index 697a9ce8..62da7a32 100644 --- a/classifier-e2e/run_skip_basics.ipynb +++ b/classifier-e2e/run_skip_basics.ipynb @@ -38,7 +38,7 @@ "source": [ "! pip3 install -r requirements.txt\n", "! zenml integration install sklearn xgboost -y\n", - "! zenml connect --url https://1cf18d95-zenml.cloudinfra.zenml.io \n", + "! zenml login https://1cf18d95-zenml.cloudinfra.zenml.io \n", "\n", "import IPython\n", "IPython.Application.instance().kernel.do_shutdown(restart=True)" @@ -829,10 +829,17 @@ " .ravel()\n", " .tolist(),\n", " }\n", - " log_model_metadata(metadata={\"wandb_url\": wandb.run.url})\n", - " log_artifact_metadata(\n", + "\n", + " try:\n", + " if get_step_context().model:\n", + " log_metadata(metadata=metadata, infer_model=True)\n", + " except StepContextError:\n", + " # If a model is not configured, it is not able to log metadata\n", + " pass\n", + "\n", + " log_metadata(\n", " metadata=metadata,\n", - " artifact_name=\"breast_cancer_classifier\",\n", + " artifact_version_id=get_step_context().inputs[\"model\"].id,\n", " )\n", "\n", " wandb.log({\"train_accuracy\": metadata[\"train_accuracy\"]})\n", @@ -1211,7 +1218,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/classifier-e2e/steps/deploy_endpoint.py b/classifier-e2e/steps/deploy_endpoint.py index ad166525..9f1c56b8 100644 --- a/classifier-e2e/steps/deploy_endpoint.py +++ b/classifier-e2e/steps/deploy_endpoint.py @@ -7,6 +7,7 @@ from utils.aws import get_aws_config from utils.sagemaker_materializer import SagemakerPredictorMaterializer from zenml import ArtifactConfig, get_step_context, log_artifact_metadata, step +from zenml.enums import ArtifactType @step( @@ -16,7 +17,10 @@ def deploy_endpoint() -> ( Annotated[ Predictor, - ArtifactConfig(name="sagemaker_endpoint", is_deployment_artifact=True), + ArtifactConfig( + name="sagemaker_endpoint", + artifact_type=ArtifactType.SERVICE + ), ] ): role, session, region = get_aws_config() diff --git a/classifier-e2e/steps/model_evaluator.py b/classifier-e2e/steps/model_evaluator.py index dd335ae5..54ee2fbf 100644 --- a/classifier-e2e/steps/model_evaluator.py +++ b/classifier-e2e/steps/model_evaluator.py @@ -21,12 +21,7 @@ import wandb from sklearn.base import ClassifierMixin from sklearn.metrics import confusion_matrix -from zenml import ( - get_step_context, - log_artifact_metadata, - log_model_metadata, - step, -) +from zenml import step, log_metadata, get_step_context from zenml.client import Client from zenml.exceptions import StepContextError from zenml.logger import get_logger @@ -60,12 +55,12 @@ def model_evaluator( step to force the pipeline run to fail early and all subsequent steps to be skipped. - This step is parameterized to configure the step independently of the step code, - before running it in a pipeline. In this example, the step can be configured - to use different values for the acceptable model performance thresholds and - to control whether the pipeline run should fail if the model performance - does not meet the minimum criteria. See the documentation for more - information: + This step is parameterized to configure the step independently of the step + code, before running it in a pipeline. In this example, the step can be + configured to use different values for the acceptable model performance + thresholds and to control whether the pipeline run should fail if the model + performance does not meet the minimum criteria. See the documentation for + more information: https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines @@ -89,17 +84,19 @@ def model_evaluator( dataset_tst.drop(columns=[target]), dataset_tst[target], ) - logger.info(f"Train accuracy={trn_acc*100:.2f}%") - logger.info(f"Test accuracy={tst_acc*100:.2f}%") + logger.info(f"Train accuracy={trn_acc * 100:.2f}%") + logger.info(f"Test accuracy={tst_acc * 100:.2f}%") messages = [] if trn_acc < min_train_accuracy: messages.append( - f"Train accuracy {trn_acc*100:.2f}% is below {min_train_accuracy*100:.2f}% !" + f"Train accuracy {trn_acc * 100:.2f}% is below " + f"{min_train_accuracy * 100:.2f}% !" ) if tst_acc < min_test_accuracy: messages.append( - f"Test accuracy {tst_acc*100:.2f}% is below {min_test_accuracy*100:.2f}% !" + f"Test accuracy {tst_acc * 100:.2f}% is below " + f"{min_test_accuracy * 100:.2f}% !" ) else: for message in messages: @@ -115,14 +112,14 @@ def model_evaluator( } try: if get_step_context().model: - log_model_metadata(metadata={"wandb_url": wandb.run.url}) + log_metadata(metadata=metadata, infer_model=True) except StepContextError: # if model not configured not able to log metadata pass - log_artifact_metadata( + log_metadata( metadata=metadata, - artifact_name="breast_cancer_classifier", + artifact_version_id=get_step_context().inputs["model"].id, ) wandb.log( diff --git a/classifier-e2e/steps/model_trainer.py b/classifier-e2e/steps/model_trainer.py index aef08e27..3a04f1d8 100644 --- a/classifier-e2e/steps/model_trainer.py +++ b/classifier-e2e/steps/model_trainer.py @@ -13,7 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# from typing import Optional @@ -23,6 +22,7 @@ from typing_extensions import Annotated from utils.sagemaker_materializer import SagemakerMaterializer from zenml import ArtifactConfig, step +from zenml.enums import ArtifactType from zenml.logger import get_logger logger = get_logger(__name__) @@ -39,7 +39,10 @@ def model_trainer( target: Optional[str] = "target", ) -> Annotated[ ClassifierMixin, - ArtifactConfig(name="breast_cancer_classifier", is_model_artifact=True), + ArtifactConfig( + name="breast_cancer_classifier", + artifact_tyoe=ArtifactType.MODEL, + ), ]: """Configure and train a model on the training dataset. diff --git a/customer-satisfaction/README.md b/customer-satisfaction/README.md index 65681caa..6e581544 100644 --- a/customer-satisfaction/README.md +++ b/customer-satisfaction/README.md @@ -33,6 +33,7 @@ your choice, run: git clone https://github.com/zenml-io/zenml-projects.git cd zenml-projects/customer-satisfaction pip install -r requirements.txt +zenml integration install lightgbm xgboost sklearn mlflow -y ``` ZenML comes bundled with a React-based dashboard. @@ -46,7 +47,7 @@ In case you already have an account, here is how you connect to a deployed server. ```bash -zenml connect -u +zenml login ``` To run locally, you need @@ -55,22 +56,14 @@ but first you must install the optional dependencies for the ZenML server: ```bash pip install zenml["server"] -zenml up +zenml login --local ``` -If you are running the `run_deployment.py` script, you will also need to install -some integrations using ZenML: - -```bash -zenml integration install mlflow -y -``` - The project can only be executed with a ZenML stack that has an MLflow experiment tracker and model deployer as a component. Configuring a new stack with the two components are as follows: ```bash -zenml integration install mlflow -y zenml experiment-tracker register mlflow_tracker --flavor=mlflow zenml model-deployer register mlflow --flavor=mlflow zenml stack register local-mlflow-stack -a default -o default -d mlflow -e mlflow_tracker --set @@ -237,7 +230,7 @@ A browser window should open for you and let you configure a product to run a pr zenml integration install mlflow -y ``` -2. If you are trying to start the ZenML server with `zenml up`, if you're running +2. If you are trying to start the ZenML server with `zenml login --local`, if you're running on a Mac, you might want to set the following environment variable in your `.zshrc` file or in the environment in which you're running the pipeline: diff --git a/customer-satisfaction/requirements.txt b/customer-satisfaction/requirements.txt index 7b7a64ec..448a07ad 100644 --- a/customer-satisfaction/requirements.txt +++ b/customer-satisfaction/requirements.txt @@ -1,10 +1,6 @@ -catboost==1.0.4 +zenml>=0.70.0 joblib>=1.1.0 -lightgbm==4.1.0 -optuna==2.10.0 -streamlit==1.29.0 -xgboost==2.0.3 -markupsafe==1.1.1 -zenml>=0.52.0 -scikit-learn>=1.3.2 +optuna +streamlit +markupsafe altair diff --git a/customer-satisfaction/steps/model_promoter.py b/customer-satisfaction/steps/model_promoter.py index 8276156e..25142eb7 100644 --- a/customer-satisfaction/steps/model_promoter.py +++ b/customer-satisfaction/steps/model_promoter.py @@ -30,8 +30,7 @@ def model_promoter(mse: float, stage: str = "production") -> bool: # In case there already is a model version at the correct stage previous_production_model_mse = float( previous_production_model.get_artifact("sklearn_regressor") - .run_metadata["metrics"] - .value["mse"] + .run_metadata["metrics"]["mse"] ) except RuntimeError: # In case no model version has been promoted before, diff --git a/databricks-demo/README.md b/databricks-demo/README.md index 41e24e2d..d490ae21 100644 --- a/databricks-demo/README.md +++ b/databricks-demo/README.md @@ -51,9 +51,8 @@ source .venv/bin/activate make setup # Optionally, provision default local stack make install-stack-local -# Start the ZenML UI locally (recommended, but optional); -# the default username is "admin" with an empty password -zenml up +# Start the ZenML UI locally (recommended, but optional) +zenml login --local # Run the pipeline included in the project python run.py ``` diff --git a/databricks-demo/requirements.txt b/databricks-demo/requirements.txt index 75c16209..b3922fd9 100644 --- a/databricks-demo/requirements.txt +++ b/databricks-demo/requirements.txt @@ -1 +1 @@ -zenml[server] +zenml[server]>=0.70.0 diff --git a/end-to-end-computer-vision/README.md b/end-to-end-computer-vision/README.md index 20fcf34b..11e4d2a4 100644 --- a/end-to-end-computer-vision/README.md +++ b/end-to-end-computer-vision/README.md @@ -32,7 +32,7 @@ deployed instance of ZenML: ```bash pip install -r requirements.txt -zenml integration install torch gcp mlflow label_studio -y +zenml integration install pytorch gcp mlflow label_studio -y pip uninstall wandb # This comes in automatically ``` @@ -41,7 +41,6 @@ need to install them: ```bash fiftyone plugins download https://github.com/jacobmarks/fiftyone-albumentations-plugin - fiftyone plugins download https://github.com/voxel51/fiftyone-plugins --plugin-names @voxel51/annotation ``` @@ -55,7 +54,7 @@ export WANDB_DISABLED=True ### Connect to your deployed ZenML instance ```bash -zenml connect --url +zenml login ``` ## Cloud Provider @@ -64,9 +63,7 @@ We will use GCP in the commands listed below, but it will work for other cloud providers. 1) Follow our guide to set up your credentials for GCP [here](https://docs.zenml.io/how-to/auth-management/gcp-service-connector) - 2) Set up a bucket in GCP to persist your training data - 3) Set up a bucket to use as artifact store within ZenML Learn how to set up a GCP artifact store stack component within ZenML [here](https://docs.zenml.io/stack-components/artifact-stores) @@ -76,18 +73,18 @@ Learn how to set up a Vertex orchestrator stack component within ZenML 5) For training on accelerators like GPUs/TPUs set up Vertex Learn how to set up a Vertex step operator stack component within ZenML [here](https://docs.zenml.io/stack-components/step-operators/vertex) -6) Set up a Container Registry in GCP. Learn how to set up a google cloud container registry component within ZenML +6) Set up a Container Registry in GCP. Learn how to set up a Google Cloud Container Registry component within ZenML [here](https://docs.zenml.io/stack-components/container-registries/gcp) ## Label Studio 1) [Start Label Studio locally](https://labelstud.io/guide/start) -For Label Studio we recommend using docker/docker-compose to deploy a local instance -```bash -git clone https://github.com/HumanSignal/label-studio.git -cd label-studio -docker-compose up -d # starts label studio at http://localhost:8080 -``` + For Label Studio we recommend using docker/docker-compose to deploy a local instance + ```bash + git clone https://github.com/HumanSignal/label-studio.git + cd label-studio + docker-compose up -d # starts label studio at http://localhost:8080 + ``` 2) [Follow these ZenML instructions to set up Label Studio as a stack component](https://docs.zenml.io/stack-components/annotators/label-studio#how-to-deploy-it) 3) Create a project within Label Studio and name it `ship_detection_gcp` ![img.png](_assets/project_creation_label_studio.png) @@ -95,8 +92,8 @@ docker-compose up -d # starts label studio at http://localhost:8080 ![img.png](_assets/labeling_setup.png) In the following screen you now need to configure the labeling interface. This is where you define the different classes that you want to detect. In our case this should be a single `ship` class. ![img.png](_assets/labeling_interface.png) -Additionally you might want to allow users to zoom during labeling. This can be configured when you scroll down on this same screen. -6) [Set up Label Studio to use external storage](https://labelstud.io/guide/storage) +Additionally, you might want to allow users to zoom during labeling. This can be configured when you scroll down on this same screen. +5) [Set up Label Studio to use external storage](https://labelstud.io/guide/storage) Use the first bucket that you created for data persistence ## Hugging Face @@ -134,7 +131,7 @@ zenml stack register -o -a python run.py --ingest ``` -## data_export_pipeline +## `data_export_pipeline` This pipeline exports the annotations from Label Studio and loads it into the ZenML artifact store to make them accessible to downstream pipelines. ### Configure this pipeline + The configuration file for this pipeline lives at `./configs/data_export.yaml`. Make sure in particular to change `dataset_name` to reflect the name of the dataset within Label Studio. @@ -180,11 +179,12 @@ zenml stack set python run.py --export ``` -## training_pipeline +## `training_pipeline` This pipeline trains a YOLOv8 object detection model. ### Configure this pipeline + You can choose to run this pipeline locally or on the cloud. These two options use two different configuration files. For local training: `./configs/training_pipeline.yaml`. For training on the cloud: @@ -216,7 +216,7 @@ zenml stack set python run.py --training ``` -## inference_pipeline +## `inference_pipeline` This pipeline performs inference on the object detection model. @@ -253,4 +253,4 @@ python run.py --fiftyone ``` Within FiftyOne, you can now analyze all the predictions and export them back to -Label Studio for finetuned labeling and retraining. +Label Studio for fine-tuned labeling and retraining. diff --git a/end-to-end-computer-vision/requirements.txt b/end-to-end-computer-vision/requirements.txt index 11282667..7895809c 100644 --- a/end-to-end-computer-vision/requirements.txt +++ b/end-to-end-computer-vision/requirements.txt @@ -1,4 +1,4 @@ -zenml[server]>=0.55.3 +zenml[server]>=0.70.0 notebook scikit-learn<1.3 pyarrow diff --git a/end-to-end-computer-vision/steps/export_label_studio.py b/end-to-end-computer-vision/steps/export_label_studio.py index 076149c0..c2b84f52 100644 --- a/end-to-end-computer-vision/steps/export_label_studio.py +++ b/end-to-end-computer-vision/steps/export_label_studio.py @@ -16,7 +16,7 @@ # from typing import Annotated, List, Tuple -from zenml import log_artifact_metadata, step +from zenml import log_metadata, step from zenml.client import Client from zenml.logger import get_logger @@ -67,11 +67,12 @@ def load_data_from_label_studio( current_labeled_task_ids = dataset.get_labeled_tasks_ids() ls_dataset.task_ids = current_labeled_task_ids - log_artifact_metadata( + log_metadata( metadata={ "num_images": len(current_labeled_task_ids), }, artifact_name=LABELED_DATASET_NAME, + infer_artifact=True, ) return ls_dataset, current_labeled_task_ids except: diff --git a/end-to-end-computer-vision/steps/fiftyone_inference.py b/end-to-end-computer-vision/steps/fiftyone_inference.py index 8c33f7eb..c05e1736 100644 --- a/end-to-end-computer-vision/steps/fiftyone_inference.py +++ b/end-to-end-computer-vision/steps/fiftyone_inference.py @@ -18,7 +18,7 @@ from typing import Annotated import fiftyone as fo -from zenml import log_artifact_metadata, step +from zenml import log_metadata, step from zenml.client import Client from zenml.io import fileio from zenml.logger import get_logger @@ -72,8 +72,9 @@ def create_fiftyone_dataset( dataset.apply_model(yolo_model, label_field="boxes") - log_artifact_metadata( - artifact_name="predictions_dataset_json", + log_metadata( + artifact_name=PREDICTIONS_DATASET_ARTIFACT_NAME, + infer_artifact=True, metadata={ "summary_info": dataset.summary(), "persistence": dataset.persistent, diff --git a/end-to-end-computer-vision/steps/load_model.py b/end-to-end-computer-vision/steps/load_model.py index 827b691e..56932455 100644 --- a/end-to-end-computer-vision/steps/load_model.py +++ b/end-to-end-computer-vision/steps/load_model.py @@ -13,11 +13,12 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + from typing import Annotated from ultralytics import YOLO from zenml import ArtifactConfig, step +from zenml.enums import ArtifactType from zenml.logger import get_logger logger = get_logger(__name__) @@ -26,7 +27,7 @@ @step(enable_cache=True, enable_step_logs=False) def load_model( model_checkpoint: str, -) -> Annotated[YOLO, ArtifactConfig(name="Raw_YOLO", is_model_artifact=True)]: +) -> Annotated[YOLO, ArtifactConfig(name="Raw_YOLO", artifact_type=ArtifactType.MODEL)]: """Loads a YOLO model from a checkpoint. Args: diff --git a/end-to-end-computer-vision/steps/train_model.py b/end-to-end-computer-vision/steps/train_model.py index 302cd9a1..e1cf3dc3 100644 --- a/end-to-end-computer-vision/steps/train_model.py +++ b/end-to-end-computer-vision/steps/train_model.py @@ -13,11 +13,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# + from typing import Annotated, Any, Dict, Tuple from ultralytics import YOLO -from zenml import ArtifactConfig, log_artifact_metadata, step +from zenml import ArtifactConfig, log_metadata, step from zenml.logger import get_logger from materializers.label_studio_export_materializer import ( @@ -25,7 +25,7 @@ ) from materializers.ultralytics_materializer import UltralyticsMaterializer from utils.dataset_utils import load_and_split_data - +from zenml.enums import ArtifactType logger = get_logger(__name__) @@ -45,7 +45,7 @@ def train_model( is_apple_silicon_env: bool = False, ) -> Tuple[ Annotated[ - YOLO, ArtifactConfig(name="Trained_YOLO", is_model_artifact=True) + YOLO, ArtifactConfig(name="Trained_YOLO", artifact_type=ArtifactType.MODEL) ], Annotated[Dict[str, Any], "validation_metrics"], Annotated[Dict[str, Any], "model_names"], @@ -107,8 +107,9 @@ def train_model( logger.info("Evaluating model...") metrics = model.val() # evaluate model performance on the validation set - log_artifact_metadata( + log_metadata( artifact_name="Trained_YOLO", + infer_artifact=True, metadata={"metrics": metrics.results_dict, "names": model.names}, ) diff --git a/explainability-shap/requirements.txt b/explainability-shap/requirements.txt index 9c206ca2..da6d548a 100644 --- a/explainability-shap/requirements.txt +++ b/explainability-shap/requirements.txt @@ -2,6 +2,7 @@ scikit-learn shap matplotlib scipy -zenml +zenml>=0.70.0 pyarrow -fastparquet \ No newline at end of file +fastparquet +numpy<2.0.0 \ No newline at end of file diff --git a/explainability-shap/run.ipynb b/explainability-shap/run.ipynb index fe7af0d1..b1026ef9 100644 --- a/explainability-shap/run.ipynb +++ b/explainability-shap/run.ipynb @@ -15,12 +15,12 @@ "metadata": {}, "outputs": [], "source": [ - "!zenml connect --url=https://d13d987c-zenml.cloudinfra.zenml.io" + "!zenml login https://d13d987c-zenml.cloudinfra.zenml.io" ] }, { "cell_type": "code", - "execution_count": 32, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -29,7 +29,7 @@ }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -38,48 +38,9 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[?25l\u001b[32mโ ‹\u001b[0m Describing the stack...\n", - "\u001b[2K\u001b[1A\u001b[2K\u001b[3m Stack Configuration \u001b[0m\n", - "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ฏโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”“\n", - "โ”ƒ\u001b[1m \u001b[0m\u001b[1mCOMPONENT_TYPE \u001b[0m\u001b[1m \u001b[0mโ”‚\u001b[1m \u001b[0m\u001b[1mCOMPONENT_NAME \u001b[0m\u001b[1m \u001b[0mโ”ƒ\n", - "โ” โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”จ\n", - "โ”ƒ ORCHESTRATOR โ”‚ default โ”ƒ\n", - "โ” โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”จ\n", - "โ”ƒ STEP_OPERATOR โ”‚ aws-sagemaker-pipelines โ”ƒ\n", - "โ” โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”จ\n", - "โ”ƒ ARTIFACT_STORE โ”‚ aws-sagemaker-pipelines โ”ƒ\n", - "โ” โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”จ\n", - "โ”ƒ CONTAINER_REGISTRY โ”‚ aws-sagemaker-pipelines โ”ƒ\n", - "โ” โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”จ\n", - "โ”ƒ IMAGE_BUILDER โ”‚ aws-sagemaker-pipelines โ”ƒ\n", - "โ”—โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ทโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”›\n", - "\u001b[2;3m 'local-aws-step-operator' stack \u001b[0m\n", - "\u001b[32mโ ™\u001b[0m Describing the stack...\n", - "\u001b[2K\u001b[1A\u001b[2K\u001b[3m Labels \u001b[0m\n", - "โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ฏโ”โ”โ”โ”โ”โ”โ”โ”“\n", - "โ”ƒ\u001b[1m \u001b[0m\u001b[1mLABEL \u001b[0m\u001b[1m \u001b[0mโ”‚\u001b[1m \u001b[0m\u001b[1mVALUE\u001b[0m\u001b[1m \u001b[0mโ”ƒ\n", - "โ” โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”จ\n", - "โ”ƒ zenml:full_stack โ”‚ True โ”ƒ\n", - "โ”—โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ทโ”โ”โ”โ”โ”โ”โ”โ”›\n", - "\u001b[32mโ ™\u001b[0m Describing the stack...\n", - "\u001b[2K\u001b[1A\u001b[2K\u001b[2;36mStack \u001b[0m\u001b[2;32m'local-aws-step-operator'\u001b[0m\u001b[2;36m with id \u001b[0m\u001b[2;32m'1a600008-41e2-448e-b71f-5f525b564730'\u001b[0m\u001b[2;36m \u001b[0m\n", - "\u001b[2;36mis owned by user stefan@zenml.io.\u001b[0m\n", - "\u001b[2;32mโ ™\u001b[0m\u001b[2;36m Describing the stack...\u001b[0m\n", - "\u001b[2K\u001b[1A\u001b[2K\u001b[32mโ ™\u001b[0m Describing the stack...\n", - "\n", - "\u001b[1A\u001b[2K\u001b[1A\u001b[2K\u001b[2;36mDashboard URL: \u001b[0m\n", - "\u001b[2;4;94mhttps://d13d987c-zenml.cloudinfra.zenml.io/workspaces/default/stacks/1a600008-41\u001b[0m\n", - "\u001b[2;4;94me2-448e-b71f-5f525b564730/configuration\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "!zenml stack describe 'local-aws-step-operator'\n", "Client().activate_stack('local-aws-step-operator')" @@ -87,7 +48,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -97,7 +58,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -105,8 +66,7 @@ "import numpy as np\n", "from sklearn.datasets import load_iris\n", "from sklearn.model_selection import train_test_split\n", - "from zenml import step\n", - "from zenml import log_artifact_metadata\n", + "from zenml import step, log_metadata\n", "from typing import Tuple, Dict, Any\n", "from typing_extensions import Annotated\n", "\n", @@ -117,6 +77,7 @@ " metadata[\"columns\"] = list(data.columns)\n", " return metadata\n", "\n", + "\n", "@step\n", "def load_data() -> Tuple[\n", " Annotated[pd.DataFrame, \"X_train\"],\n", @@ -128,12 +89,15 @@ " iris = load_iris(as_frame=True)\n", " X = iris.data\n", " y = iris.target\n", - " X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)\n", + " X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2,\n", + " random_state=42)\n", "\n", - " for name, data in [(\"X_train\", X_train), (\"X_test\", X_test), (\"y_train\", y_train), (\"y_test\", y_test)]:\n", - " log_artifact_metadata(\n", + " for name, data in [(\"X_train\", X_train), (\"X_test\", X_test),\n", + " (\"y_train\", y_train), (\"y_test\", y_test)]:\n", + " log_metadata(\n", " artifact_name=name,\n", - " metadata={\"dataset_info\": safe_metadata(data)}\n", + " metadata={\"dataset_info\": safe_metadata(data)},\n", + " infer_artifact=True,\n", " )\n", "\n", " return X_train, X_test, y_train, y_test" @@ -141,17 +105,17 @@ }, { "cell_type": "code", - "execution_count": 55, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "from sklearn.svm import SVC\n", - "from zenml import step, ArtifactConfig\n", - "from zenml import log_model_metadata, log_artifact_metadata\n", + "from zenml import step, ArtifactConfig, log_metadata\n", "from typing_extensions import Annotated\n", "from zenml.config import ResourceSettings\n", "from zenml.integrations.aws.flavors.sagemaker_step_operator_flavor import SagemakerStepOperatorSettings\n", + "from zenml.enums import ArtifactType\n", "\n", "@step(\n", " enable_cache=False,\n", @@ -163,13 +127,13 @@ "def train_model(\n", " X_train: pd.DataFrame,\n", " y_train: pd.Series,\n", - ") -> Annotated[SVC, ArtifactConfig(name=\"model\", is_model_artifact=True)]:\n", + ") -> Annotated[SVC, ArtifactConfig(name=\"model\", artifact_type=ArtifactType.MODEL)]:\n", " \"\"\"Train an SVM classifier.\"\"\"\n", " model = SVC(kernel='rbf', probability=True)\n", " model.fit(X_train, y_train)\n", " train_accuracy = model.score(X_train, y_train)\n", "\n", - " log_model_metadata(\n", + " log_metadata(\n", " metadata={\n", " \"training_metrics\": {\n", " \"train_accuracy\": float(train_accuracy),\n", @@ -178,18 +142,20 @@ " \"model_type\": type(model).__name__,\n", " \"kernel\": model.kernel,\n", " }\n", - " }\n", + " },\n", + " infer_model=True,\n", " )\n", "\n", - " log_artifact_metadata(\n", - " artifact_name=\"model\",\n", + " log_metadata(\n", " metadata={\n", " \"model_details\": {\n", " \"type\": type(model).__name__,\n", " \"kernel\": model.kernel,\n", " \"n_support\": model.n_support_.tolist(),\n", " }\n", - " }\n", + " },\n", + " artifact_name=\"model\",\n", + " infer_artifact=True,\n", " )\n", "\n", " return model" @@ -197,15 +163,14 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "from sklearn.svm import SVC\n", - "from zenml import step\n", - "from zenml import log_model_metadata, log_artifact_metadata\n", + "from zenml import step, log_metadata\n", "from typing import Tuple\n", "from typing_extensions import Annotated\n", "\n", @@ -223,33 +188,36 @@ " predictions = model.predict(X_test)\n", " probabilities = model.predict_proba(X_test)\n", "\n", - " log_model_metadata(\n", + " log_metadata(\n", " metadata={\n", " \"evaluation_metrics\": {\n", " \"test_accuracy\": float(test_accuracy),\n", " }\n", - " }\n", + " },\n", + " infer_model=True,\n", " )\n", "\n", - " log_artifact_metadata(\n", - " artifact_name=\"predictions\",\n", + " log_metadata(\n", " metadata={\n", " \"prediction_info\": {\n", " \"shape\": predictions.shape,\n", " \"unique_values\": np.unique(predictions).tolist()\n", " }\n", - " }\n", + " },\n", + " artifact_name=\"predictions\",\n", + " infer_artifact=True,\n", " )\n", "\n", - " log_artifact_metadata(\n", - " artifact_name=\"probabilities\",\n", + " log_metadata(\n", " metadata={\n", " \"probability_info\": {\n", " \"shape\": probabilities.shape,\n", " \"min\": float(np.min(probabilities)),\n", " \"max\": float(np.max(probabilities))\n", " }\n", - " }\n", + " },\n", + " artifact_name=\"probabilities\",\n", + " infer_artifact=True,\n", " )\n", "\n", " return predictions, probabilities" @@ -257,7 +225,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -266,8 +234,7 @@ "import pandas as pd\n", "import shap\n", "from sklearn.svm import SVC\n", - "from zenml import step\n", - "from zenml import log_artifact_metadata\n", + "from zenml import step, log_metadata\n", "from typing import Dict\n", "from typing_extensions import Annotated\n", "import matplotlib.pyplot as plt\n", @@ -288,18 +255,22 @@ " X_train: pd.DataFrame\n", ") -> Annotated[SHAPVisualization, \"shap_visualization\"]:\n", " \"\"\"Generate SHAP values for model explainability and create a visualization.\"\"\"\n", - " explainer = shap.KernelExplainer(model.predict_proba, shap.sample(X_train, 100))\n", + " explainer = shap.KernelExplainer(\n", + " model.predict_proba,\n", + " shap.sample(X_train, 100)\n", + " )\n", " shap_values = explainer.shap_values(X_train.iloc[:100])\n", "\n", - " log_artifact_metadata(\n", - " artifact_name=\"shap_values\",\n", + " log_metadata(\n", " metadata={\n", " \"shap_info\": {\n", " \"shape\": [arr.shape for arr in shap_values],\n", " \"n_classes\": len(shap_values),\n", " \"n_features\": shap_values[0].shape[1],\n", " }\n", - " }\n", + " },\n", + " artifact_name=\"shap_visualization\",\n", + " infer_artifact=True,\n", " )\n", "\n", " return SHAPVisualization(shap_values, X_train.columns)" @@ -307,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -329,13 +300,14 @@ " _, p_value = ks_2samp(X_train[column], X_test[column])\n", " drift_metrics[column] = p_value\n", "\n", - " log_artifact_metadata(\n", - " artifact_name=\"drift_metrics\",\n", + " log_metadata(\n", " metadata={\n", " \"drift_summary\": {\n", " \"high_drift_features\": [col for col, p in drift_metrics.items() if p < 0.05]\n", " }\n", - " }\n", + " },\n", + " artifact_name=\"drift_metrics\",\n", + " infer_artifact=True,\n", " )\n", "\n", " return drift_metrics" @@ -343,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -368,30 +340,13 @@ }, { "cell_type": "code", - "execution_count": 64, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[1;35mInitiating a new run for the pipeline: \u001b[0m\u001b[1;36miris_classification_pipeline\u001b[1;35m.\u001b[0m\n", - "\u001b[1;35mArchiving notebook code...\u001b[0m\n", - "\u001b[1;35mCode already exists in artifact store, skipping upload.\u001b[0m\n" - ] - } - ], + "outputs": [], "source": [ "# Run the pipeline\n", "pipeline_run = iris_classification_pipeline()" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { @@ -410,7 +365,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.11.3" } }, "nbformat": 4, diff --git a/explainability-shap/run.py b/explainability-shap/run.py index faf6ea23..5a48e90d 100644 --- a/explainability-shap/run.py +++ b/explainability-shap/run.py @@ -14,26 +14,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import io +import os from typing import Tuple, Dict, Any -from typing_extensions import Annotated -import pandas as pd +import matplotlib.pyplot as plt import numpy as np +import pandas as pd +import shap +from scipy.stats import ks_2samp from sklearn.datasets import load_iris from sklearn.model_selection import train_test_split from sklearn.svm import SVC -import shap -import matplotlib.pyplot as plt -from scipy.stats import ks_2samp - -from zenml import pipeline, step, Model, ArtifactConfig -from zenml.logger import get_logger -from zenml import log_artifact_metadata, log_model_metadata +from typing_extensions import Annotated +from zenml import pipeline, step, Model, ArtifactConfig, log_metadata +from zenml.config import DockerSettings from zenml.enums import ArtifactType, VisualizationType from zenml.io import fileio -from zenml.config import DockerSettings +from zenml.logger import get_logger from zenml.materializers.base_materializer import BaseMaterializer logger = get_logger(__name__) @@ -55,7 +53,8 @@ def save_visualizations( self, data: SHAPVisualization ) -> Dict[str, VisualizationType]: plt.figure(figsize=(10, 6)) - shap.summary_plot(data.shap_values, feature_names=data.feature_names, plot_type="bar", show=False) + shap.summary_plot(data.shap_values, feature_names=data.feature_names, + plot_type="bar", show=False) plt.title("SHAP Feature Importance") buf = io.BytesIO() @@ -90,12 +89,15 @@ def load_data() -> Tuple[ iris = load_iris(as_frame=True) X = iris.data y = iris.target - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) + X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, + random_state=42) - for name, data in [("X_train", X_train), ("X_test", X_test), ("y_train", y_train), ("y_test", y_test)]: - log_artifact_metadata( + for name, data in [("X_train", X_train), ("X_test", X_test), + ("y_train", y_train), ("y_test", y_test)]: + log_metadata( artifact_name=name, - metadata={"dataset_info": safe_metadata(data)} + metadata={"dataset_info": safe_metadata(data)}, + infer_artifact=True, ) return X_train, X_test, y_train, y_test @@ -105,13 +107,13 @@ def load_data() -> Tuple[ def train_model( X_train: pd.DataFrame, y_train: pd.Series, -) -> Annotated[SVC, ArtifactConfig(name="model", is_model_artifact=True)]: +) -> Annotated[SVC, ArtifactConfig(name="model", artifact_type=ArtifactType.MODEL)]: """Train an SVM classifier.""" model = SVC(kernel='rbf', probability=True) model.fit(X_train, y_train) train_accuracy = model.score(X_train, y_train) - log_model_metadata( + log_metadata( metadata={ "training_metrics": { "train_accuracy": float(train_accuracy), @@ -120,18 +122,20 @@ def train_model( "model_type": type(model).__name__, "kernel": model.kernel, } - } + }, + infer_model=True, ) - log_artifact_metadata( - artifact_name="model", + log_metadata( metadata={ "model_details": { "type": type(model).__name__, "kernel": model.kernel, "n_support": model.n_support_.tolist(), } - } + }, + artifact_name="model", + infer_artifact=True, ) return model @@ -151,33 +155,36 @@ def evaluate_model( predictions = model.predict(X_test) probabilities = model.predict_proba(X_test) - log_model_metadata( + log_metadata( metadata={ "evaluation_metrics": { "test_accuracy": float(test_accuracy), } - } + }, + infer_model=True, ) - log_artifact_metadata( - artifact_name="predictions", + log_metadata( metadata={ "prediction_info": { "shape": predictions.shape, "unique_values": np.unique(predictions).tolist() } - } + }, + artifact_name="predictions", + infer_artifact=True, ) - log_artifact_metadata( - artifact_name="probabilities", + log_metadata( metadata={ "probability_info": { "shape": probabilities.shape, "min": float(np.min(probabilities)), "max": float(np.max(probabilities)) } - } + }, + artifact_name="probabilities", + infer_artifact=True, ) return predictions, probabilities @@ -189,18 +196,20 @@ def explain_model( X_train: pd.DataFrame ) -> Annotated[SHAPVisualization, "shap_visualization"]: """Generate SHAP values for model explainability and create a visualization.""" - explainer = shap.KernelExplainer(model.predict_proba, shap.sample(X_train, 100)) + explainer = shap.KernelExplainer(model.predict_proba, + shap.sample(X_train, 100)) shap_values = explainer.shap_values(X_train.iloc[:100]) - log_artifact_metadata( - artifact_name="shap_values", + log_metadata( metadata={ "shap_info": { "shape": [arr.shape for arr in shap_values], "n_classes": len(shap_values), "n_features": shap_values[0].shape[1], } - } + }, + artifact_name="shap_visualization", + infer_artifact=True, ) return SHAPVisualization(shap_values, X_train.columns) @@ -217,13 +226,15 @@ def detect_data_drift( _, p_value = ks_2samp(X_train[column], X_test[column]) drift_metrics[column] = p_value - log_artifact_metadata( - artifact_name="drift_metrics", + log_metadata( metadata={ "drift_summary": { - "high_drift_features": [col for col, p in drift_metrics.items() if p < 0.05] + "high_drift_features": [col for col, p in drift_metrics.items() + if p < 0.05] } - } + }, + artifact_name="drift_metrics", + infer_artifact=True, ) return drift_metrics diff --git a/flux-dreambooth/walkthrough.ipynb b/flux-dreambooth/walkthrough.ipynb index b2aefdf9..d881567b 100644 --- a/flux-dreambooth/walkthrough.ipynb +++ b/flux-dreambooth/walkthrough.ipynb @@ -37,7 +37,7 @@ "outputs": [], "source": [ "# Connect to ZenML server\n", - "!zenml connect --url " + "!zenml login " ] }, { diff --git a/label_studio_annotation/README.md b/label_studio_annotation/README.md index 5c819ff2..8395133a 100644 --- a/label_studio_annotation/README.md +++ b/label_studio_annotation/README.md @@ -22,7 +22,7 @@ In order to run this example, you need to install and initialize ZenML and Label Studio. ```shell -pip install "zenml[server]" torchvision +pip install "zenml[server]>=0.70.0" torchvision # clone the ZenML repository git clone https://github.com/zenml-io/zenml-projects.git @@ -32,7 +32,7 @@ cd label_studio_annotation zenml init # Start the ZenServer to enable dashboard access -zenml up +zenml login --local ``` You will need to install and start Label Studio locally: diff --git a/llm-finetuning-simple/README.md b/llm-finetuning-simple/README.md index c74540a7..980e02e4 100644 --- a/llm-finetuning-simple/README.md +++ b/llm-finetuning-simple/README.md @@ -10,12 +10,11 @@ In the fast-paced world of AI, the ability to efficiently fine-tune Large Langua 2. [Installation](#installation) 3. [Running the Pipeline](#running-the-pipeline) 4. [Configuration](#configuration) -5. [Accelerated Fine-Tuning](#accelerated-fine-tuning) -6. [Running with Remote Stack](#running-with-remote-stack) -7. [Customizing Data Preparation](#customizing-data-preparation) -8. [Project Structure](#project-structure) -9. [Benefits & Future](#benefits--future) -10. [Credits](#credits) +5. [Running with Remote Stack](#running-with-remote-stack) +6. [Customizing Data Preparation](#customizing-data-preparation) +7. [Project Structure](#project-structure) +8. [Benefits & Future](#benefits--future) +9. [Credits](#credits) ## Introduction @@ -39,13 +38,9 @@ source .venv/bin/activate # Install requirements pip install -r requirements.txt -# Install ZenML and Lightning integrations -pip install zenml -zenml integration install lightning s3 aws -y - # Initialize and connect to a deployed ZenML server zenml init -zenml connect --url +zenml login ``` ## Running the Pipeline @@ -120,7 +115,7 @@ steps: ## Running with Remote Stack -Set up a remote lightning stack with ZenML for fine tuning on remote infrastructure: +Set up a remote lightning stack with ZenML for fine-tuning on remote infrastructure: 1. **Register Orchestrator and Artifact Store:** diff --git a/llm-finetuning-simple/requirements.txt b/llm-finetuning-simple/requirements.txt index cf8f78fd..b5bbece7 100644 --- a/llm-finetuning-simple/requirements.txt +++ b/llm-finetuning-simple/requirements.txt @@ -8,7 +8,13 @@ rouge_score nltk accelerate>=0.30.0 urllib3<2 -zenml>=0.62.0 +zenml>=0.70.0 torch>=2.2.0 sentencepiece huggingface_hub +s3fs>2022.3.0 +boto3 +aws-profile-manager +sagemaker>=2.117.0 +kubernetes +lightning-sdk>=0.1.17 \ No newline at end of file diff --git a/llm-finetuning-simple/run.py b/llm-finetuning-simple/run.py index e957a231..3dcb20f0 100644 --- a/llm-finetuning-simple/run.py +++ b/llm-finetuning-simple/run.py @@ -1,27 +1,99 @@ +import argparse + import torch from datasets import load_dataset, Dataset -from transformers import AutoTokenizer, AutoModelForCausalLM, Trainer, TrainingArguments, DataCollatorForLanguageModeling -from zenml import pipeline, step, log_model_metadata +from transformers import ( + AutoTokenizer, + AutoModelForCausalLM, + DataCollatorForLanguageModeling, + Trainer, + TrainingArguments, +) from typing_extensions import Annotated -import argparse -from zenml.integrations.huggingface.materializers.huggingface_datasets_materializer import HFDatasetMaterializer +from zenml import pipeline, step, log_metadata +from zenml.integrations.huggingface.materializers.huggingface_datasets_materializer import ( + HFDatasetMaterializer +) + @step(output_materializers=HFDatasetMaterializer) -def prepare_data(base_model_id: str, dataset_name: str, dataset_size: int, max_length: int) -> Annotated[Dataset, "tokenized_dataset"]: +def prepare_data( + base_model_id: str, + dataset_name: str, + dataset_size: int, + max_length: int, +) -> Annotated[Dataset, "tokenized_dataset"]: + """ + Prepare and tokenize the dataset for fine-tuning. + + This step loads a specified dataset, tokenizes it with a given base model's + tokenizer, and prepares it for training by formatting the input as + question-answer prompts. + + Args: + base_model_id (str): Identifier of the base model to use for + tokenization. + dataset_name (str): Name of the dataset to load from Hugging Face + datasets. + dataset_size (int): Number of samples to use from the dataset. + max_length (int): Maximum sequence length for tokenization. + + Returns: + Annotated[Dataset, "tokenized_dataset"]: Tokenized dataset ready for + training. + """ tokenizer = AutoTokenizer.from_pretrained(base_model_id) tokenizer.pad_token = tokenizer.eos_token dataset = load_dataset(dataset_name, split=f"train[:{dataset_size}]") - + def tokenize_function(example): - prompt = f"Question: {example['question']}\nAnswer: {example['answers']['text'][0]}" - return tokenizer(prompt, truncation=True, padding="max_length", max_length=max_length) + """ + Tokenize a single example by formatting it as a question-answer prompt. + + Args: + example (dict): A single dataset example. - tokenized_data = dataset.map(tokenize_function, remove_columns=dataset.column_names) - log_model_metadata(metadata={"dataset_size": len(tokenized_data), "max_length": max_length}) + Returns: + dict: Tokenized input with input_ids, attention_mask, etc. + """ + prompt = f"Question: {example['question']}\n" \ + f"Answer: {example['answers']['text'][0]}" + return tokenizer(prompt, truncation=True, padding="max_length", + max_length=max_length) + + tokenized_data = dataset.map( + tokenize_function, + remove_columns=dataset.column_names + ) + log_metadata( + metadata={ + "dataset_size": len(tokenized_data), + "max_length": max_length + }, + infer_model=True, + ) return tokenized_data + @step -def finetune(base_model_id: str, tokenized_dataset: Dataset, num_train_epochs: int, per_device_train_batch_size: int) -> None: +def finetune( + base_model_id: str, + tokenized_dataset: Dataset, + num_train_epochs: int, + per_device_train_batch_size: int +) -> None: + """ + Fine-tune a pre-trained language model on the prepared dataset. + + This step loads the base model, sets up training arguments, and performs + fine-tuning using the Hugging Face Trainer. + + Args: + base_model_id (str): Identifier of the base model to fine-tune. + tokenized_dataset (Dataset): Tokenized dataset prepared for training. + num_train_epochs (int): Number of training epochs. + per_device_train_batch_size (int): Batch size per device during training. + """ torch.cuda.empty_cache() model = AutoModelForCausalLM.from_pretrained( base_model_id, @@ -49,20 +121,47 @@ def finetune(base_model_id: str, tokenized_dataset: Dataset, num_train_epochs: i model=model, args=training_args, train_dataset=tokenized_dataset, - data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False), + data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, + mlm=False), ) - + train_result = trainer.train() - log_model_metadata(metadata={"metrics": {"train_loss": train_result.metrics.get("train_loss")}}) + log_metadata( + metadata={ + "metrics": {"train_loss": train_result.metrics.get("train_loss")} + }, + infer_model=True, + ) trainer.save_model("finetuned_model") + @pipeline def llm_finetune_pipeline(base_model_id: str): + """ + ZenML pipeline for fine-tuning a language model. + + This pipeline orchestrates the data preparation and fine-tuning steps + for a language model on a specified dataset. + + Args: + base_model_id (str): Identifier of the base model to fine-tune. + """ tokenized_dataset = prepare_data(base_model_id) finetune(base_model_id, tokenized_dataset) + if __name__ == "__main__": + """ + Entry point for the script that allows configuration via command-line argument. + + Expects a YAML configuration file path to be provided. + """ parser = argparse.ArgumentParser() - parser.add_argument('--config', type=str, required=True, help='Path to the YAML config file') + parser.add_argument( + '--config', + type=str, + required=True, + help='Path to the YAML config file' + ) args = parser.parse_args() llm_finetune_pipeline.with_options(config_path=args.config)() \ No newline at end of file diff --git a/native-experiment-tracking/README.md b/native-experiment-tracking/README.md index 62b3c124..6c73b259 100644 --- a/native-experiment-tracking/README.md +++ b/native-experiment-tracking/README.md @@ -41,7 +41,7 @@ zenml integration install sklearn pandas -y zenml init # Connect to your ZenML server -zenml connect --url ... +zenml login ... python run.py --parallel ``` diff --git a/native-experiment-tracking/requirements.txt b/native-experiment-tracking/requirements.txt index 2f13da32..d66ee10a 100644 --- a/native-experiment-tracking/requirements.txt +++ b/native-experiment-tracking/requirements.txt @@ -1,4 +1,4 @@ -zenml[server]>=0.67.0 +zenml[server]>=0.70.0 notebook scikit-learn pyarrow