chore: Add instructions to develop the API server locally with kind (#…

…11459) * Add instructions to develop the API server locally with kind This provides instructions to provision a kind cluster with all components deployed except for the API server. The API server can then be run locally through VSCode for debugging and faster development. Signed-off-by: mprahl <[email protected]> * Add missing k8s_version input to the kfp-cluster GH action Signed-off-by: mprahl <[email protected]> --------- Signed-off-by: mprahl <[email protected]>
kubeflow · Dec 12, 2024 · e71825a · e71825a
1 parent 0eb67e1
commit e71825a
Show file tree

Hide file tree

Showing 8 changed files with 278 additions and 0 deletions.
diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml
@@ -17,6 +17,10 @@ on:
 jobs:
   samples:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s_version: [ "v1.29.2", "v1.30.2", "v1.31.0" ]
+    name: KFP Samples - K8s ${{ matrix.k8s_version }}
 
     steps:
     - name: Checkout code
@@ -29,6 +33,8 @@ jobs:
 
     - name: Create KFP cluster
       uses: ./.github/actions/kfp-cluster
+      with:
+        k8s_version: ${{ matrix.k8s_version }}
 
     - name: Forward API port
       run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888

diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml
@@ -14,6 +14,11 @@ on:
 jobs:
   upgrade-test:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        k8s_version: [ "v1.29.2", "v1.30.2", "v1.31.0" ]
+    name: KFP upgrade tests - K8s ${{ matrix.k8s_version }}
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -25,6 +30,8 @@ jobs:
 
       - name: Create KFP cluster
         uses: ./.github/actions/kfp-cluster
+        with:
+          k8s_version: ${{ matrix.k8s_version }}
 
       - name: Prepare upgrade tests
         working-directory: backend/test/integration

diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,7 @@ bower_components/
 
 # Build output
 dist
+__debug_bin*
 
 # Web server
 frontend/server/dist

diff --git a/backend/Makefile b/backend/Makefile
@@ -1,6 +1,7 @@
 BUILD=build
 MOD_ROOT=..
 CSV_PATH=backend/third_party_licenses
+KIND_NAME ?= dev-pipelines-api
 
 # Container Build Params
 CONTAINER_ENGINE ?= docker
@@ -88,3 +89,14 @@ image_driver:
 .PHONY: image_launcher
 image_launcher:
 	cd $(MOD_ROOT) && ${CONTAINER_ENGINE} build -t ${IMG_TAG_LAUNCHER} -f backend/Dockerfile.launcher .
+
+.PHONY: dev-kind-cluster
+dev-kind-cluster:
+	-kind create cluster --name $(KIND_NAME) --config $(CURDIR)/../tools/kind/kind-config.yaml
+	kubectl config use-context kind-$(KIND_NAME)
+	kind get kubeconfig --name $(KIND_NAME) > $(CURDIR)/../kubeconfig_$(KIND_NAME)
+	kubectl apply -k $(CURDIR)/../manifests/kustomize/cluster-scoped-resources
+	kubectl wait --for condition=established --timeout=1m crd/applications.app.k8s.io
+	kubectl apply -k $(CURDIR)/../manifests/kustomize/env/dev-kind
+	kubectl -n kubeflow wait --for condition=Available --timeout=10m deployment/mysql
+	kubectl -n kubeflow wait --for condition=Available --timeout=3m deployment/metadata-grpc-deployment
diff --git a/backend/README.md b/backend/README.md
@@ -78,3 +78,92 @@ Run
 ```
 docker build . -f backend/Dockerfile.conformance -t <tag>
 ```
+
+## API Server Development
+
+### Run Locally With a Kind Cluster
+
+This deploys a local Kubernetes cluster leveraging [kind](https://kind.sigs.k8s.io/), with all the components required
+to run the Kubeflow Pipelines API server. Note that the `ml-pipeline` `Deployment` (API server) has its replicas set to
+0 so that the API server can be run locally for debugging and faster development. The local API server is available by
+pods on the cluster using the `ml-pipeline` `Service`.
+
+#### Prerequisites
+
+* The [kind CLI](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) is installed.
+* The following ports are available on your localhost: 3000, 3306, 8080, 9000, and 8889. If these are unavailable,
+  modify [kind-config.yaml](../tools/kind/kind-config.yaml) and configure the API server with alternative ports when
+  running locally.
+* If using a Mac, you will need to modify the
+  [Endpoints](../manifests/kustomize/env/dev-kind/forward-local-api-endpoint.yaml) manifest to leverage the bridge
+  network interface through Docker/Podman Desktop. See
+  [kind #1200](https://github.com/kubernetes-sigs/kind/issues/1200#issuecomment-1304855791) for an example manifest.
+* Optional: VSCode is installed to leverage a sample `launch.json` file.
+
+#### Provisioning the Cluster
+
+To provision the kind cluster, run the following from the Git repository's root directory,:
+
+```bash
+make -C backend dev-kind-cluster
+```
+
+This may take several minutes since there are many pods. Note that many pods will be in "CrashLoopBackOff" status until
+all the pods have started.
+
+#### Deleting the Cluster
+
+Run the following to delete the cluster:
+
+```bash
+kind delete clusters dev-pipelines-api
+```
+
+#### Launch the API Server With VSCode
+
+After the cluster is provisioned, you may leverage the following sample `.vscode/launch.json` file to run the API
+server locally:
+
+```json
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Launch API Server (Kind)",
+      "type": "go",
+      "request": "launch",
+      "mode": "debug",
+      "program": "${workspaceFolder}/backend/src/apiserver",
+      "env": {
+        "POD_NAMESPACE": "kubeflow",
+        "DBCONFIG_MYSQLCONFIG_HOST": "localhost",
+        "MINIO_SERVICE_SERVICE_HOST": "localhost",
+        "MINIO_SERVICE_SERVICE_PORT": "9000",
+        "METADATA_GRPC_SERVICE_SERVICE_HOST": "localhost",
+        "METADATA_GRPC_SERVICE_SERVICE_PORT": "8080",
+        "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST": "localhost",
+        "ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT": "8888"
+      },
+      "args": [
+        "--config",
+        "${workspaceFolder}/backend/src/apiserver/config",
+        "-logtostderr=true"
+      ]
+    }
+  ]
+}
+```
+
+#### Using the Environment
+
+Once the cluster is provisioned and the API server is running, you can access the API server at
+[http://localhost:8888](http://localhost:8888)
+(e.g. [http://localhost:8888/apis/v2beta1/pipelines](http://localhost:8888/apis/v2beta1/pipelines)).
+
+You can also access the Kubeflow Pipelines web interface at [http://localhost:3000](http://localhost:3000).
+
+You can also directly connect to the MariaDB database server with:
+
+```bash
+mysql -h 127.0.0.1 -u root
+```
diff --git a/manifests/kustomize/env/dev-kind/forward-local-api-endpoint.yaml b/manifests/kustomize/env/dev-kind/forward-local-api-endpoint.yaml
@@ -0,0 +1,17 @@
+---
+apiVersion: v1
+kind: Endpoints
+metadata:
+  name: ml-pipeline
+subsets:
+  - addresses:
+      - ip: 172.17.0.1 # docker0 bridge ip
+    ports:
+      - name: http
+        appProtocol: http
+        port: 8887
+        protocol: TCP
+      - name: grpc
+        appProtocol: http
+        port: 8888
+        protocol: TCP
diff --git a/manifests/kustomize/env/dev-kind/kustomization.yaml b/manifests/kustomize/env/dev-kind/kustomization.yaml
@@ -0,0 +1,130 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - ../../base/application
+  - ./forward-local-api-endpoint.yaml
+  - ../platform-agnostic
+
+# !!! If you want to customize the namespace,
+# please refer sample/cluster-scoped-resources to update the namespace for cluster-scoped-resources
+namespace: kubeflow
+
+patches:
+- patch: |-
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ml-pipeline-ui
+      namespace: kubeflow
+    spec:
+      ports:
+        - $patch: replace
+        - name: http
+          port: 80
+          protocol: TCP
+          targetPort: 3000
+          nodePort: 30300
+      type: NodePort
+- patch: |-
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: mysql
+      namespace: kubeflow
+    spec:
+      ports:
+        - $patch: replace
+        - port: 3306
+          protocol: TCP
+          targetPort: 3306
+          nodePort: 30330
+      type: NodePort
+- patch: |-
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: metadata-grpc-service
+      namespace: kubeflow
+    spec:
+      ports:
+        - $patch: replace
+        - port: 8080
+          protocol: TCP
+          targetPort: 8080
+          nodePort: 30808
+      type: NodePort
+- patch: |-
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: minio-service
+      namespace: kubeflow
+    spec:
+      ports:
+        - $patch: replace
+        - port: 9000
+          protocol: TCP
+          targetPort: 9000
+          nodePort: 30900
+      type: NodePort
+- patch: |-
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ml-pipeline-visualizationserver
+      namespace: kubeflow
+    spec:
+      ports:
+        - $patch: replace
+        - port: 8888
+          protocol: TCP
+          targetPort: 8888
+          nodePort: 30889
+      type: NodePort
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: ml-pipeline
+      namespace: kubeflow
+    spec:
+      replicas: 0
+- patch: |-
+    apiVersion: apps/v1
+    kind: Deployment
+    metadata:
+      name: ml-pipeline-ui
+      namespace: kubeflow
+    spec:
+      template:
+        spec:
+          containers:
+            - name: ml-pipeline-ui
+              env:
+                - name: ML_PIPELINE_SERVICE_HOST
+                  value: ml-pipeline
+                - name: ML_PIPELINE_SERVICE_PORT
+                  value: "8888"
+                - name: DISABLE_GKE_METADATA
+                  value: "true"
+- patch: |-
+    apiVersion: v1
+    kind: Service
+    metadata:
+      name: ml-pipeline
+      namespace: kubeflow
+    spec:
+      clusterIP: None
+      ports:
+      - $patch: replace
+      - name: http
+        port: 8888
+        protocol: TCP
+        targetPort: 8888
+      - name: grpc
+        port: 8887
+        protocol: TCP
+        targetPort: 8887
+      selector:
+        $patch: delete
diff --git a/tools/kind/kind-config.yaml b/tools/kind/kind-config.yaml
@@ -0,0 +1,16 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  # Expose the UI, MySQL, MLMD, Minio, and the visualization server
+  extraPortMappings:
+  - containerPort: 30300
+    hostPort: 3000
+  - containerPort: 30330
+    hostPort: 3306
+  - containerPort: 30808
+    hostPort: 8080
+  - containerPort: 30900
+    hostPort: 9000
+  - containerPort: 30889
+    hostPort: 8889