Add Helm hook to upgrade CRDs

Signed-off-by: Yi Chen <[email protected]>

.dockerignore

@@ -2,7 +2,6 @@
 .idea/
 .vscode/
 bin/
-charts/
 docs/
 config/
 examples/

Dockerfile

@@ -55,6 +55,8 @@ USER ${SPARK_UID}:${SPARK_GID}
 
 COPY --from=builder /workspace/bin/spark-operator /usr/bin/spark-operator
 
+COPY --from=builder /workspace/charts/spark-operator-chart/crds /etc/spark-operator/crds
+
 COPY entrypoint.sh /usr/bin/
 
 ENTRYPOINT ["/usr/bin/entrypoint.sh"]

charts/spark-operator-chart/README.md

@@ -83,6 +83,9 @@ See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall) for command docum
 | image.tag | string | If not set, the chart appVersion will be used. | Image tag. |
 | image.pullPolicy | string | `"IfNotPresent"` | Image pull policy. |
 | image.pullSecrets | list | `[]` | Image pull secrets for private image registry. |
+| hook.upgradeCrd | bool | `true` | Specifies whether to update CRDs with a Helm hook job. |
+| hook.resources | object | `{"limits":{"cpu":"100m","memory":"64Mi"},"requests":{"cpu":"100m","memory":"64Mi"}}` | Resource requests and limits for hook containers. |
+| hook.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"privileged":false,"readOnlyRootFilesystem":true,"runAsNonRoot":true}` | Security context for hook containers. |
 | controller.replicas | int | `1` | Number of replicas of controller. |
 | controller.workers | int | `10` | Reconcile concurrency, higher values might increase memory usage. |
 | controller.logLevel | string | `"info"` | Configure the verbosity of logging, can be one of `debug`, `info`, `error`. |

charts/spark-operator-chart/templates/hook/_helpers.tpl

@@ -0,0 +1,65 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{/*
+Create the name of Helm hook
+*/}}
+{{- define "spark-operator.hook.name" -}}
+{{- include "spark-operator.fullname" . }}-hook
+{{- end -}}
+
+{{/*
+Common labels for the Helm hook
+*/}}
+{{- define "spark-operator.hook.labels" -}}
+{{ include "spark-operator.labels" . }}
+app.kubernetes.io/component: hook
+{{- end -}}
+
+{{/*
+Selector labels for the Helm hook
+*/}}
+{{- define "spark-operator.hook.selectorLabels" -}}
+{{ include "spark-operator.hook.labels" . }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to be used by the Helm hooks.
+*/}}
+{{- define "spark-operator.hook.serviceAccountName" -}}
+{{ include "spark-operator.hook.name" . }}
+{{- end -}}
+
+{{/*
+Create the name of the cluster role to be used by the Helm hooks.
+*/}}
+{{- define "spark-operator.hook.clusterRoleName" -}}
+{{ include "spark-operator.hook.name" . }}
+{{- end }}
+
+{{/*
+Create the name of the cluster role binding to be used by the Helm hooks.
+*/}}
+{{- define "spark-operator.hook.clusterRoleBindingName" -}}
+{{ include "spark-operator.hook.clusterRoleName" . }}
+{{- end }}
+
+{{/*
+Create the name of the Helm hook job.
+*/}}
+{{- define "spark-operator.hook.jobName" -}}
+{{ include "spark-operator.hook.name" . }}
+{{- end }}

charts/spark-operator-chart/templates/hook/job.yaml

@@ -0,0 +1,58 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.hook.upgradeCrd }}
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: {{ include "spark-operator.hook.jobName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded
+    helm.sh/hook-weight: "3"
+spec:
+  template:
+    spec:
+      containers:
+      - name: spark-operator-hook
+        image: {{ include "spark-operator.image" . }}
+        {{- with .Values.image.pullPolicy }}
+        imagePullPolicy: {{ . }}
+        {{- end }}
+        args:
+        - hook
+        - start
+        - --upgrade-crds
+        - --crds-path
+        - /etc/spark-operator/crds
+        {{- with .Values.hook.resources }}
+        resources:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+        {{- with .Values.hook.securityContext }}
+        securityContext:
+          {{- toYaml . | nindent 10 }}
+        {{- end }}
+      {{- with .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "spark-operator.hook.serviceAccountName" . }}
+      restartPolicy: Never
+{{- end }}

charts/spark-operator-chart/templates/hook/rbac.yaml

@@ -0,0 +1,61 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.hook.upgradeCrd }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "spark-operator.hook.clusterRoleName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "2"
+rules:
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  resourceNames:
+  - sparkapplications.sparkoperator.k8s.io
+  - scheduledsparkapplications.sparkoperator.k8s.io
+  verbs:
+  - get
+  - update
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "spark-operator.hook.clusterRoleBindingName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "2"
+subjects:
+- kind: ServiceAccount
+  name: {{ include "spark-operator.hook.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "spark-operator.hook.clusterRoleName" . }}
+{{- end }}

charts/spark-operator-chart/templates/hook/serviceaccount.yaml

@@ -0,0 +1,29 @@
+{{/*
+Copyright 2024 The Kubeflow authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/}}
+
+{{- if .Values.hook.upgradeCrd }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "spark-operator.hook.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "spark-operator.hook.labels" . | nindent 4 }}
+  annotations:
+    helm.sh/hook: pre-install,pre-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed
+    helm.sh/hook-weight: "1"
+{{- end }}

charts/spark-operator-chart/tests/hook/job_test.yaml

@@ -0,0 +1,110 @@
+#
+# Copyright 2024 The Kubeflow authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+suite: Test hook job
+
+templates:
+  - hook/job.yaml
+
+release:
+  name: spark-operator
+  namespace: spark-operator
+
+tests:
+  - it: Should not create hook job if `hook.upgradeCrd` is false
+    set:
+      hook:
+        upgradeCrd: false
+    asserts:
+      - hasDocuments:
+          count: 0
+
+  - it: Should create hook job by default
+    asserts:
+      - containsDocument:
+          apiVersion: batch/v1
+          kind: Job
+          name: spark-operator-hook
+
+  - it: Should use the specified image repository if `image.registry`, `image.repository` and `image.tag` are set
+    set:
+      image:
+        registry: test-registry
+        repository: test-repository
+        tag: test-tag
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[?(@.name=="spark-operator-hook")].image
+          value: test-registry/test-repository:test-tag
+
+  - it: Should use the specified image pull policy if `image.pullPolicy` is set
+    set:
+      image:
+        pullPolicy: Always
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[*].imagePullPolicy
+          value: Always
+
+  - it: Should add resources if `hook.resources` is set
+    set:
+      hook:
+        resources:
+          requests:
+            memory: 64Mi
+            cpu: 250m
+          limits:
+            memory: 128Mi
+            cpu: 500m
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[?(@.name=="spark-operator-hook")].resources
+          value:
+            requests:
+              memory: 64Mi
+              cpu: 250m
+            limits:
+              memory: 128Mi
+              cpu: 500m
+
+  - it: Should add container securityContext if `hook.securityContext` is set
+    set:
+      hook:
+        securityContext:
+          readOnlyRootFilesystem: true
+          runAsUser: 1000
+          runAsGroup: 2000
+          fsGroup: 3000
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+              - ALL
+          runAsNonRoot: true
+          privileged: false
+    asserts:
+      - equal:
+          path: spec.template.spec.containers[?(@.name=="spark-operator-hook")].securityContext
+          value:
+            readOnlyRootFilesystem: true
+            runAsUser: 1000
+            runAsGroup: 2000
+            fsGroup: 3000
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+                - ALL
+            runAsNonRoot: true
+            privileged: false