SwissDataScienceCenter · olevski · Nov 19, 2024 · Jun 21, 2024 · Jul 15, 2024 · Jul 15, 2024
@@ -58,6 +58,7 @@ jobs:
       renku-ui: ${{ steps.deploy-comment.outputs.renku-ui}}
       renku-data-services: ${{ steps.deploy-comment.outputs.renku-data-services}}
       amalthea: ${{ steps.deploy-comment.outputs.amalthea}}
+      amalthea-sessions: ${{ steps.deploy-comment.outputs.amalthea-sessions}}
       test-enabled: ${{ steps.deploy-comment.outputs.test-enabled}}
       extra-values: ${{ steps.deploy-comment.outputs.extra-values}}
     steps:
@@ -99,6 +100,7 @@ jobs:
           renku_ui: "${{ needs.check-deploy.outputs.renku-ui }}"
           renku_data_services: "${{ needs.check-deploy.outputs.renku-data-services }}"
           amalthea: "${{ needs.check-deploy.outputs.amalthea }}"
+          amalthea_sessions: "${{ needs.check-deploy.outputs.amalthea-sessions }}"
           extra_values: "${{ needs.check-deploy.outputs.extra-values }}"
       - name: Check existing renkubot comment
         if: needs.check-deploy.outputs.pr-contains-string == 'true'

@@ -3,7 +3,56 @@
 0.61.0
 ------
 
+Renku 0.61.0 introduces a new version of Amalthea that supports running sessions with Docker images
+that do not contain Jupyter server.
+
+NOTE to administrators: This upgrade introduces a brand new CRD for sessions. All services that support
+sessions for Renku v2 will switch to this new CRD. Renku v1 sessions remain unchanged.
+Therefore any old sessions for Renku v2 will not be visible to users after this upgrade. The sessions
+themselves will not be immediately deleted and as long as users have saved links to their old sessions they
+should be able to access their sessions and save data. However we recommend that administrators
+notify users of the change and allow for enough time so that existing Renku v2 sessions can be saved and
+cleaned up, rather than asking users to save the url to their sessions. In addition to users not being able
+to see old Renku v2 sessions, they will also not be able to pause, resume or delete old Renku v2 sessions.
+Therefore it best is most sessions are properly saved and cleaned up before this update is rolled out. In order 
+to support the new CRD we have also created a new operator that will manage the new `amaltheasession` resources.
 
+User-Facing Changes
+~~~~~~~~~~~~~~~~~~~
+
+**✨ Improvements**
+
+- **UI**: Enable the use of custom images that don’t contain Jupyter, streamlining the image-building process and allowing for the use of “off-the-shelf” images (`#3341 <https://github.com/SwissDataScienceCenter/renku-ui/pull/3341>`__).
+- **Sessions**: Enable running session images that do not container Jupyter in them.
+- **Amalthea**: Brand new operator for sessions with images that do not have to container Jupyter
+
+Internal Changes
+~~~~~~~~~~~~~~~~
+
+**Improvements**
+
+- **Data services**: Add support for OAuth storage providers
+- **Data services**: Move notebooks code to data services
+
+**Bug Fixes**
+
+- **Gateway**: Pass on session cookie to data services for anonymous session authentication
+- **Data services**: Correct pagination for namespaces
+- **Data services**: Add creation data and created_by for namespaces
+- **Data services**: Pin RClone version in data services image
+- **Data services**: Properly handle multi-architecture docker images when getting working directory
+- **Amalthea**: Add readiness and health checks to sessions.
+
+Individual Components
+~~~~~~~~~~~~~~~~~~~~~
+
+- `renku-gateway 1.3.1 <https://github.com/SwissDataScienceCenter/renku-gateway/releases/tag/1.3.1>`_
+- `renku-ui 3.42.0 <https://github.com/SwissDataScienceCenter/renku-ui/releases/tag/3.42.0>`_
+- `renku-data-services 0.26.0 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.26.0>`_
+- `amalthea 0.13.0 <https://github.com/SwissDataScienceCenter/amalthea/releases/tag/0.13.0>`_
+- `amalthea 0.14.0 <https://github.com/SwissDataScienceCenter/amalthea/releases/tag/0.14.0>`_
+- `amalthea 0.14.1 <https://github.com/SwissDataScienceCenter/amalthea/releases/tag/0.14.1>`_
+- `amalthea 0.14.2 <https://github.com/SwissDataScienceCenter/amalthea/releases/tag/0.14.2>`_
 
 0.60.0
 ------
@@ -79,11 +128,16 @@ Internal Changes
 **Bug Fixes**
 
 - **Data services**: Handle spaces in ``provider_id`` for connected services (`#482 <https://github.com/SwissDataScienceCenter/renku-data-services/pull/482>`__).
+- **csi-rclone**: Do not log potentially sensitive data in error messages.
+- **csi-rclone**: Properly handle encrypted secrets with the new annotation-based storage class.
+
 
 Individual Components
 ~~~~~~~~~~~~~~~~~~~~~
 
 - `renku-data-services 0.24.2 <https://github.com/SwissDataScienceCenter/renku-data-services/releases/tag/v0.24.2>`__
+- `csi-rclone 0.3.4 <https://github.com/SwissDataScienceCenter/csi-rclone/releases/tag/v0.3.4>`__
+- `csi-rclone 0.3.5 <https://github.com/SwissDataScienceCenter/csi-rclone/releases/tag/v0.3.5>`__
 
 0.59.1
 ------

@@ -189,6 +189,8 @@ describe("Basic public project functionality", () => {
 
     // Stop the session -- mind that anonymous users cannot pause sessions
     cy.deleteSession({ fromSessionPage: true });
+
+    cy.robustLogin();
   });
 
   it("Start a new session on a project without permissions.", () => {

@@ -23,7 +23,7 @@ export function getUserNamespaceAPIV2(): Cypress.Chainable<string | null> {
 
 /** Get a project by using only the API. */
 export function getProjectByNamespaceAPIV2(newProjectProps: ProjectIdentifierV2): Cypress.Chainable<any | null> {
-  return cy.request({ failOnStatusCode: false, method: "GET", url: `api/data/projects/${newProjectProps.namespace}/${newProjectProps.slug}` });
+  return cy.request({ failOnStatusCode: false, method: "GET", url: `api/data/namespaces/${newProjectProps.namespace}/projects/${newProjectProps.slug}` });
 }
 
 /** Create a project (if the project is missing) by using only the API. */

@@ -23,7 +23,10 @@ dependencies:
     alias: jena
   - name: amalthea
     repository: "https://swissdatasciencecenter.github.io/helm-charts/"
-    version: "0.12.3"
+    version: "0.14.2"
+  - name: amalthea-sessions
+    repository: "https://swissdatasciencecenter.github.io/helm-charts/"
+    version: "0.14.2"
   - name: dlf-chart
     repository: "https://swissdatasciencecenter.github.io/datashim/"
     version: "0.3.9-renku-2"

@@ -149,6 +149,10 @@ KC_DB_PASSWORD: {{ default (randAlphaNum 64) .Values.global.keycloak.postgresPas
 {{- end -}}
 {{- end -}}
 
+{{- define "renku.keycloakIssuerUrl" -}}
+{{- printf "%s/realms/%s" (include "renku.keycloakUrl" . | trimSuffix "/") (include "renku.keycloak.realm" .) -}}
+{{- end -}}
+
 {{/*
 Common labels
 */}}

@@ -43,6 +43,9 @@ spec:
             - name: http
               containerPort: 8000
               protocol: TCP
+          envFrom:
+            - secretRef:
+                name: {{ template "renku.notebooks.fullname" . }}
           env:
             - name: VERSION
               value: {{ .Values.dataService.image.tag | quote }}
@@ -70,10 +73,6 @@ spec:
               value: /secrets/encryptionKey/encryptionKey
             - name: SECRETS_SERVICE_PUBLIC_KEY_PATH
               value: /secrets/publicKey/publicKey
-            - name: SERVER_DEFAULTS
-              value: /etc/renku-data-service/server_options/server_defaults.json
-            - name: SERVER_OPTIONS
-              value: /etc/renku-data-service/server_options/server_options.json
             - name: K8S_NAMESPACE
               value: {{ .Release.Namespace | quote }}
             - name: GITLAB_URL
@@ -116,11 +115,21 @@ spec:
               value: {{ .Values.dataService.trustedProxies.proxiesCount | default "" | quote }}
             - name: REAL_IP_HEADER
               value: {{ .Values.dataService.trustedProxies.realIpHeader | default "" | quote }}
+            - name: KUBERNETES_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
+            - name: SESSIONS_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: metadata.namespace
             {{- include "certificates.env.python" . | nindent 12 }}
             {{- include "certificates.env.grpc" . | nindent 12 }}
           volumeMounts:
             - name: server-options
-              mountPath: /etc/renku-data-service/server_options
+              mountPath: /etc/renku-notebooks/server_options
             - mountPath: "/secrets/encryptionKey"
               name: encryption-key
               readOnly: true

@@ -19,6 +19,67 @@ rules:
       - patch
       - delete
       - create
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/log
+      - services
+      - endpoints
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - secrets
+    verbs:
+      - delete
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - create
+      - update
+      - delete
+      - patch
+  - apiGroups:
+      - {{ .Values.amalthea.crdApiGroup }}
+    resources:
+      - {{ .Values.amalthea.crdNames.plural }}
+    verbs:
+      - create
+      - update
+      - delete
+      - patch
+      - list
+      - get
+      - watch
+  - apiGroups:
+      - amalthea.dev
+    resources:
+      - amaltheasessions
+    verbs:
+      - create
+      - update
+      - delete
+      - patch
+      - list
+      - get
+      - watch
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

@@ -43,6 +43,12 @@ spec:
               value: {{ .Values.amalthea.crdApiVersion }}
             - name: K8S_WATCHER_CR_PLURAL
               value: {{ .Values.amalthea.crdNames.plural }}
+            - name: K8S_WATCHER_AMALTHEA_SESSION_GROUP
+              value: amalthea.dev
+            - name: K8S_WATCHER_AMALTHEA_SESSION_VERSION
+              value: v1alpha1
+            - name: K8S_WATCHER_AMALTHEA_SESSION_PLURAL
+              value: amaltheasessions
             - name: K8S_WATCHER_NAMESPACES
               {{ if .Values.notebooks.sessionsNamespace }}
               value: {{ list .Release.Namespace .Values.notebooks.sessionsNamespace | uniq | toJson | quote }}

@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "renku.notebooks.fullname" . }}
+  labels:
+    chart: {{ template "renku.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+type: Opaque
+stringData:
+  NB_SESSIONS__STORAGE__PVS_ENABLED: {{ .Values.notebooks.userSessionPersistentVolumes.enabled | quote }}
+  {{ if .Values.notebooks.userSessionPersistentVolumes.enabled }}
+  NB_SESSIONS__STORAGE__PVS_STORAGE_CLASS: {{ .Values.notebooks.userSessionPersistentVolumes.storageClass | quote}}
+  {{ end }}
+  NB_SESSIONS__STORAGE__USE_EMPTY_DIR_SIZE_LIMIT: {{ .Values.notebooks.userSessionPersistentVolumes.useEmptyDirSizeLimit | quote }}
+  NB_SESSIONS__DEFAULT_IMAGE: "{{ .Values.notebooks.defaultSessionImage }}"
+  NB_SERVER_OPTIONS__DEFAULTS_PATH: /etc/renku-notebooks/server_options/server_defaults.json
+  NB_SERVER_OPTIONS__UI_CHOICES_PATH: /etc/renku-notebooks/server_options/server_options.json
+  NB_SESSIONS__OIDC__CLIENT_ID: {{ .Values.notebooks.oidc.clientId }}
+  NB_SESSIONS__OIDC__CLIENT_SECRET: {{ .Values.notebooks.oidc.clientSecret }}
+  NB_SESSIONS__OIDC__AUTH_URL: {{ .Values.notebooks.oidc.authUrl }}
+  NB_SESSIONS__OIDC__TOKEN_URL: {{ .Values.notebooks.oidc.tokenUrl }}
+  NB_SESSIONS__OIDC__ISSUER_URL: {{ template "renku.keycloakIssuerUrl" . }}
+  NB_SESSIONS__OIDC__ALLOW_UNVERIFIED_EMAIL: {{ .Values.notebooks.oidc.allowUnverifiedEmail | quote }}
+  NB_SESSIONS__INGRESS__HOST: {{ .Values.notebooks.sessionIngress.host }}
+  NB_SESSIONS__INGRESS__TLS_SECRET: {{ .Values.notebooks.sessionIngress.tlsSecret }}
+  NB_SESSIONS__INGRESS__ANNOTATIONS: |
+    {{- .Values.notebooks.sessionIngress.annotations | toYaml | nindent 4 }}
+  NB_GIT__URL: {{ .Values.global.gitlab.url | quote }}
+  NB_GIT__REGISTRY: {{ required "An image registry must be specified." .Values.global.gitlab.registry.host }}
+  NB_SESSIONS__GIT_RPC_SERVER__IMAGE: "{{ .Values.notebooks.gitRpcServer.image.name }}:{{ .Values.notebooks.gitRpcServer.image.tag }}"
+  NB_SESSIONS__GIT_PROXY__IMAGE: "{{ .Values.notebooks.gitHttpsProxy.image.name }}:{{ .Values.notebooks.gitHttpsProxy.image.tag }}"
+  NB_SESSIONS__GIT_CLONE__IMAGE: "{{ .Values.notebooks.gitClone.image.name }}:{{ .Values.notebooks.gitClone.image.tag }}"
+  NB_ANONYMOUS_SESSIONS_ENABLED: {{ .Values.global.anonymousSessions.enabled | quote }}
+  NB_SSH_ENABLED: {{ .Values.notebooks.ssh.enabled | quote }}
+  NB_SESSIONS__CULLING__REGISTERED__IDLE_SECONDS: {{ .Values.notebooks.culling.idleSecondsThreshold.registered | quote }}
+  NB_SESSIONS__CULLING__ANONYMOUS__IDLE_SECONDS: {{ .Values.notebooks.culling.idleSecondsThreshold.anonymous | quote }}
+  NB_SESSIONS__CULLING__REGISTERED__HIBERNATED_SECONDS: {{ .Values.notebooks.culling.hibernatedSecondsThreshold.registered | quote }}
+  NB_SESSIONS__CULLING__REGISTERED__MAX_AGE_SECONDS: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.registered | quote }}
+  NB_SESSIONS__CULLING__ANONYMOUS__MAX_AGE_SECONDS: {{ .Values.notebooks.culling.maxAgeSecondsThreshold.anonymous | quote }}
+  NB_AMALTHEA__GROUP: {{ .Values.amalthea.crdApiGroup }}
+  NB_AMALTHEA__VERSION: {{ .Values.amalthea.crdApiVersion }}
+  NB_AMALTHEA__PLURAL: {{ .Values.amalthea.crdNames.plural }}
+  NB_AMALTHEA__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher"
+  NB_AMALTHEA_V2__CACHE_URL: "http://{{ template "renku.notebooks.fullname" . }}-k8s-watcher"
+  NB_SESSIONS__GIT_CLONE__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.gitClone.enabled | quote }}
+  NB_SESSIONS__GIT_CLONE__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.gitClone.dsn | quote }}
+  NB_SESSIONS__GIT_CLONE__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.gitClone.environment | quote }}
+  NB_SESSIONS__GIT_CLONE__SENTRY__SAMPLE_RATE: {{ .Values.notebooks.sessionSentry.gitClone.sampleRate | quote }}
+  NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENABLED: {{ .Values.notebooks.sessionSentry.sidecar.enabled | quote }}
+  NB_SESSIONS__GIT_RPC_SERVER__SENTRY__DSN: {{ .Values.notebooks.sessionSentry.sidecar.dsn | quote }}
+  NB_SESSIONS__GIT_RPC_SERVER__SENTRY__ENV: {{ .Values.notebooks.sessionSentry.sidecar.environment | quote }}
+  NB_SESSIONS__GIT_RPC_SERVER__SENTRY__SAMPLE_RATE: {{ .Values.notebooks.sessionSentry.sidecar.sampleRate | quote }}
+  NB_SESSIONS__CA_CERTS__IMAGE: "{{ .Values.global.certificates.image.repository }}:{{ .Values.global.certificates.image.tag }}"
+  NB_SESSIONS__CA_CERTS__SECRETS: |
+    {{- .Values.global.certificates.customCAs | toYaml | nindent 4 }}
+  {{- with .Values.notebooks.sessionNodeSelector }}
+  NB_SESSIONS__NODE_SELECTOR: |
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.notebooks.sessionAffinity }}
+  NB_SESSIONS__AFFINITY: |
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  {{- with .Values.notebooks.sessionTolerations }}
+  NB_SESSIONS__TOLERATIONS: |
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  NB_SESSIONS__ENFORCE_CPU_LIMITS: {{ .Values.notebooks.enforceCPULimits | quote }}
+  NB_CLOUD_STORAGE__ENABLED: {{ .Values.notebooks.cloudstorage.enabled | quote }}
+  NB_CLOUD_STORAGE__STORAGE_CLASS: {{ .Values.notebooks.cloudstorage.storageClass | default "csi-rclone" | quote }}
+  NB_SESSIONS__TERMINATION_WARNING_DURATION_SECONDS: {{ .Values.notebooks.sessionAutosave.terminationWarningDurationSeconds | quote }}
+  NB_VERSION: {{ .Values.notebooks.image.tag | quote }}
+  {{ if .Values.notebooks.sessionsNamespace }}
+  NB_K8S__SESSIONS_NAMESPACE: {{ .Values.notebooks.sessionsNamespace | quote }}
+  {{ end }}
+  NB_K8S__RENKU_NAMESPACE: {{ .Release.Namespace | quote }}
+  NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_ID: renku
+  NB_SESSIONS__GIT_PROXY__RENKU_CLIENT_SECRET: {{ .Values.global.gateway.clientSecret | quote }}
+  NB_KEYCLOAK_REALM: {{ include "renku.keycloak.realm" . | quote }}
+  NB_SESSIONS__SSH__ENABLED: {{ .Values.notebooks.ssh.enabled | quote }}
+  {{- if not (kindIs "invalid" .Values.notebooks.ssh.hostKeySecret) }}
+  NB_SESSIONS__SSH__HOST_KEY_SECRET: {{ .Values.notebooks.ssh.hostKeySecret | quote }}
+  {{- end }}
+  NB_DATA_SERVICE_URL: {{ printf "http://%s-data-service/api/data" .Release.Name }}
+  NB_USER_SECRETS__SECRETS_STORAGE_SERVICE_URL: {{ printf "http://%s-secrets-storage" .Release.Name }}
+  NB_USER_SECRETS__IMAGE: "{{ .Values.notebooks.secretsMount.image.repository}}:{{.Values.notebooks.secretsMount.image.tag }}"
+---
+
@@ -15,6 +15,10 @@ spec:
             matchLabels:
               app: {{ template "renku.notebooks.name" . }}
               release: {{ .Release.Name }}
+        - podSelector:
+            matchLabels:
+              app: renku-data-service
+              release: {{ .Release.Name }}
       ports:
         - protocol: TCP
           port: http