From b11b79e47a0a7b92cbd5651738ea312aca9e5ffc Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 27 Oct 2023 11:39:46 -0700 Subject: [PATCH 1/8] add db-backups cronjob --- chart/templates/db-backup.yaml | 51 ++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 chart/templates/db-backup.yaml diff --git a/chart/templates/db-backup.yaml b/chart/templates/db-backup.yaml new file mode 100644 index 0000000000..5ba7dde1f4 --- /dev/null +++ b/chart/templates/db-backup.yaml @@ -0,0 +1,51 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mongodb-backup + namespace: {{ .Release.Namespace }} +spec: + schedule: "*/1 * * * *" + failedJobsHistoryLimit: 2 + successfulJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + restartPolicy: Never + initContainers: + - name: mongodump + image: {{ .Values.mongo_image }} + imagePullPolicy: {{ .Values.mongo_pull_policy }} + volumeMounts: + - name: backups + mountPath: /backups + + command: + - mongodump + {{- if .Values.mongo_auth.db_url }} + - --uri={{ .Values.mongo_auth.db_url }} + {{- else }} + - --uri=mongodb://{{ .Values.mongo_auth.username }}:{{ .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin + {{- end }} + - --archive=/backups/backup.archive + + containers: + - name: minioupload + image: {{ .Values.minio_mc_image }} + imagePullPolicy: {{ .Values.minio_pull_policy }} + volumeMounts: + - name: backups + mountPath: /backups + + {{- with (index .Values.storages 0) }} + command: + - /bin/sh + - -c + - mc alias set BACKUP {{ .endpoint_url }} {{ .access_key }} {{ .secret_key }}; mc cp /db-backups/backup.archive BACKUP/{{ .bucket_name }}/backups/mongodb-$(date +%Y-%m-%d).archive + + {{- end }} + volumes: + - name: backups + emptyDir: {} + + From 929f1668af4abb481019f7a793651b896330a627 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 27 Oct 2023 14:32:30 -0700 Subject: [PATCH 2/8] work --- chart/templates/db-backup.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chart/templates/db-backup.yaml b/chart/templates/db-backup.yaml index 5ba7dde1f4..59ed933b0b 100644 --- a/chart/templates/db-backup.yaml +++ b/chart/templates/db-backup.yaml @@ -9,6 +9,7 @@ spec: successfulJobsHistoryLimit: 1 jobTemplate: spec: + activeDeadlineSeconds: 60 template: spec: restartPolicy: Never @@ -37,11 +38,11 @@ spec: - name: backups mountPath: /backups - {{- with (index .Values.storages 0) }} + {{- with .Values.backups }} command: - /bin/sh - -c - - mc alias set BACKUP {{ .endpoint_url }} {{ .access_key }} {{ .secret_key }}; mc cp /db-backups/backup.archive BACKUP/{{ .bucket_name }}/backups/mongodb-$(date +%Y-%m-%d).archive + - set -e; mc alias set BACKUP {{ .endpoint_url }} {{ .access_key }} {{ .secret_key }}; mc cp /backups/backup.archive BACKUP/mongodb-$(date +%Y-%m-%d).archive {{- end }} volumes: From ea0c4204705a203bf3bd9ecd50a259f1eb792110 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 27 Oct 2023 16:52:00 -0700 Subject: [PATCH 3/8] run conditionally on first storage with 'is_db_backup' set! --- chart/templates/db-backup.yaml | 42 +++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/chart/templates/db-backup.yaml b/chart/templates/db-backup.yaml index 59ed933b0b..e74ed06261 100644 --- a/chart/templates/db-backup.yaml +++ b/chart/templates/db-backup.yaml @@ -1,10 +1,20 @@ +{{ $backup := false }} +{{- range $storage := .Values.storages -}} + {{- if $storage.is_db_backup -}} + {{- $backup = $storage }} + + {{- end }} +{{- end}} + +{{- if $backup -}} +--- apiVersion: batch/v1 kind: CronJob metadata: name: mongodb-backup namespace: {{ .Release.Namespace }} spec: - schedule: "*/1 * * * *" + schedule: "26 12 * * *" failedJobsHistoryLimit: 2 successfulJobsHistoryLimit: 1 jobTemplate: @@ -14,7 +24,7 @@ spec: spec: restartPolicy: Never initContainers: - - name: mongodump + - name: dump image: {{ .Values.mongo_image }} imagePullPolicy: {{ .Values.mongo_pull_policy }} volumeMounts: @@ -31,22 +41,38 @@ spec: - --archive=/backups/backup.archive containers: - - name: minioupload + - name: upload image: {{ .Values.minio_mc_image }} imagePullPolicy: {{ .Values.minio_pull_policy }} volumeMounts: - name: backups mountPath: /backups - {{- with .Values.backups }} command: - - /bin/sh + - /bin/bash - -c - - set -e; mc alias set BACKUP {{ .endpoint_url }} {{ .access_key }} {{ .secret_key }}; mc cp /backups/backup.archive BACKUP/mongodb-$(date +%Y-%m-%d).archive + {{- if $backup.bucket_name }} + - | + mc alias set BACKUP {{ $backup.endpoint_url }} {{ $backup.access_key }} {{ $backup.secret_key }} || exit 1; + mc cp /backups/backup.archive BACKUP/{{ $backup.bucket_name }}/db-backup/mongodb-$(date +%Y-%m-%d).archive || exit 2 + + {{- else }} + - | + do_upload() { + [[ "$1" =~ (https?://[^/]+/)([^/]+)/(.*)[/]$ ]]; + origin=${BASH_REMATCH[1]}; + bucket=${BASH_REMATCH[2]}; + path=${BASH_REMATCH[3]}; + + mc alias set BACKUP $origin {{ $backup.access_key }} {{ $backup.secret_key }} || exit 1; + mc cp /backups/backup.archive BACKUP/$bucket/$path/db-backup/mongodb-$(date +%Y-%m-%d).archive || exit 2; + }; + do_upload "{{ $backup.endpoint_url }}"; + exit $? + {{- end }} - {{- end }} volumes: - name: backups emptyDir: {} - + {{- end }} \ No newline at end of file From eb2091fb1325da5284145599f7a818dc5ee69aea Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 27 Oct 2023 17:45:23 -0700 Subject: [PATCH 4/8] use secrets for mongo / minio auth instead of inserting credentials directly enable db backups by default --- chart/templates/db-backup.yaml | 71 ++++++++++++++++++++-------------- chart/templates/mongo.yaml | 10 +++++ chart/values.yaml | 4 ++ 3 files changed, 57 insertions(+), 28 deletions(-) diff --git a/chart/templates/db-backup.yaml b/chart/templates/db-backup.yaml index e74ed06261..24db29b1ca 100644 --- a/chart/templates/db-backup.yaml +++ b/chart/templates/db-backup.yaml @@ -1,7 +1,7 @@ {{ $backup := false }} {{- range $storage := .Values.storages -}} {{- if $storage.is_db_backup -}} - {{- $backup = $storage }} + {{- $backup = print "storage-" $storage.name }} {{- end }} {{- end}} @@ -12,9 +12,9 @@ apiVersion: batch/v1 kind: CronJob metadata: name: mongodb-backup - namespace: {{ .Release.Namespace }} + namespace: {{ $.Values.crawler_namespace }} spec: - schedule: "26 12 * * *" + schedule: "* * * * *" failedJobsHistoryLimit: 2 successfulJobsHistoryLimit: 1 jobTemplate: @@ -32,13 +32,16 @@ spec: mountPath: /backups command: - - mongodump - {{- if .Values.mongo_auth.db_url }} - - --uri={{ .Values.mongo_auth.db_url }} - {{- else }} - - --uri=mongodb://{{ .Values.mongo_auth.username }}:{{ .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin - {{- end }} - - --archive=/backups/backup.archive + - /bin/bash + - -c + - mongodump --uri=$MONGO_DB_URL --archive=/backups/backup.archive + + env: + - name: MONGO_DB_URL + valueFrom: + secretKeyRef: + name: mongo-auth + key: MONGO_DB_URL containers: - name: upload @@ -51,25 +54,37 @@ spec: command: - /bin/bash - -c - {{- if $backup.bucket_name }} - | - mc alias set BACKUP {{ $backup.endpoint_url }} {{ $backup.access_key }} {{ $backup.secret_key }} || exit 1; - mc cp /backups/backup.archive BACKUP/{{ $backup.bucket_name }}/db-backup/mongodb-$(date +%Y-%m-%d).archive || exit 2 - - {{- else }} - - | - do_upload() { - [[ "$1" =~ (https?://[^/]+/)([^/]+)/(.*)[/]$ ]]; - origin=${BASH_REMATCH[1]}; - bucket=${BASH_REMATCH[2]}; - path=${BASH_REMATCH[3]}; - - mc alias set BACKUP $origin {{ $backup.access_key }} {{ $backup.secret_key }} || exit 1; - mc cp /backups/backup.archive BACKUP/$bucket/$path/db-backup/mongodb-$(date +%Y-%m-%d).archive || exit 2; - }; - do_upload "{{ $backup.endpoint_url }}"; - exit $? - {{- end }} + [[ "$ENDPOINT_URL" =~ (https?://[^/]+/)([^/]+)/(.*)$ ]]; + origin=${BASH_REMATCH[1]}; + bucket=${BASH_REMATCH[2]}; + path=${BASH_REMATCH[3]}; + + echo "endpoint: $ENDPOINT_URL" + echo "origin: $origin" + echo "bucket: $bucket" + echo "path: ${path}db-backup/" + + mc alias set BACKUP $origin $ACCESS_KEY $SECRET_KEY || exit 1; + mc cp /backups/backup.archive BACKUP/${bucket}/${path}db-backup/mongodb-$(date +%Y-%m-%d).archive || exit 2; + env: + - name: ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $backup }} + key: STORE_ACCESS_KEY + + - name: SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $backup }} + key: STORE_SECRET_KEY + + - name: ENDPOINT_URL + valueFrom: + secretKeyRef: + name: {{ $backup }} + key: STORE_ENDPOINT_URL volumes: - name: backups diff --git a/chart/templates/mongo.yaml b/chart/templates/mongo.yaml index 0028211a9c..e2aee7d86a 100644 --- a/chart/templates/mongo.yaml +++ b/chart/templates/mongo.yaml @@ -13,7 +13,12 @@ stringData: MONGO_INITDB_ROOT_USERNAME: "{{ .Values.mongo_auth.username }}" MONGO_INITDB_ROOT_PASSWORD: "{{ .Values.mongo_auth.password }}" MONGO_HOST: "{{ .Values.mongo_host }}" + +{{- if .Values.mongo_auth.db_url }} MONGO_DB_URL: "{{ .Values.mongo_auth.db_url }}" +{{- else }} + MONGO_DB_URL: "mongodb://{{ .Values.mongo_auth.username }}:{{ .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin" +{{- end }} --- apiVersion: v1 @@ -27,7 +32,12 @@ stringData: MONGO_INITDB_ROOT_USERNAME: "{{ .Values.mongo_auth.username }}" MONGO_INITDB_ROOT_PASSWORD: "{{ .Values.mongo_auth.password }}" MONGO_HOST: "{{ .Values.mongo_host }}" +{{- if .Values.mongo_auth.db_url }} MONGO_DB_URL: "{{ .Values.mongo_auth.db_url }}" +{{- else }} + MONGO_DB_URL: "mongodb://{{ .Values.mongo_auth.username }}:{{ .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin" +{{- end }} + {{- if .Values.mongo_local }} diff --git a/chart/values.yaml b/chart/values.yaml index 3aa6f30f0c..89e00e88c6 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -272,6 +272,10 @@ storages: endpoint_url: "http://local-minio.default:9000/" + # if set to true, mongodb dump backups will be created in this storage + # daily under /db-backups + is_db_backup: true + # optional: duration in minutes for WACZ download links to be valid # used by webhooks and replay From b4eea010ad4a5bd36c99d1912b5325a7902e3061 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 27 Oct 2023 17:59:24 -0700 Subject: [PATCH 5/8] escape user/pass in url --- chart/templates/mongo.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/chart/templates/mongo.yaml b/chart/templates/mongo.yaml index e2aee7d86a..5aff4b9f39 100644 --- a/chart/templates/mongo.yaml +++ b/chart/templates/mongo.yaml @@ -17,7 +17,7 @@ stringData: {{- if .Values.mongo_auth.db_url }} MONGO_DB_URL: "{{ .Values.mongo_auth.db_url }}" {{- else }} - MONGO_DB_URL: "mongodb://{{ .Values.mongo_auth.username }}:{{ .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin" + MONGO_DB_URL: "mongodb://{{ urlquery .Values.mongo_auth.username }}:{{ urlquery .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin" {{- end }} --- @@ -35,7 +35,7 @@ stringData: {{- if .Values.mongo_auth.db_url }} MONGO_DB_URL: "{{ .Values.mongo_auth.db_url }}" {{- else }} - MONGO_DB_URL: "mongodb://{{ .Values.mongo_auth.username }}:{{ .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin" + MONGO_DB_URL: "mongodb://{{ urlquery .Values.mongo_auth.username }}:{{ urlquery .Values.mongo_auth.password }}@{{ .Values.mongo_host }}:27017/browsertrixcloud?tls=false&authSource=admin" {{- end }} From 293186e27140c230a5f4a775e5b9c5f89fe55c2c Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Fri, 27 Oct 2023 18:33:03 -0700 Subject: [PATCH 6/8] use separate 'db_backups' value, enable backups if db_backups.storage is set, also allow customizing db_backups.schedule and db_backups.path --- chart/templates/db-backup.yaml | 29 +++++++++++++++++++---------- chart/values.yaml | 10 +++++++--- 2 files changed, 26 insertions(+), 13 deletions(-) diff --git a/chart/templates/db-backup.yaml b/chart/templates/db-backup.yaml index 24db29b1ca..73d787c100 100644 --- a/chart/templates/db-backup.yaml +++ b/chart/templates/db-backup.yaml @@ -1,12 +1,18 @@ -{{ $backup := false }} -{{- range $storage := .Values.storages -}} - {{- if $storage.is_db_backup -}} - {{- $backup = print "storage-" $storage.name }} +{{- if .Values.db_backup.storage -}} + +{{ $backup := print "storage-" .Values.db_backup.storage }} +{{ $found := false }} +{{- range $storage := .Values.storages -}} + {{- if eq $storage.name $.Values.db_backup.storage -}} + {{- $found = true }} {{- end }} {{- end}} -{{- if $backup -}} +{{- if not $found }} +{{ fail (print "Backup storage '" .Values.db_backup.storage "' not found, check the value of 'db_backups.storage' and 'storages' to ensure this is a valid storage name")}} +{{- end }} + --- apiVersion: batch/v1 kind: CronJob @@ -14,7 +20,7 @@ metadata: name: mongodb-backup namespace: {{ $.Values.crawler_namespace }} spec: - schedule: "* * * * *" + schedule: "{{ .Values.db_backup.schedule | default "26 0 * * *" }}" failedJobsHistoryLimit: 2 successfulJobsHistoryLimit: 1 jobTemplate: @@ -59,14 +65,14 @@ spec: origin=${BASH_REMATCH[1]}; bucket=${BASH_REMATCH[2]}; path=${BASH_REMATCH[3]}; - + echo "endpoint: $ENDPOINT_URL" echo "origin: $origin" echo "bucket: $bucket" - echo "path: ${path}db-backup/" + echo "path: ${path}${DB_PATH}/" mc alias set BACKUP $origin $ACCESS_KEY $SECRET_KEY || exit 1; - mc cp /backups/backup.archive BACKUP/${bucket}/${path}db-backup/mongodb-$(date +%Y-%m-%d).archive || exit 2; + mc cp /backups/backup.archive BACKUP/${bucket}/${path}${DB_PATH}/mongodb-$(date +%Y-%m-%dT%H-%M-%S).archive || exit 2; env: - name: ACCESS_KEY valueFrom: @@ -86,8 +92,11 @@ spec: name: {{ $backup }} key: STORE_ENDPOINT_URL + - name: DB_PATH + value: {{ .Values.db_backup.path | default "db-backup" }} + volumes: - name: backups emptyDir: {} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/chart/values.yaml b/chart/values.yaml index 89e00e88c6..a7ce905c11 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -272,9 +272,13 @@ storages: endpoint_url: "http://local-minio.default:9000/" - # if set to true, mongodb dump backups will be created in this storage - # daily under /db-backups - is_db_backup: true + +# if storage is set, mongodb backups will be created in this storage +# daily under /db-backups +db_backup: + storage: "default" + #schedule: "26 0 * * *" + #path: "db-backup" # optional: duration in minutes for WACZ download links to be valid From 8fe364b3190a01c326ccefbf79d99c434078c220 Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 28 Oct 2023 11:10:49 -0700 Subject: [PATCH 7/8] disable by default --- chart/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chart/values.yaml b/chart/values.yaml index a7ce905c11..50b126a3aa 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -276,7 +276,7 @@ storages: # if storage is set, mongodb backups will be created in this storage # daily under /db-backups db_backup: - storage: "default" + #storage: "default" #schedule: "26 0 * * *" #path: "db-backup" From 29a6e0192e1657f59ad06e4ce56435305707266f Mon Sep 17 00:00:00 2001 From: Ilya Kreymer Date: Sat, 28 Oct 2023 11:33:33 -0700 Subject: [PATCH 8/8] fix empty value check --- chart/templates/db-backup.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/chart/templates/db-backup.yaml b/chart/templates/db-backup.yaml index 73d787c100..b614d3f286 100644 --- a/chart/templates/db-backup.yaml +++ b/chart/templates/db-backup.yaml @@ -1,7 +1,8 @@ -{{- if .Values.db_backup.storage -}} +{{- if (.Values.db_backup).storage -}} {{ $backup := print "storage-" .Values.db_backup.storage }} +# find matching storage in storages list {{ $found := false }} {{- range $storage := .Values.storages -}} {{- if eq $storage.name $.Values.db_backup.storage -}} @@ -9,6 +10,7 @@ {{- end }} {{- end}} +# error here if backup storage doesn't match one of the specified storages {{- if not $found }} {{ fail (print "Backup storage '" .Values.db_backup.storage "' not found, check the value of 'db_backups.storage' and 'storages' to ensure this is a valid storage name")}} {{- end }} @@ -25,7 +27,7 @@ spec: successfulJobsHistoryLimit: 1 jobTemplate: spec: - activeDeadlineSeconds: 60 + activeDeadlineSeconds: 600 template: spec: restartPolicy: Never