diff --git a/charts/latest/blob-csi-driver-v0.0.0.tgz b/charts/latest/blob-csi-driver-v0.0.0.tgz index 8ba56091e0..97bc68eae2 100644 Binary files a/charts/latest/blob-csi-driver-v0.0.0.tgz and b/charts/latest/blob-csi-driver-v0.0.0.tgz differ diff --git a/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml b/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml index f7b6a48f3b..4874f89b61 100644 --- a/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml +++ b/charts/latest/blob-csi-driver/templates/csi-blob-node.yaml @@ -66,7 +66,6 @@ spec: tolerations: {{ toYaml . | indent 8 }} {{- end }} -{{- if .Values.node.enableBlobfuseProxy }} initContainers: - name: install-blobfuse-proxy {{- if hasPrefix "/" .Values.image.blob.repository }} @@ -90,6 +89,8 @@ spec: value: "{{ .Values.node.blobfuseProxy.installBlobfuse2 }}" - name: BLOBFUSE2_VERSION value: "{{ .Values.node.blobfuseProxy.blobfuse2Version }}" + - name: INSTALL_BLOBFUSE_PROXY + value: "{{ .Values.node.enableBlobfuseProxy }}" - name: SET_MAX_OPEN_FILE_NUM value: "{{ .Values.node.blobfuseProxy.setMaxOpenFileNum }}" - name: MAX_FILE_NUM @@ -99,9 +100,10 @@ spec: volumeMounts: - name: host-usr mountPath: /host/usr + - name: host-usr-local + mountPath: /host/usr/local - name: host-etc mountPath: /host/etc -{{- end }} containers: - name: liveness-probe imagePullPolicy: {{ .Values.image.livenessProbe.pullPolicy }} @@ -265,14 +267,15 @@ spec: name: mountpoint-dir {{- end }} volumes: -{{- if .Values.node.enableBlobfuseProxy }} - name: host-usr hostPath: path: /usr + - name: host-usr-local + hostPath: + path: /usr/local - name: host-etc hostPath: path: /etc -{{- end }} - hostPath: path: {{ .Values.linux.kubelet }}/plugins/{{ .Values.driver.name }} type: DirectoryOrCreate diff --git a/charts/latest/blob-csi-driver/values.yaml b/charts/latest/blob-csi-driver/values.yaml index 8ff043a423..814c43d2e5 100644 --- a/charts/latest/blob-csi-driver/values.yaml +++ b/charts/latest/blob-csi-driver/values.yaml @@ -114,7 +114,7 @@ node: livenessProbe: healthPort: 29633 logLevel: 5 - enableBlobfuseProxy: false + enableBlobfuseProxy: true blobfuseProxy: installBlobfuse: true blobfuseVersion: "1.4.5" diff --git a/deploy/csi-blob-node.yaml b/deploy/csi-blob-node.yaml index 45980e3002..18836c22b6 100644 --- a/deploy/csi-blob-node.yaml +++ b/deploy/csi-blob-node.yaml @@ -68,6 +68,8 @@ spec: volumeMounts: - name: host-usr mountPath: /host/usr + - name: host-usr-local + mountPath: /host/usr/local - name: host-etc mountPath: /host/etc containers: @@ -177,6 +179,9 @@ spec: - name: host-usr hostPath: path: /usr + - name: host-usr-local + hostPath: + path: /usr/local - name: host-etc hostPath: path: /etc diff --git a/pkg/blobfuse-proxy/init.sh b/pkg/blobfuse-proxy/init.sh index c0e2ff1c5c..5826f21858 100755 --- a/pkg/blobfuse-proxy/init.sh +++ b/pkg/blobfuse-proxy/init.sh @@ -16,7 +16,6 @@ set -xe -INSTALL_BLOBFUSE_PROXY=${INSTALL_BLOBFUSE_PROXY:-true} DISABLE_UPDATEDB=${DISABLE_UPDATEDB:-true} SET_MAX_OPEN_FILE_NUM=${SET_MAX_OPEN_FILE_NUM:-true} SET_READ_AHEAD_SIZE=${SET_READ_AHEAD_SIZE:-true} @@ -25,113 +24,30 @@ READ_AHEAD_KB=${READ_AHEAD_KB:-15380} HOST_CMD="nsenter --mount=/proc/1/ns/mnt" DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"') -echo "Linux distribution: $DISTRIBUTION" ARCH=$($HOST_CMD uname -m) -echo "Linux Arch is $(uname -m)" - -if [ "${ARCH}" != "aarch64" ] -then - if [ "${DISTRIBUTION}" = "ubuntu" ] && { [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]; } - then - release=$($HOST_CMD lsb_release -rs) - echo "Ubuntu release: $release" - - if [ "$(expr "$release" \< "22.04")" -eq 1 ] - then - cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb - else - cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb - fi - - # when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue. - # refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively - yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update - - pkg_list="" - if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ] - then - pkg_list="${pkg_list} fuse" - # install blobfuse with latest version or specific version - if [ -z "${BLOBFUSE_VERSION}" ]; then - echo "install blobfuse with latest version" - pkg_list="${pkg_list} blobfuse" - else - pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}" - fi - fi - - if [ "${INSTALL_BLOBFUSE2}" = "true" ] - then - if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then - echo "install fuse for blobfuse2" - pkg_list="${pkg_list} fuse" - else - echo "install fuse3 for blobfuse2, current release is $release" - pkg_list="${pkg_list} fuse3" - fi - - # install blobfuse2 with latest version or specific version - if [ -z "${BLOBFUSE2_VERSION}" ]; then - echo "install blobfuse2 with latest version" - pkg_list="${pkg_list} blobfuse2" - else - pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}" - fi - fi - echo "begin to install ${pkg_list}" - $HOST_CMD apt-get install -y $pkg_list - $HOST_CMD rm -f /etc/packages-microsoft-prod.deb - fi - - updateBlobfuseProxy="true" - if [ -f "/host/usr/bin/blobfuse-proxy" ];then - old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}') - new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}') - if [ "$old" = "$new" ];then - updateBlobfuseProxy="false" - echo "no need to update blobfuse-proxy" - fi - fi - - if [ "$updateBlobfuseProxy" = "true" ];then - echo "copy blobfuse-proxy...." - rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock - rm -rf /host/usr/bin/blobfuse-proxy - cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy - chmod 755 /host/usr/bin/blobfuse-proxy - fi - - updateService="true" - if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then - old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}') - new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}') - if [ "$old" = "$new" ];then - updateService="false" - echo "no need to update blobfuse-proxy.service" - fi - fi - - if [ "$updateService" = "true" ];then - echo "copy blobfuse-proxy.service...." - mkdir -p /host/usr/lib/systemd/system - cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service - fi - - if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then - if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then - echo "start blobfuse-proxy...." - $HOST_CMD systemctl daemon-reload - $HOST_CMD systemctl enable blobfuse-proxy.service - $HOST_CMD systemctl restart blobfuse-proxy.service - fi - fi -fi - +echo "Linux distribution: $DISTRIBUTION, Arch: $ARCH" + +# install blobfuse-proxy and blobfuse/blofuse2 if needed +case "${DISTRIBUTION}" in + "ubuntu") + . ./pkg/blobfuse-proxy/install-proxy-ubuntu.sh + ;; + "rhcos") + . ./pkg/blobfuse-proxy/install-proxy-rhcos.sh + ;; + *) + echo "Unsupported distribution: ${DISTRIBUTION}" + exit 1 + ;; +esac + +# set max open file num if [ "${SET_MAX_OPEN_FILE_NUM}" = "true" ] then $HOST_CMD sysctl -w fs.file-max="${MAX_FILE_NUM}" fi +# disable updatedb updateDBConfigPath="/host/etc/updatedb.conf" if [ "${DISABLE_UPDATEDB}" = "true" ] && [ -f ${updateDBConfigPath} ] then @@ -143,6 +59,7 @@ then cat ${updateDBConfigPath} fi +# set read ahead size if [ "${SET_READ_AHEAD_SIZE}" = "true" ] then echo "set read ahead size to ${READ_AHEAD_KB}KB" diff --git a/pkg/blobfuse-proxy/install-proxy-rhcos.sh b/pkg/blobfuse-proxy/install-proxy-rhcos.sh new file mode 100644 index 0000000000..7e866e4848 --- /dev/null +++ b/pkg/blobfuse-proxy/install-proxy-rhcos.sh @@ -0,0 +1,81 @@ +#!/bin/sh + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +if [ "${ARCH}" == "aarch64" ] +then + echo "aarch64 is not supported yet..." + return 0 +fi + +# install blobfuse2 +if [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ] +then + echo "copy blobfuse2...." + cp /usr/bin/blobfuse2 /host/usr/local/bin/blobfuse2 +fi + +# install blobfuse-proxy +updateBlobfuseProxy="true" +if [ -f "/host/usr/local/bin/blobfuse-proxy" ];then + old=$(sha256sum /host/usr/local/bin/blobfuse-proxy | awk '{print $1}') + new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}') + if [ "$old" = "$new" ];then + updateBlobfuseProxy="false" + echo "no need to update blobfuse-proxy" + fi +fi +if [ "$updateBlobfuseProxy" = "true" ];then + echo "copy blobfuse-proxy...." + rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock + rm -rf /host/usr/local/bin/blobfuse-proxy + cp /blobfuse-proxy/blobfuse-proxy /host/usr/local/bin/blobfuse-proxy + chmod 755 /host/usr/local/bin/blobfuse-proxy +fi + +updateService="true" +echo "change from /usr/bin/blobfuse-proxy to /usr/local/bin/blobfuse-proxy in blobfuse-proxy.service" +sed -i 's/\/usr\/bin\/blobfuse-proxy/\/usr\/local\/bin\/blobfuse-proxy/g' /blobfuse-proxy/blobfuse-proxy.service +if [ -f "/host/etc/systemd/system/blobfuse-proxy.service" ];then + old=$(sha256sum /host/etc/systemd/system/blobfuse-proxy.service | awk '{print $1}') + new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}') + if [ "$old" = "$new" ];then + updateService="false" + echo "no need to update blobfuse-proxy.service" + fi +fi +if [ "$updateService" = "true" ];then + echo "copy blobfuse-proxy.service...." + mkdir -p /host/etc/systemd/system/ + cp /blobfuse-proxy/blobfuse-proxy.service /host/etc/systemd/system/blobfuse-proxy.service +fi + +if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then + if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then + echo "start blobfuse-proxy...." + $HOST_CMD systemctl daemon-reload + $HOST_CMD systemctl enable blobfuse-proxy.service + $HOST_CMD systemctl restart blobfuse-proxy.service + fi +fi + + + + + + + diff --git a/pkg/blobfuse-proxy/install-proxy-ubuntu.sh b/pkg/blobfuse-proxy/install-proxy-ubuntu.sh new file mode 100644 index 0000000000..b9abab80fc --- /dev/null +++ b/pkg/blobfuse-proxy/install-proxy-ubuntu.sh @@ -0,0 +1,127 @@ +#!/bin/sh + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe + +if [ "${ARCH}" == "aarch64" ] +then + echo "aarch64 is not supported yet..." + return 0 +fi + +# install blobfuse/blobfuse2 +if [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ] +then + echo "start to install blobfuse/blobfuse2...." + + release=$($HOST_CMD lsb_release -rs) + echo "Ubuntu release: $release" + + if [ "$(expr "$release" \< "22.04")" -eq 1 ] + then + cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb + else + cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb + fi + # when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue. + # refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively + yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update + + pkg_list="" + # blobfuse + if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ] + then + pkg_list="${pkg_list} fuse" + if [ -z "${BLOBFUSE_VERSION}" ]; then + echo "install blobfuse with latest version" + pkg_list="${pkg_list} blobfuse" + else + pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}" + fi + fi + + # blobfuse2 + if [ "${INSTALL_BLOBFUSE2}" = "true" ] + then + if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then + echo "install fuse for blobfuse2" + pkg_list="${pkg_list} fuse" + else + echo "install fuse3 for blobfuse2, current release is $release" + pkg_list="${pkg_list} fuse3" + fi + + if [ -z "${BLOBFUSE2_VERSION}" ]; then + echo "install blobfuse2 with latest version" + pkg_list="${pkg_list} blobfuse2" + else + pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}" + fi + fi + + echo "begin to install ${pkg_list}" + $HOST_CMD apt-get install -y $pkg_list + $HOST_CMD rm -f /etc/packages-microsoft-prod.deb +fi + +# install blobfuse-proxy +updateBlobfuseProxy="true" +if [ -f "/host/usr/bin/blobfuse-proxy" ];then + old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}') + new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}') + if [ "$old" = "$new" ];then + updateBlobfuseProxy="false" + echo "no need to update blobfuse-proxy" + fi +fi +if [ "$updateBlobfuseProxy" = "true" ];then + echo "copy blobfuse-proxy...." + rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock + rm -rf /host/usr/bin/blobfuse-proxy + cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy + chmod 755 /host/usr/bin/blobfuse-proxy +fi + +updateService="true" +if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then + old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}') + new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}') + if [ "$old" = "$new" ];then + updateService="false" + echo "no need to update blobfuse-proxy.service" + fi +fi +if [ "$updateService" = "true" ];then + echo "copy blobfuse-proxy.service...." + mkdir -p /host/usr/lib/systemd/system + cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service +fi + +if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then + if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then + echo "start blobfuse-proxy...." + $HOST_CMD systemctl daemon-reload + $HOST_CMD systemctl enable blobfuse-proxy.service + $HOST_CMD systemctl restart blobfuse-proxy.service + fi +fi + + + + + + + diff --git a/pkg/blobplugin/Dockerfile b/pkg/blobplugin/Dockerfile index 1b5ab5c0ac..10633bddd8 100644 --- a/pkg/blobplugin/Dockerfile +++ b/pkg/blobplugin/Dockerfile @@ -21,10 +21,14 @@ COPY ${binary} /blobplugin RUN mkdir /blobfuse-proxy/ COPY ./pkg/blobfuse-proxy/init.sh /blobfuse-proxy/ +COPY ./pkg/blobfuse-proxy/install-proxy-ubuntu.sh /blobfuse-proxy/ +COPY ./pkg/blobfuse-proxy/install-proxy-rhcos.sh /blobfuse-proxy/ COPY ./pkg/blobfuse-proxy/blobfuse-proxy.service /blobfuse-proxy/ COPY ./_output/${ARCH}/blobfuse-proxy /blobfuse-proxy/ RUN chmod +x /blobfuse-proxy/init.sh && \ + chmod +x /blobfuse-proxy/install-proxy-ubuntu.sh && \ + chmod +x /blobfuse-proxy/install-proxy-rhcos.sh && \ chmod +x /blobfuse-proxy/blobfuse-proxy.service && \ chmod +x /blobfuse-proxy/blobfuse-proxy