From 6d2f889db9478fee8e73dd82c940f173386806f7 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Tue, 18 Jun 2024 11:14:36 -0700 Subject: [PATCH] Docker images: fix raspbian-bullseye, install python3-venv (#286) **Issue:** https://github.com/awslabs/aws-crt-python/pull/573 needs python3's `venv` installed on all docker images, due to [PEP 668](https://peps.python.org/pep-0668/). While trying to update our `raspbian-bullseye` image, I discovered it's been broken for months. We'd been [hacking around this](https://github.com/awslabs/aws-crt-builder/pull/256) by not even trying to build it anymore, copy/pasting an old working version instead. **Description of changes:** - Install `python3-venv` in Docker images that didn't already have it - Fix `raspbian-bullseye` image. - The old image was getting 404 from some repository while running `apt-get update` - Use different base image: https://github.com/dtcooper/raspberrypi-os-docker. This was the only easily available `bullseye` image I could find. The implementation looks simple, and it's working. - Remove extra stuff from this docker image, like building additional versions of python. I don't know what these were used for (@xiazhvera do you know?). None of our other docker images go through this trouble to build extra python versions. This is already the slowest docker image, due to being armv7, so removing extra work and complication seems extra good. --- .../raspbian-bullseye/Dockerfile | 90 +------------------ .../docker-images/ubuntu-18-x64/Dockerfile | 1 + .github/workflows/create-channel.yml | 33 +------ .github/workflows/create-release.yml | 33 +------ 4 files changed, 7 insertions(+), 150 deletions(-) diff --git a/.github/docker-images/raspbian-bullseye/Dockerfile b/.github/docker-images/raspbian-bullseye/Dockerfile index 620078dda..f816ea623 100644 --- a/.github/docker-images/raspbian-bullseye/Dockerfile +++ b/.github/docker-images/raspbian-bullseye/Dockerfile @@ -1,29 +1,8 @@ -FROM raspbian/stretch as raspbian-bullseye - -############################################################################### -# Upgrade to bullseye from stretch -############################################################################### - -RUN apt-get update \ - && apt-get upgrade -y \ - && apt-get full-upgrade -y \ - && apt-get autoremove --purge -y \ - && apt-get clean -y \ - # Switch to bullseye repository. - && sed -i 's/stretch/bullseye/g' /etc/apt/sources.list \ - # Update all packages. - && apt-get update \ - && apt-get upgrade -y \ - && apt-get full-upgrade -y \ - && apt-get autoremove --purge -y \ - && apt-get clean -y - +FROM --platform=linux/arm/v7 dtcooper/raspberrypi-os:bullseye ############################################################################### # Install prereqs ############################################################################### -# Setup nodejs policy for npm -RUN apt-cache policy nodejs RUN apt-get update -qq \ && apt-get -y install \ git \ @@ -36,6 +15,7 @@ RUN apt-get update -qq \ python3.9 \ python3-dev \ python3-pip \ + python3-venv \ build-essential \ # For PPAs software-properties-common \ @@ -43,78 +23,16 @@ RUN apt-get update -qq \ ca-certificates \ && apt-get clean - -# Override python with python3 -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 - -# Install libs for pythons -RUN sudo apt-get install build-essential checkinstall -y -RUN sudo apt-get install libreadline-gplv2-dev libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev -y - ############################################################################### # Python/AWS CLI ############################################################################### -RUN python -m pip install --upgrade setuptools virtualenv \ - && python -m pip install --upgrade awscli \ +RUN python3 -m pip install --upgrade setuptools virtualenv \ + && python3 -m pip install --upgrade awscli \ && ln -s `find /opt -name aws` /usr/local/bin/aws \ && which aws \ && aws --version - -############################################################################### -# Install Python3.6 for Deployment -############################################################################### -WORKDIR /tmp -RUN wget https://www.python.org/ftp/python/3.6.0/Python-3.6.0.tgz \ - && tar zxf Python-3.6.0.tgz \ - && cd Python-3.6.0 \ - && ./configure \ - && make -j 4 \ - && make altinstall \ - && python3.6 -m pip install --no-input wheel \ - && cd .. - - -############################################################################### -# Install Python3.7 for Deployment -############################################################################### -RUN wget https://www.python.org/ftp/python/3.7.0/Python-3.7.0.tgz \ - && tar zxf Python-3.7.0.tgz \ - && cd Python-3.7.0 \ - && ./configure \ - && make -j 4 \ - && make altinstall \ - && python3.7 -m pip install --no-input wheel \ - && cd .. - -############################################################################### -# Install Python3.8 for Deployment -############################################################################### -RUN wget https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz \ - && tar zxf Python-3.8.0.tgz \ - && cd Python-3.8.0 \ - && ./configure \ - && make -j 4 \ - && make altinstall \ - && python3.8 -m pip install --no-input wheel \ - && cd .. - - -############################################################################### -# Install Python3.10 for Deployment -############################################################################### -RUN wget https://www.python.org/ftp/python/3.10.0/Python-3.10.0.tgz \ - && tar zxf Python-3.10.0.tgz \ - && cd Python-3.10.0 \ - && ./configure \ - && make -j 4 \ - && make altinstall \ - && python3.10 -m pip install --no-input wheel \ - && cd .. - - - ############################################################################### # Install entrypoint ############################################################################### diff --git a/.github/docker-images/ubuntu-18-x64/Dockerfile b/.github/docker-images/ubuntu-18-x64/Dockerfile index 43b6106b2..d5780017c 100644 --- a/.github/docker-images/ubuntu-18-x64/Dockerfile +++ b/.github/docker-images/ubuntu-18-x64/Dockerfile @@ -14,6 +14,7 @@ RUN apt-get update -qq \ # Ubuntu18's default python3 is python3.6, which is EOL. # So install python3.8 (latest version on this distro, circa Oct 2022) python3.8-dev \ + python3.8-venv \ # This installs pip for all python versions on the system # (there is no "python3.8-pip") python3-pip \ diff --git a/.github/workflows/create-channel.yml b/.github/workflows/create-channel.yml index ce86e7cef..dd2d82202 100644 --- a/.github/workflows/create-channel.yml +++ b/.github/workflows/create-channel.yml @@ -79,6 +79,7 @@ jobs: - rhel8-x64 - opensuse-leap - fedora-34-x64 + - raspbian-bullseye - alpine-3.16-x64 - alpine-3.16-x86 - alpine-3.16-arm64 @@ -124,35 +125,3 @@ jobs: run: | export IMAGE_TAG=${{ steps.tag.outputs.release_tag }} docker push ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG - - # The job will directly pull the image from the latest, and push to the new tag. - # This job is only for a temporary fix. The cheated image would not get updated. - passthrough-images: - name: ${{ matrix.variant }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - variant: - - raspbian-bullseye - - steps: - - name: Checkout Sources - uses: actions/checkout@v4 - - - name: Get release tag - uses: ./.github/actions/release-tag - id: tag - - - name: Login to docker repo - run: aws ecr get-login-password --region us-east-1 | docker login ${{ secrets.AWS_ECR_REPO }} -u AWS --password-stdin - - - name: Pull latest image and push - run: | - if ! aws --region us-east-1 ecr describe-repositories --repository-names aws-crt-${{ matrix.variant }} > /dev/null 2>&1; then \ - exit 1 - fi - export IMAGE_TAG=${{ steps.tag.outputs.release_tag }} - docker pull ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest - docker tag ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG - docker push ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 6debfc797..32e462c8c 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -72,6 +72,7 @@ jobs: - rhel8-x64 - opensuse-leap - fedora-34-x64 + - raspbian-bullseye - alpine-3.16-x64 - alpine-3.16-x86 - alpine-3.16-arm64 @@ -129,38 +130,6 @@ jobs: | aws s3 cp - s3://${{env.AWS_S3_BUCKET}}/releases/$IMAGE_TAG/aws-crt-${{ matrix.variant }}.tar.gz aws s3 cp s3://${{env.AWS_S3_BUCKET}}/releases/$IMAGE_TAG/aws-crt-${{ matrix.variant }}.tar.gz s3://${{env.AWS_S3_BUCKET}}/LATEST/aws-crt-${{ matrix.variant }}.tar.gz - # The job will directly pull the image from the latest, and push to the new tag. - # This job is only for a temporary fix. The image would not get updated. - passthrough-images: - name: ${{ matrix.variant }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - variant: - - raspbian-bullseye - - steps: - - name: Checkout Sources - uses: actions/checkout@v4 - - - name: Get release tag - uses: ./.github/actions/release-tag - id: tag - - - name: Login to docker repo - run: aws ecr get-login-password --region us-east-1 | docker login ${{ secrets.AWS_ECR_REPO }} -u AWS --password-stdin - - - name: Pull latest image and push - run: | - if ! aws --region us-east-1 ecr describe-repositories --repository-names aws-crt-${{ matrix.variant }} > /dev/null 2>&1; then \ - exit 1 - fi - export IMAGE_TAG=${{ steps.tag.outputs.release_tag }} - docker pull ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest - docker tag ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:latest ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG - docker push ${{ secrets.AWS_ECR_REPO }}/aws-crt-${{ matrix.variant }}:$IMAGE_TAG - upload-ci-script: name: Upload container ci script runs-on: ubuntu-latest