From c646b1f6e96a4b9623c340e2ae45a47bf8900842 Mon Sep 17 00:00:00 2001 From: Panagiotis Astithas Date: Mon, 14 Oct 2024 14:00:00 -0700 Subject: [PATCH] Reapply "Upgrade to Debian Bookworm and Python 3.11 (#4028)" (#4056) This reverts commit bbf937cd7259dc91186cb1ccf3df157290abc48d. --- Dockerfile | 42 ++++++++++++++++-------------- Makefile | 4 +-- api/query/cache/service/Dockerfile | 2 +- docs/upgrading-go.md | 2 +- results-processor/.python-version | 2 +- results-processor/Dockerfile | 4 +-- results-processor/README.md | 8 +++--- results-processor/requirements.txt | 8 ++---- results-processor/tox.ini | 2 +- 9 files changed, 37 insertions(+), 37 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f8a91bba76..7cdad69a6e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,33 @@ # vim: set expandtab sw=4 -FROM golang:1.22.7-bullseye +FROM golang:1.22.7-bookworm # Create a non-priviledged user to run browsers as (Firefox and Chrome do not # like to run as root). RUN chmod a+rx $HOME && useradd --uid 9999 --user-group --create-home browser +# Add apt repositories for Java, Node.js and Google Cloud CLI +RUN export DISTRO_CODENAME=$(awk -F= '/^VERSION_CODENAME/{print$2}' /etc/os-release) && \ + echo "deb [signed-by=/usr/share/keyrings/adoptium.gpg] https://packages.adoptium.net/artifactory/deb $DISTRO_CODENAME main" > /etc/apt/sources.list.d/adoptium.list && \ + curl -s https://packages.adoptium.net/artifactory/api/gpg/key/public | gpg --dearmor -o /usr/share/keyrings/adoptium.gpg && \ + export NODE_VERSION="18.x" && \ + export ARCH=$(dpkg --print-architecture) && \ + echo "deb [arch=$ARCH signed-by=/usr/share/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_VERSION nodistro main" > /etc/apt/sources.list.d/nodesource.list && \ + curl -s https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /usr/share/keyrings/nodesource.gpg && \ + echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk-$DISTRO_CODENAME main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ + curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg + # Sort the package names! # firefox-esr: provides deps for Firefox (we don't use ESR directly) -# openjdk-11-jdk: provides JDK/JRE to Selenium & gcloud SDK # python-crcmod: native module to speed up CRC checksum in gsutil +# temurin-11-jdk: provides JDK/JRE to Selenium & gcloud SDK RUN apt-get update -qqy && apt-get install -qqy --no-install-suggests \ curl \ firefox-esr \ - lsb-release \ - openjdk-11-jdk \ - python3.9 \ + nodejs \ + python3.11 \ python3-crcmod \ sudo \ + temurin-11-jdk \ tox \ wget \ xvfb && \ @@ -27,21 +38,14 @@ RUN apt-get update -qqy && apt-get install -qqy --no-install-suggests \ # discard the setting. RUN echo "root ALL=(ALL:ALL) ALL" > /etc/sudoers -# Node LTS -RUN curl -sL https://deb.nodesource.com/setup_18.x | bash - && \ - apt-get install -qqy nodejs - -# Google Cloud SDK +# Google Cloud SDK configuration # Based on https://github.com/GoogleCloudPlatform/cloud-sdk-docker/blob/master/Dockerfile -RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" && \ - echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update -qqy && apt-get install -qqy \ - google-cloud-sdk \ - google-cloud-sdk-app-engine-python \ - google-cloud-sdk-app-engine-python-extras \ - google-cloud-sdk-app-engine-go \ - google-cloud-sdk-datastore-emulator && \ +RUN apt-get update -qqy && apt-get install -qqy --no-install-suggests \ + google-cloud-cli \ + google-cloud-cli-app-engine-python \ + google-cloud-cli-app-engine-python-extras \ + google-cloud-cli-app-engine-go \ + google-cloud-cli-datastore-emulator && \ gcloud config set core/disable_usage_reporting true && \ gcloud config set component_manager/disable_update_check true && \ gcloud --version diff --git a/Makefile b/Makefile index aa7eb813d18..8fffb341ad0 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ gcc: apt-get-gcc git: apt-get-git jq: apt-get-jq psmisc: apt-get-psmisc -python3: apt-get-python3.9 +python3: apt-get-python3.11 tox: apt-get-tox unzip: apt-get-unzip wget: apt-get-wget @@ -237,7 +237,7 @@ wget: apt-get-wget java: @ # java has a different apt-get package name. if [[ "$$(which java)" == "" ]]; then \ - sudo apt-get install -qqy --no-install-suggests openjdk-11-jdk; \ + sudo apt-get install -qqy --no-install-suggests temurin-11-jdk; \ fi gpg: diff --git a/api/query/cache/service/Dockerfile b/api/query/cache/service/Dockerfile index 87ff3d459af..5e398105b53 100644 --- a/api/query/cache/service/Dockerfile +++ b/api/query/cache/service/Dockerfile @@ -1,6 +1,6 @@ # Production deployment spec for query cache service. -FROM golang:1.22.7-bullseye as builder +FROM golang:1.22.7-bookworm as builder RUN apt-get update RUN apt-get install -qy --no-install-suggests git diff --git a/docs/upgrading-go.md b/docs/upgrading-go.md index 70ff3875837..57a27cab1a9 100644 --- a/docs/upgrading-go.md +++ b/docs/upgrading-go.md @@ -15,7 +15,7 @@ Once you have confirmed that the desired version is available: - tooling [Dockerfile](../Dockerfile) at the root of the repo - searchcache [Dockerfile](../api/query/cache/service/Dockerfile) -The tooling image and the first stage of searchcache use the same Golang image. Check out the Golang [page](https://hub.docker.com/_/golang?tab=tags) on Docker Hub for the latest tags. Currently, we are using the `bullseye` [release](https://wiki.debian.org/DebianReleases) of Debian. As a result pick the `golang:-bullseye` tag. If bullseye is superseded by a new version, you should change that as well. +The tooling image and the first stage of searchcache use the same Golang image. Check out the Golang [page](https://hub.docker.com/_/golang?tab=tags) on Docker Hub for the latest tags. Currently, we are using the `bookworm` [release](https://wiki.debian.org/DebianReleases) of Debian. As a result pick the `golang:-bookworm` tag. If bookworm is superseded by a new version, you should change that as well. ## Step 3 - Change the version in go.mod diff --git a/results-processor/.python-version b/results-processor/.python-version index bd28b9c5c27..2c0733315e4 100644 --- a/results-processor/.python-version +++ b/results-processor/.python-version @@ -1 +1 @@ -3.9 +3.11 diff --git a/results-processor/Dockerfile b/results-processor/Dockerfile index 9a075a6be9f..3da92879250 100644 --- a/results-processor/Dockerfile +++ b/results-processor/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.20-bullseye +FROM python:3.11.9-bookworm # Install runtime dependencies. # python3-crcmod for faster gsutil checksum @@ -17,7 +17,7 @@ RUN gcloud config set disable_usage_reporting false RUN rm -f $HOME/.config/gcloud/gce # Setup and activate virtualenv. -RUN virtualenv -p python3.9 /env +RUN virtualenv -p python3.11 /env ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH diff --git a/results-processor/README.md b/results-processor/README.md index 2db00cd33da..da2e50bb951 100644 --- a/results-processor/README.md +++ b/results-processor/README.md @@ -1,6 +1,6 @@ ## Basics -The results processor runs on Python 3.9. The entry point is a Flask web server +The results processor runs on Python 3.11. The entry point is a Flask web server (`main.py`). In production, gunicorn is used as the WSGI (see `Dockerfile`) and the container runs as a custom AppEngine Flex instance (see `app.yaml`). @@ -10,7 +10,7 @@ We can create a virtualenv to recreate a setup close to production for daily development. ```bash -virtualenv env -p python3.9 +virtualenv env -p python3.11 . env/bin/activate pip install -r requirements.txt ``` @@ -38,8 +38,8 @@ Dependabot is used to automatically update `requirements.txt`. To manually update dependencies, run the following commands: ```bash -pip3.9 install --user pip-tools -python3.9 -m piptools compile requirements.in +pip3.11 install --user pip-tools +python3.11 -m piptools compile requirements.in ``` ## Local debugging diff --git a/results-processor/requirements.txt b/results-processor/requirements.txt index cb4e58454b1..38e9ad9e93d 100644 --- a/results-processor/requirements.txt +++ b/results-processor/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.9 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile requirements.in @@ -53,14 +53,12 @@ grpcio==1.53.2 # via # google-api-core # grpcio-status -grpcio-status==1.44.0 +grpcio-status==1.53.2 # via google-api-core gunicorn==23.0.0 # via -r requirements.in idna==3.7 # via requests -importlib-metadata==6.8.0 - # via flask itsdangerous==2.1.2 # via flask jinja2==3.1.4 @@ -115,5 +113,3 @@ urllib3==2.2.3 # types-requests werkzeug==3.0.6 # via flask -zipp==3.19.1 - # via importlib-metadata diff --git a/results-processor/tox.ini b/results-processor/tox.ini index b5e8a2ae228..82364cb506c 100644 --- a/results-processor/tox.ini +++ b/results-processor/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py39 +envlist = py311 # We don't have or need setup.py for now. skipsdist=True