[8.16](backport #6393) Switch back to Ubuntu 20.04-based Docker images in the cloud

changelog/fragments/1734687016-revert-cloud-image.yaml

@@ -0,0 +1,22 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: breaking-change
+
+# Change summary; a 80ish characters long description of the change.
+summary: Revert cloud Docker image to be based on Ubuntu 20.04
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+description: According to the support matrix, ECE versions up to 3.6 are marked as compatible with Docker 19.03 and the Wolfi images require Docker version of 20.10.10. The cloud image was reverted to be based on Ubuntu 20.04 to restore the compatibility.
+
+# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
+component: elastic-agent

dev-tools/packaging/packages.yml

@@ -225,11 +225,11 @@ shared:
   - &docker_ubuntu_spec
     docker_variant: 'basic'
     extra_vars:
-      from: '--platform=linux/amd64 ubuntu:24.04'
+      from: '--platform=linux/amd64 ubuntu:20.04'
   - &docker_ubuntu_arm_spec
     docker_variant: 'basic'
     extra_vars:
-      from: '--platform=linux/arm64 ubuntu:24.04'
+      from: '--platform=linux/arm64 ubuntu:20.04'
 
   - &docker_ubi_spec
     docker_variant: 'ubi'
@@ -986,15 +986,17 @@ specs:
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
-    #### ubi-based ####
+
+    #### Cloud specific docker images ####
     - os: linux
       arch: amd64
       types: [docker]
       spec:
-        <<: *docker_ubi_spec
-        <<: *docker_builder_spec
         <<: *agent_docker_spec
-        <<: *docker_elastic_spec
+        # The cloud image is always based on Ubuntu
+        <<: *docker_ubuntu_spec
+        <<: *docker_builder_spec
+        <<: *agent_docker_cloud_spec
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
@@ -1003,25 +1005,25 @@ specs:
       arch: arm64
       types: [docker]
       spec:
-        <<: *docker_ubi_arm_spec
-        <<: *docker_builder_arm_spec
         <<: *agent_docker_spec
-        <<: *docker_elastic_spec
+        # The cloud image is always based on Ubuntu
+        <<: *docker_ubuntu_arm_spec
+        <<: *docker_builder_arm_spec
+        <<: *agent_docker_cloud_spec
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
-    #### wolfi-based ####
-    #### Cloud specific docker images ####
+
+    #### ubi-based ####
     - os: linux
       arch: amd64
       types: [docker]
       spec:
-        <<: *agent_docker_spec
-        # The cloud image is always based on Wolfi
-        <<: *docker_wolfi_spec
+        <<: *docker_ubi_spec
         <<: *docker_builder_spec
-        <<: *agent_docker_cloud_spec
+        <<: *agent_docker_spec
+        <<: *docker_elastic_spec
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
@@ -1030,15 +1032,16 @@ specs:
       arch: arm64
       types: [docker]
       spec:
-        <<: *agent_docker_spec
-        # The cloud image is always based on Wolfi
-        <<: *docker_wolfi_arm_spec
+        <<: *docker_ubi_arm_spec
         <<: *docker_builder_arm_spec
-        <<: *agent_docker_cloud_spec
+        <<: *agent_docker_spec
+        <<: *docker_elastic_spec
         <<: *elastic_license_for_binaries
         files:
           '{{.BeatName}}{{.BinaryExt}}':
             source: ./build/golang-crossbuild/{{.BeatName}}-{{.GOOS}}-{{.Platform.Arch}}{{.BinaryExt}}
+
+    #### wolfi-based ####
     #### Service specific docker images ####
     - os: linux
       arch: amd64

dev-tools/packaging/templates/docker/Dockerfile.elastic-agent.tmpl

@@ -74,9 +74,6 @@ RUN for iter in {1..10}; do microdnf update -y && microdnf install -y tar gzip f
 {{- end }}
 
 {{- if contains .from "ubuntu" }}
-# Delete the existing "ubuntu" user to prevent conflict with the user that is added later.
-# See https://bugs.launchpad.net/cloud-images/+bug/2005129/comments/2.
-RUN touch /var/mail/ubuntu && chown ubuntu /var/mail/ubuntu && userdel -r ubuntu
 RUN for iter in {1..10}; do \
         apt-get update -y && \
         DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --yes ca-certificates curl gawk xz-utils && \