From 5796088cc44890152985e9d2d1f5e0304bde6927 Mon Sep 17 00:00:00 2001
From: Daniele De Lorenzi <daniele.delorenzi@sysdig.com>
Date: Mon, 24 Jun 2024 21:09:57 +0200
Subject: [PATCH] feat(probe-builder): Ability to crawl EulerOS kernel packages

Signed-off-by: Daniele De Lorenzi <daniele.delorenzi@sysdig.com>
---
 probe_builder/__init__.py                |  1 +
 probe_builder/kernel_crawler/__init__.py |  2 ++
 probe_builder/kernel_crawler/euleros.py  | 20 ++++++++++++++++++++
 probe_builder/kernel_crawler/rpm.py      |  5 ++++-
 4 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 probe_builder/kernel_crawler/euleros.py

diff --git a/probe_builder/__init__.py b/probe_builder/__init__.py
index 73017cd..aa13147 100644
--- a/probe_builder/__init__.py
+++ b/probe_builder/__init__.py
@@ -84,6 +84,7 @@ def get_kernels(self, _workspace, packages, _download_config, _crawler_filter):
     'CentOS': CrawlDistro('centos', 'centos', 'CentOS'),
     'CentOSStream': CrawlDistro('centosstream', 'centos', 'CentOSStream'),
     'Debian': CrawlDistro('debian', 'debian', 'Debian'),
+    'EulerOS': CrawlDistro('euleros', 'centos', 'EulerOS'),
     'Fedora': CrawlDistro('fedora', 'centos', 'Fedora'),
     'Flatcar': CrawlDistro('flatcar', 'flatcar', 'Flatcar'),
     'Oracle6': CrawlDistro('oracle6', 'oracle', 'Oracle6'),
diff --git a/probe_builder/kernel_crawler/__init__.py b/probe_builder/kernel_crawler/__init__.py
index 60a48c8..b836870 100644
--- a/probe_builder/kernel_crawler/__init__.py
+++ b/probe_builder/kernel_crawler/__init__.py
@@ -2,6 +2,7 @@
 from .almalinux import AlmaLinuxMirror
 from .amazonlinux import AmazonLinux1Mirror, AmazonLinux2Mirror, AmazonLinux2022Mirror
 from .centos import CentosMirror, CentosStreamMirror
+from .euleros import EulerOSMirror
 from .fedora import FedoraMirror
 from .oracle import Oracle6Mirror, Oracle7Mirror, Oracle8Mirror, Oracle9Mirror
 from .photon_os import PhotonOsMirror
@@ -20,6 +21,7 @@
     'AmazonLinux2022': AmazonLinux2022Mirror,
     'CentOS': CentosMirror,
     'CentOSStream': CentosStreamMirror,
+    'EulerOS': EulerOSMirror,
     'Fedora': FedoraMirror,
     'Oracle6': Oracle6Mirror,
     'Oracle7': Oracle7Mirror,
diff --git a/probe_builder/kernel_crawler/euleros.py b/probe_builder/kernel_crawler/euleros.py
new file mode 100644
index 0000000..f6af0b4
--- /dev/null
+++ b/probe_builder/kernel_crawler/euleros.py
@@ -0,0 +1,20 @@
+from . import repo
+from . import rpm
+
+def v2(ver):
+    return ver.startswith('2')
+
+class EulerOSMirror(repo.Distro):
+    def get_mirrors(self, crawler_filter):
+        mirrors = [
+            # EulerOS 2
+            # Lifecycle:
+            # https://developer.huaweicloud.com/intl/en-us/euleros/lifecycle-management.html
+            # Mirror list:
+            # http://mirrors.huaweicloud.com/euler/
+            rpm.RpmMirror('http://mirrors.huaweicloud.com/euler/', 'os/{}/'.format(crawler_filter.machine), v2),
+            rpm.RpmMirror('http://mirrors.huaweicloud.com/euler/', 'updates/{}/'.format(crawler_filter.machine), v2),
+            rpm.RpmMirror('http://mirrors.huaweicloud.com/euler/', '/os/{}/'.format(crawler_filter.machine)),
+
+        ]
+        return mirrors
diff --git a/probe_builder/kernel_crawler/rpm.py b/probe_builder/kernel_crawler/rpm.py
index 206f2db..d705cf5 100644
--- a/probe_builder/kernel_crawler/rpm.py
+++ b/probe_builder/kernel_crawler/rpm.py
@@ -118,7 +118,10 @@ def list_drel_repos(self, crawler_filter):
         dists.raise_for_status()
         dists = dists.content
         doc = html.fromstring(dists, self.base_url)
-        dists = doc.xpath('/html/body//a[not(@href="../")]/@href')
+        # Huawei Cloud loves do things differently
+        # Their page uses a table ouside body tags
+        # this additional filter catch it
+        dists = doc.xpath('/html/body//a[not(@href="../")]/@href | //table/tbody//a[not(@href="../")]/@href')
 
         fdists = [dist for dist in dists
                 if dist.endswith('/')