MAIN 1815 - Handle playbooks from external tar/tgz files etc (#1475)

* Handle playbooks from remote .tar.gz or .tgz 

Co-authored-by: Robert Szefler <[email protected]>
Co-authored-by: Roi Glinik <[email protected]>

docs/playbook-reference/defining-playbooks/external-playbook-repositories.rst

@@ -1,20 +1,27 @@
 Loading External Actions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Robusta can load playbook actions from external git repositories. This extends Robusta with additional actions for
-use in :ref:`customPlaybooks<customPlaybooks>`.
+Robusta can load playbook actions from external git repositories and externally hosted
+Python packages provided as .tgz or .tar.gz files. This extends Robusta with additional
+actions for use in :ref:`customPlaybooks<customPlaybooks>`.
 
 .. warning::
 
-    Robusta does not watch for changes in git repositories. Playbooks are reloaded when:
+    Robusta does not watch for changes in git repositories/externally hosted Python packages.
+    Playbooks are reloaded when:
 
     * Robusta starts
     * Robusta's configuration changes
     * ``robusta playbooks reload`` is run
 
-External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH.
+External actions are loaded using the ``playbookRepos`` Helm value, with either HTTPs or SSH
+in the case of git repositories, and appropriate URLs in the case of externally hosted
+Python packages. The way Robusta distinguishes between the case of git repository and an
+external package is to check if the URL ends with `.tgz` or `.tar.gz`
+- if that is the case, the source is treated as an external package; otherwise the
+URL is treated as a git repository address.
 
-If you are going to be using an external repository via HTTPS, you just need to configure
+If you are going to be using an external git repository via HTTPS, you just need to configure
 correct read access credentials (see below). When connecting via SSH, however, there is an
 additional requirement to verify the remote host's identity on the client side, as SSH
 generally does not provide any method of doing that automatically (in contrast with HTTPS,
@@ -109,6 +116,23 @@ The ``key`` parameter must contain a ``base64`` encoded deployment key with ``re
         ewfrcfsfvC1rZXktdjEAAAAABG5vb.....
         -----END OPENSSH PRIVATE KEY-----
 
+Loading Actions from an external Python Package
+---------------------------------------------------
+
+For external Python packages, just specify an URL starting with http(s), and ending with
+either .tar.gz or .tgz.
+
+.. code-block:: yaml
+
+    playbookRepos:
+      web_playbooks:
+        url: "https://my-domain.com/bla/web-playbooks.tgz"
+        http_headers: # optional, may be used for auth
+          Authorization: Bearer XXXYYY
+        # pip_install: true      # optional: load this playbook's dependencies (default True)
+        # build_isolation: false
+
+The `http_headers` option is only available for this method of loading actions.
 
 Handling Secrets
 *******************
@@ -131,6 +155,14 @@ Then reference it using an environment variable:
         url: "[email protected]:robusta-dev/robusta-chaos.git"
         key: "{{env.GITHUB_SSH_KEY}}"
 
+Build Isolation
+*****************
+
+``build_isolation`` is optional (defaults to `true`). If specified as `false`, the `pip`
+install command for the package being installed will be run with `--no-build-isolation` (see
+the `pip docs <https://pip.pypa.io/en/stable/cli/pip_install/#cmdoption-no-build-isolation>`_
+for details).
+
 Baking Actions into a Custom Image
 --------------------------------------
 

src/robusta/core/model/runner_config.py

@@ -1,7 +1,7 @@
 import base64
 from typing import Dict, List, Optional, Union
 
-from pydantic import BaseModel, SecretStr, validator
+from pydantic import BaseModel, SecretStr, root_validator, validator
 
 from robusta.core.playbooks.playbook_utils import get_env_replacement, replace_env_vars_values
 from robusta.core.sinks.datadog.datadog_sink_params import DataDogSinkConfigWrapper
@@ -36,6 +36,14 @@ class PlaybookRepo(BaseModel):
     key: Optional[SecretStr] = SecretStr("")
     branch: Optional[str] = None  # when url is a git repo
     pip_install: bool = True  # Set to False, if the playbooks package is already in site-packages.
+    http_headers: Optional[Dict[str, str]] = None
+    build_isolation: bool = False
+
+    @root_validator
+    def validate_pip_build_isolation(cls, values: Dict) -> Dict:
+        if values.get("build_isolation") and not values.get("pip_install"):
+            raise ValueError("build_isolation should not be specified for non-pip builds")
+        return values
 
 
 class RunnerConfig(BaseModel):

src/robusta/runner/config_loader.py

@@ -6,14 +6,17 @@
 import signal
 import subprocess
 import sys
+import tarfile
+import tempfile
 import threading
 from inspect import getmembers
 from typing import Dict, Optional
 
+import dpath.util
+import requests
+import toml
 import yaml
 
-import toml
-import dpath.util
 from robusta.core.model.env_vars import (
     CUSTOM_PLAYBOOKS_ROOT,
     DEFAULT_PLAYBOOKS_PIP_INSTALL,
@@ -116,32 +119,54 @@ def __load_playbooks_repos(
         for playbook_package, playbooks_repo in playbooks_repos.items():
             try:
                 if playbooks_repo.pip_install:  # skip playbooks that are already in site-packages
-                    if playbooks_repo.url.startswith(GIT_SSH_PREFIX) or playbooks_repo.url.startswith(GIT_HTTPS_PREFIX):
-                        repo = GitRepo(playbooks_repo.url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch)
-                        local_path = repo.repo_local_path
-                    elif playbooks_repo.url.startswith(LOCAL_PATH_URL_PREFIX):
-                        local_path = playbooks_repo.url.replace(LOCAL_PATH_URL_PREFIX, "")
+                    # Check that the config specifies an external Python package to be downloaded
+                    # and installed in Robusta.
+                    url = playbooks_repo.url
+                    if url.startswith(("https://", "http://")) and url.endswith((".tar.gz", ".tgz")):
+                        if url.startswith("http://"):
+                            logging.warning(f"Downloading a playbook package from non-https source f{url}")
+
+                        playbook_package = self.install_package_remote_tgz(
+                            url=url, headers=playbooks_repo.http_headers, build_isolation=playbooks_repo.build_isolation
+                        )
+                    elif url.startswith((GIT_SSH_PREFIX, GIT_HTTPS_PREFIX)):
+                        repo = GitRepo(url, playbooks_repo.key.get_secret_value(), playbooks_repo.branch)
+                        pkg_path = repo.repo_local_path
+                        self.install_package(
+                            pkg_path=repo.repo_local_path, build_isolation=playbooks_repo.build_isolation
+                        )
+                        playbook_package = self.__get_package_name(local_path=pkg_path)
+                    elif url.startswith(LOCAL_PATH_URL_PREFIX):
+                        pkg_path = url.replace(LOCAL_PATH_URL_PREFIX, "")
+                        self.install_package(pkg_path=pkg_path, build_isolation=playbooks_repo.build_isolation)
+                        playbook_package = self.__get_package_name(local_path=pkg_path)
                     else:
                         raise Exception(
-                            f"Illegal playbook repo url {playbooks_repo.url}. "
+                            f"Illegal playbook repo url {url}. "
                             f"Must start with '{GIT_SSH_PREFIX}', '{GIT_HTTPS_PREFIX}' or '{LOCAL_PATH_URL_PREFIX}'"
                         )
 
-                    if not os.path.exists(local_path):  # in case the repo url was defined before it was actually loaded
-                        logging.error(f"Playbooks local path {local_path} does not exist. Skipping")
-                        continue
-
-                    # Adding to pip the playbooks repo from local_path
-                    subprocess.check_call([sys.executable, "-m", "pip", "install", "--no-build-isolation", local_path])
-                    playbook_package = self.__get_package_name(local_path=local_path)
-
                 playbook_packages.append(playbook_package)
             except Exception:
                 logging.error(f"Failed to add playbooks repo {playbook_package}", exc_info=True)
 
         for package_name in playbook_packages:
             self.__import_playbooks_package(actions_registry, package_name)
 
+    @classmethod
+    def install_package(cls, pkg_path: str, build_isolation: bool) -> str:
+        logging.debug(f"Installing package {pkg_path}")
+        if not os.path.exists(pkg_path):
+            # In case the repo url was defined before it was actually loaded. Note we don't
+            # perform this check when the pkg is sourced from an externally hosted tgz etc.
+            logging.error(f"Playbook local path {pkg_path} does not exist. Skipping")
+            raise Exception(f"Playbook local path {pkg_path} does not exist. Skipping")
+
+        # Adding to pip the playbooks repo from local_path
+        extra_pip_args = ["--no-build-isolation"] if build_isolation else []
+
+        subprocess.check_call([sys.executable, "-m", "pip", "install"] + extra_pip_args + [pkg_path])
+
     @classmethod
     def __import_playbooks_package(cls, actions_registry: ActionsRegistry, package_name: str):
         logging.info(f"Importing actions package {package_name}")
@@ -292,3 +317,24 @@ def __load_runner_config(cls, config_file_path) -> Optional[RunnerConfig]:
         with open(config_file_path) as file:
             yaml_content = yaml.safe_load(file)
             return RunnerConfig(**yaml_content)
+
+    @classmethod
+    def install_package_remote_tgz(cls, url: str, headers, build_isolation: bool) -> Optional[str]:
+        with tempfile.NamedTemporaryFile(suffix=".tgz") as f:
+            r = requests.get(url, stream=True, headers=headers)
+            r.raise_for_status()
+            for chunk in r.iter_content(chunk_size=65536):
+                f.write(chunk)
+
+            f.flush()
+
+            with tarfile.open(f.name, "r:gz") as tar, tempfile.TemporaryDirectory() as temp_dir:
+                tar.extractall(path=temp_dir)
+                extracted_items = os.listdir(temp_dir)
+
+                pkg_path = temp_dir
+                if len(extracted_items) == 1:
+                    pkg_path = os.path.join(temp_dir, extracted_items[0])
+
+                cls.install_package(pkg_path=pkg_path, build_isolation=build_isolation)
+                return cls.__get_package_name(local_path=pkg_path)