From 20222460509a83b114603e0db75f0192bccd5de4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Fri, 12 Nov 2021 15:01:58 +0100 Subject: [PATCH 01/17] Fixes #2: idna version unpinned --- requirements.txt | 2 -- setup.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index 2a75951..5a91a05 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,3 @@ -# TEMPORARY, see #2: -idna==2.8 schema==0.7.1 pyyaml==5.1.2 configcrunch==0.3.7 diff --git a/setup.py b/setup.py index 307e9f7..c6ce49e 100644 --- a/setup.py +++ b/setup.py @@ -18,8 +18,6 @@ long_description_content_type='text/x-rst', url='https://github.com/Parakoopa/riptide-lib/', install_requires=[ - # TEMPORARY, see #2: - 'idna <= 2.8', 'configcrunch >= 0.3.7', 'schema >= 0.6', 'pyyaml >= 5.1', @@ -40,5 +38,6 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) From 2f8cba1c754cf088351162d5f669f265927c411c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Fri, 12 Nov 2021 15:02:18 +0100 Subject: [PATCH 02/17] Python 3.10 support --- .github/workflows/build.yml | 4 ++-- tox.ini | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d5c8b5e..415bc6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,9 +29,9 @@ jobs: run: | python -m pip install --upgrade pip - name: Build Python wheels - uses: RalfG/python-wheels-manylinux-build@v0.3.3-manylinux2010_x86_64 + uses: RalfG/python-wheels-manylinux-build@v0.3.4-manylinux2010_x86_64 with: - python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39' + python-versions: 'cp36-cp36m cp37-cp37m cp38-cp38 cp39-cp39 cp310-cp310' - name: Upload wheels uses: actions/upload-artifact@v2 with: diff --git a/tox.ini b/tox.ini index 28a775f..48bc0a7 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ # test suite on all supported python versions. To use it, "pip install tox" # and then run "tox" from this directory. [tox] -envlist = py36,py37,py38 +envlist = py36,py37,py38,py39,py310 [testenv] commands = pytest -rfs --junitxml test_reports/all.xml riptide/tests From 4ade0b62a15f99cda680713f7463dea1a503f719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Wed, 15 Dec 2021 22:09:58 +0100 Subject: [PATCH 03/17] Create dependabot.yml --- .github/workflows/dependabot.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/dependabot.yml diff --git a/.github/workflows/dependabot.yml b/.github/workflows/dependabot.yml new file mode 100644 index 0000000..cf7a39f --- /dev/null +++ b/.github/workflows/dependabot.yml @@ -0,0 +1,11 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "pip" # See documentation for possible values + directory: "/" # Location of package manifests + schedule: + interval: "daily" From 44490dabdba9ae3f7159bfec54d0f7b7347c03c3 Mon Sep 17 00:00:00 2001 From: Parakoopa Date: Wed, 15 Dec 2021 22:11:30 +0100 Subject: [PATCH 04/17] moved dependabot file to correct path --- .github/{workflows => }/dependabot.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => }/dependabot.yml (100%) diff --git a/.github/workflows/dependabot.yml b/.github/dependabot.yml similarity index 100% rename from .github/workflows/dependabot.yml rename to .github/dependabot.yml From 6d9c89fc4e36506b3beb4eebc3e26cce32c149c5 Mon Sep 17 00:00:00 2001 From: Parakoopa Date: Wed, 15 Dec 2021 22:22:21 +0100 Subject: [PATCH 05/17] Update janus to 0.7+ --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 5a91a05..cfbf6eb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ schema==0.7.1 pyyaml==5.1.2 configcrunch==0.3.7 appdirs==1.4.3 -janus==0.4.0 +janus==0.7.0 psutil==5.6.6 GitPython==3.0.8 pywinpty==0.5.5; sys_platform == 'win32' diff --git a/setup.py b/setup.py index c6ce49e..7bd7d8e 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ 'schema >= 0.6', 'pyyaml >= 5.1', 'appdirs >= 1.4', - 'janus >= 0.4.0', + 'janus >= 0.7.0', 'psutil >= 5.6', 'GitPython >= 3.0', 'pywinpty >= 0.5.5; sys_platform == "win32"', From 9adf13cf7a018d785a9a9d072b6edfe923f860bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Dec 2021 21:32:02 +0000 Subject: [PATCH 06/17] Bump gitpython from 3.0.8 to 3.1.24 Bumps [gitpython](https://github.com/gitpython-developers/GitPython) from 3.0.8 to 3.1.24. - [Release notes](https://github.com/gitpython-developers/GitPython/releases) - [Changelog](https://github.com/gitpython-developers/GitPython/blob/main/CHANGES) - [Commits](https://github.com/gitpython-developers/GitPython/compare/3.0.8...3.1.24) --- updated-dependencies: - dependency-name: gitpython dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cfbf6eb..69c82b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ configcrunch==0.3.7 appdirs==1.4.3 janus==0.7.0 psutil==5.6.6 -GitPython==3.0.8 +GitPython==3.1.24 pywinpty==0.5.5; sys_platform == 'win32' python-hosts==0.4.7 python-dotenv==0.19.0 From 0829117e138308c164044b7d867bfebfdf4c0ee9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Dec 2021 21:32:40 +0000 Subject: [PATCH 07/17] Bump schema from 0.7.1 to 0.7.5 Bumps [schema](https://github.com/keleshev/schema) from 0.7.1 to 0.7.5. - [Release notes](https://github.com/keleshev/schema/releases) - [Changelog](https://github.com/keleshev/schema/blob/master/CHANGELOG.md) - [Commits](https://github.com/keleshev/schema/compare/v0.7.1...v0.7.5) --- updated-dependencies: - dependency-name: schema dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cfbf6eb..00896c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -schema==0.7.1 +schema==0.7.5 pyyaml==5.1.2 configcrunch==0.3.7 appdirs==1.4.3 From dcc0e7d8ad0f9ad9720c3357d28d246ec71e48f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 16:35:17 +0000 Subject: [PATCH 08/17] Bump pyyaml from 5.1.2 to 6.0 Bumps [pyyaml](https://github.com/yaml/pyyaml) from 5.1.2 to 6.0. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/master/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/5.1.2...6.0) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aaa3682..8ddd3b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ schema==0.7.5 -pyyaml==5.1.2 +pyyaml==5.4 configcrunch==0.3.7 appdirs==1.4.3 janus==0.7.0 From ee3d1aa4eedb8aebffda42dc269f9eb3987a50de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 16:35:24 +0000 Subject: [PATCH 09/17] Bump appdirs from 1.4.3 to 1.4.4 Bumps [appdirs](https://github.com/ActiveState/appdirs) from 1.4.3 to 1.4.4. - [Release notes](https://github.com/ActiveState/appdirs/releases) - [Changelog](https://github.com/ActiveState/appdirs/blob/master/CHANGES.rst) - [Commits](https://github.com/ActiveState/appdirs/compare/1.4.3...1.4.4) --- updated-dependencies: - dependency-name: appdirs dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aaa3682..8101569 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ schema==0.7.5 pyyaml==5.1.2 configcrunch==0.3.7 -appdirs==1.4.3 +appdirs==1.4.4 janus==0.7.0 psutil==5.6.6 GitPython==3.1.24 From 111778005a0cddcbe93d7a4e2b438a6b39a411d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 16:35:28 +0000 Subject: [PATCH 10/17] Bump psutil from 5.6.6 to 5.8.0 Bumps [psutil](https://github.com/giampaolo/psutil) from 5.6.6 to 5.8.0. - [Release notes](https://github.com/giampaolo/psutil/releases) - [Changelog](https://github.com/giampaolo/psutil/blob/master/HISTORY.rst) - [Commits](https://github.com/giampaolo/psutil/compare/release-5.6.6...release-5.8.0) --- updated-dependencies: - dependency-name: psutil dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aaa3682..b10f4a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ pyyaml==5.1.2 configcrunch==0.3.7 appdirs==1.4.3 janus==0.7.0 -psutil==5.6.6 +psutil==5.8.0 GitPython==3.1.24 pywinpty==0.5.5; sys_platform == 'win32' python-hosts==0.4.7 From 18eb3f34485220e042f7b950733fa3a7a4d0373a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 16:37:30 +0000 Subject: [PATCH 11/17] Bump python-dotenv from 0.19.0 to 0.19.2 Bumps [python-dotenv](https://github.com/theskumar/python-dotenv) from 0.19.0 to 0.19.2. - [Release notes](https://github.com/theskumar/python-dotenv/releases) - [Changelog](https://github.com/theskumar/python-dotenv/blob/master/CHANGELOG.md) - [Commits](https://github.com/theskumar/python-dotenv/compare/v0.19.0...v0.19.2) --- updated-dependencies: - dependency-name: python-dotenv dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61b9494..0d65c20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,4 +7,4 @@ psutil==5.8.0 GitPython==3.1.24 pywinpty==0.5.5; sys_platform == 'win32' python-hosts==0.4.7 -python-dotenv==0.19.0 +python-dotenv==0.19.2 From edc78b81cd3c91986c6f32ff6a2a31d724a64877 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Mon, 13 Dec 2021 23:12:14 +0100 Subject: [PATCH 12/17] Compatibility with Configcrunch 1.0.0 --- requirements.txt | 2 +- riptide/config/command/in_service.py | 6 +- riptide/config/document/app.py | 54 +++----- riptide/config/document/command.py | 15 +-- .../config/document/common_service_command.py | 4 + riptide/config/document/config.py | 32 ++--- riptide/config/document/project.py | 34 +++--- riptide/config/document/service.py | 115 +++++++++--------- riptide/config/loader.py | 29 ++--- riptide/config/repositories.py | 2 +- riptide/config/service/config_files.py | 58 +++++---- riptide/db/driver/db_driver_for_service.py | 10 +- run_tests.sh | 8 +- setup.py | 2 +- test_assets/riptide-docker-tox/Dockerfile | 5 +- 15 files changed, 180 insertions(+), 196 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0d65c20..d45f04a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ schema==0.7.5 pyyaml==5.4 -configcrunch==0.3.7 +configcrunch==1.0.0 appdirs==1.4.4 janus==0.7.0 psutil==5.8.0 diff --git a/riptide/config/command/in_service.py b/riptide/config/command/in_service.py index 81de51f..67f452c 100644 --- a/riptide/config/command/in_service.py +++ b/riptide/config/command/in_service.py @@ -19,14 +19,16 @@ def convert_in_service_to_normal(app: App, command_name: str) -> Command: env = {} env.update(service['environment'] if 'environment' in service else {}) env.update(old_cmd['environment'] if 'environment' in old_cmd else {}) - new_cmd = Command({ + new_cmd = Command.from_dict({ '$name': command_name, 'image': service['image'], 'command': old_cmd['command'], 'additional_volumes': service['additional_volumes'] if 'additional_volumes' in service else {}, 'environment': env, 'config_from_roles': [old_cmd['in_service_with_role']] - }, parent=app) + }) + new_cmd.parent_doc = app + new_cmd.freeze() return new_cmd diff --git a/riptide/config/document/app.py b/riptide/config/document/app.py index 35f70d2..d326c02 100644 --- a/riptide/config/document/app.py +++ b/riptide/config/document/app.py @@ -1,9 +1,8 @@ from schema import Optional, Schema, Or -from typing import List, Union, TYPE_CHECKING +from typing import List, Union, TYPE_CHECKING, Tuple, Type from configcrunch import YamlConfigDocument, DocReference, ConfigcrunchError, REMOVE -from configcrunch import load_subdocument -from configcrunch.abstract import variable_helper +from configcrunch import variable_helper from riptide.config.document.command import Command from riptide.config.document.service import Service @@ -120,6 +119,13 @@ def schema(cls) -> Schema: } ) + @classmethod + def subdocuments(cls) -> List[Tuple[str, Type[YamlConfigDocument]]]: + return [ + ("services[]", Service), + ("commands[]", Command), + ] + def validate(self): """ Initialise the optional services and command dicts. @@ -127,38 +133,14 @@ def validate(self): """ ret_val = super().validate() if ret_val: - if "services" not in self: - self.doc["services"] = {} - - if "commands" not in self: - self.doc["commands"] = {} + if not self.internal_contains("services"): + self.internal_set("services", {}) + if not self.internal_contains("commands"): + self.internal_set("commands", {}) return ret_val - def _load_subdocuments(self, lookup_paths: List[str]): - if "services" in self and self["services"] != REMOVE: - for key, servicedoc in self["services"].items(): - if servicedoc != REMOVE: - self["services"][key] = load_subdocument(servicedoc, self, Service, lookup_paths) - if not isinstance(self["services"][key].doc, dict): - raise ConfigcrunchError( - f"Error loading Service for App: The service with the name {key} needs to be an object in the source document." - ) - self["services"][key]["$name"] = key - - if "commands" in self and self["commands"] != REMOVE: - for key, commanddoc in self["commands"].items(): - if commanddoc != REMOVE: - self["commands"][key] = load_subdocument(commanddoc, self, Command, lookup_paths) - if not isinstance(self["commands"][key].doc, dict): - raise ConfigcrunchError( - f"Error loading Command for App: The command with the name {key} needs to be an object in the source document." - ) - self["commands"][key]["$name"] = key - - return self - def error_str(self) -> str: - return f"{self.__class__.__name__}<{(self['name'] if 'name' in self else '???')}>" + return f"{self.__class__.__name__}<{(self.internal_get('name') if self.internal_contains('name') else '???')}>" @variable_helper def parent(self) -> 'Project': @@ -191,8 +173,8 @@ def get_service_by_role(self, role_name: str) -> Union[Service, None]: :param role_name: Role to search for """ - for service in self["services"].values(): - if "roles" in service and role_name in service["roles"]: + for service in self.internal_get("services").values(): + if service.internal_contains("roles") and role_name in service.internal_get("roles"): return service raise ValueError(f"No service with role {role_name} found in the app.") @@ -204,7 +186,7 @@ def get_services_by_role(self, role_name: str) -> List[Service]: :param role_name: Role to search for """ services = [] - for service in self["services"].values(): - if "roles" in service and role_name in service["roles"]: + for service in self.internal_get("services").values(): + if service.internal_contains("roles") and role_name in service.internal_get("roles"): services.append(service) return services diff --git a/riptide/config/document/command.py b/riptide/config/document/command.py index b4445c1..7097ba8 100644 --- a/riptide/config/document/command.py +++ b/riptide/config/document/command.py @@ -7,7 +7,7 @@ from schema import Schema, Optional, Or from typing import TYPE_CHECKING, Union -from configcrunch.abstract import variable_helper +from configcrunch import variable_helper from riptide.config.document.common_service_command import ContainerDefinitionYamlConfigDocument from riptide.config.files import get_project_meta_folder, CONTAINER_SRC_PATH from riptide.config.service.config_files import process_config @@ -196,14 +196,15 @@ def schema_alias(cls): 'aliases': str }) - def _initialize_data_after_variables(self): + def _initialize_data_after_variables(self, data: dict) -> dict: """ Normalize all host-paths to only use the system-type directory separator """ - if "additional_volumes" in self: - for obj in self.doc["additional_volumes"].values(): + if "additional_volumes" in data: + for obj in data["additional_volumes"].values(): obj["host"] = cppath.normalize(obj["host"]) if "read_env_file" not in self: - self.doc["read_env_file"] = True + data["read_env_file"] = True + return data def get_project(self) -> 'Project': """ @@ -344,7 +345,7 @@ def get_service(self, app: 'App') -> Union[str, None]: f"No service with this role found in the app.") def error_str(self) -> str: - return f"{self.__class__.__name__}<{(self['$name'] if '$name' in self else '???')}>" + return f"{self.__class__.__name__}<{(self.internal_get('$name') if self.internal_contains('$name') else '???')}>" @variable_helper def parent(self) -> 'App': @@ -381,6 +382,6 @@ def volume_path(self) -> str: host: '/home/peter/my_projects/project1/_riptide/cmd_data/command_name/command_cache' container: '/foo/bar/cache' """ - path = os.path.join(get_project_meta_folder(self.get_project().folder()), 'cmd_data', self["$name"]) + path = os.path.join(get_project_meta_folder(self.get_project().folder()), 'cmd_data', self.internal_get("$name")) os.makedirs(path, exist_ok=True) return path diff --git a/riptide/config/document/common_service_command.py b/riptide/config/document/common_service_command.py index 9f76bfc..d00d8bd 100644 --- a/riptide/config/document/common_service_command.py +++ b/riptide/config/document/common_service_command.py @@ -13,6 +13,10 @@ class ContainerDefinitionYamlConfigDocument(YamlConfigDocument, ABC): + @classmethod + def subdocuments(cls): + return [] + @variable_helper def system_config(self) -> 'Config': """ diff --git a/riptide/config/document/config.py b/riptide/config/document/config.py index 0f0f052..6cc5c2c 100644 --- a/riptide/config/document/config.py +++ b/riptide/config/document/config.py @@ -155,10 +155,11 @@ def schema(cls) -> Schema: } ) - def _load_subdocuments(self, lookup_paths: List[str]): + @classmethod + def subdocuments(cls): # Can not contain references to other documents other than # the "project" reference which is added by the system. - return self + return [] def error_str(self) -> str: return "System Configuration" @@ -197,19 +198,20 @@ def get_plugin_flag(self, inp: str) -> any: def upgrade(self): """Update the system configuration file after Riptide version upgrades. To be run before validation.""" changed = False - if "performance" not in self.doc: - self.doc["performance"] = {} - changed = True - if "dont_sync_named_volumes_with_host" not in self.doc["performance"]: - self.doc["performance"]["dont_sync_named_volumes_with_host"] = "auto" - changed = True - if "dont_sync_unimportant_src" not in self.doc["performance"]: - self.doc["performance"]["dont_sync_unimportant_src"] = "auto" - changed = True - - if changed: - with open(riptide_main_config_file(), "w") as f: - f.write(yaml.dump(self.to_dict(), default_flow_style=False, sort_keys=False)) + with self.internal_access(): + if "performance" not in self.doc: + self.doc["performance"] = {} + changed = True + if "dont_sync_named_volumes_with_host" not in self.doc["performance"]: + self.doc["performance"]["dont_sync_named_volumes_with_host"] = "auto" + changed = True + if "dont_sync_unimportant_src" not in self.doc["performance"]: + self.doc["performance"]["dont_sync_unimportant_src"] = "auto" + changed = True + + if changed: + with open(riptide_main_config_file(), "w") as f: + f.write(yaml.dump(self.to_dict(), default_flow_style=False, sort_keys=False)) def load_performance_options(self, engine: 'AbstractEngine'): """Initializes performance options set to 'auto' based on the engine used.""" diff --git a/riptide/config/document/project.py b/riptide/config/document/project.py index 71c74c2..15e06b0 100644 --- a/riptide/config/document/project.py +++ b/riptide/config/document/project.py @@ -5,7 +5,6 @@ from schema import Schema, Optional from configcrunch import YamlConfigDocument, DocReference, ConfigcrunchError, variable_helper, REMOVE -from configcrunch import load_subdocument from riptide.config.document.app import App HEADER = 'project' @@ -75,38 +74,37 @@ def schema(cls) -> Schema: } ) + @classmethod + def subdocuments(cls): + return [ + ("app", App) + ] + def validate(self) -> bool: r = super().validate() - if '_' in self['name']: + if '_' in self.internal_get('name'): raise ValueError("Project name is invalid: Must not contain underscores (_).") return r - def _load_subdocuments(self, lookup_paths: List[str]): - if "app" in self and self["app"] != REMOVE: - self["app"] = load_subdocument(self["app"], self, App, lookup_paths) - if not isinstance(self["app"].doc, dict): - raise ConfigcrunchError("Error loading App for Project: " - "The app needs to be an object in the source document.") - return self - - def _initialize_data_after_merge(self): - if 'links' not in self.doc: - self.doc['links'] = [] + def _initialize_data_after_merge(self, data): + if 'links' not in data: + data['links'] = [] + return data def folder(self): """Returns the project folder if the special internal field "$path" if set or None otherwise""" - if "$path" in self: - return os.path.dirname(self["$path"]) + if self.internal_contains("$path"): + return os.path.dirname(self.internal_get("$path")) return None def src_folder(self): """Returns the absolute path to the folder specified by self['src']. Requires "$path" to be set.""" - if "$path" not in self: + if not self.internal_contains("$path"): return None - return os.path.join(self.folder(), self["src"]) + return os.path.join(self.folder(), self.internal_get("src")) def error_str(self) -> str: - return f"{self.__class__.__name__}<{(self['name'] if 'name' in self else '???')}>" + return f"{self.__class__.__name__}<{(self.internal_get('name') if self.internal_contains('name') else '???')}>" @variable_helper def parent(self) -> 'Config': diff --git a/riptide/config/document/service.py b/riptide/config/document/service.py index 9c58503..9256934 100644 --- a/riptide/config/document/service.py +++ b/riptide/config/document/service.py @@ -7,7 +7,7 @@ from schema import Schema, Optional, Or from configcrunch import YamlConfigDocument, ConfigcrunchError -from configcrunch.abstract import variable_helper +from configcrunch import variable_helper from riptide.config.document.common_service_command import ContainerDefinitionYamlConfigDocument from riptide.config.errors import RiptideDeprecationWarning from riptide.config.files import CONTAINER_SRC_PATH @@ -39,17 +39,6 @@ class Service(ContainerDefinitionYamlConfigDocument): the service with the ``$name`` entry during runtime. """ - def __init__( - self, - document: dict, - path: str = None, - parent: 'YamlConfigDocument' = None, - already_loaded_docs: List[str] = None, - absolute_paths=None - ): - self._db_driver = None - self._loaded_port_mappings = None - super().__init__(document, path, parent, already_loaded_docs, absolute_paths) @classmethod def header(cls) -> str: @@ -368,51 +357,54 @@ def schema(cls) -> Schema: } ) - def _initialize_data_after_merge(self): + def _initialize_data_after_merge(self, data): """ Initializes non-set fields, initiliazes the database driver and creates all files for ``config`` entries. """ - if "run_as_root" in self: + self._db_driver = None + self._loaded_port_mappings = None + + if "run_as_root" in data: warnings.warn( "Deprecated key run_as_root = %r in a service found. Please replace with run_as_current_user = %r." % - (self.doc["run_as_root"], not self.doc["run_as_root"]), + (data["run_as_root"], not data["run_as_root"]), RiptideDeprecationWarning ) - self.doc["run_as_current_user"] = not self.doc["run_as_root"] - if "run_as_current_user" not in self: - self.doc["run_as_current_user"] = True - if "run_pre_start_as_current_user" not in self or self.doc["run_pre_start_as_current_user"] == "auto": - self.doc["run_pre_start_as_current_user"] = self.doc["run_as_current_user"] - if "run_post_start_as_current_user" not in self or self.doc["run_post_start_as_current_user"] == "auto": - self.doc["run_post_start_as_current_user"] = self.doc["run_as_current_user"] + data["run_as_current_user"] = not data["run_as_root"] + if "run_as_current_user" not in data: + data["run_as_current_user"] = True + if "run_pre_start_as_current_user" not in data or data["run_pre_start_as_current_user"] == "auto": + data["run_pre_start_as_current_user"] = data["run_as_current_user"] + if "run_post_start_as_current_user" not in data or data["run_post_start_as_current_user"] == "auto": + data["run_post_start_as_current_user"] = data["run_as_current_user"] - if "dont_create_user" not in self: - self.doc["dont_create_user"] = False + if "dont_create_user" not in data: + data["dont_create_user"] = False - if "pre_start" not in self: - self.doc["pre_start"] = [] + if "pre_start" not in data: + data["pre_start"] = [] - if "post_start" not in self: - self.doc["post_start"] = [] + if "post_start" not in data: + data["post_start"] = [] - if "roles" not in self: - self.doc["roles"] = [] + if "roles" not in data: + data["roles"] = [] - if "working_directory" not in self: - self.doc["working_directory"] = "." + if "working_directory" not in data: + data["working_directory"] = "." - if "read_env_file" not in self: - self.doc["read_env_file"] = True + if "read_env_file" not in data: + data["read_env_file"] = True - if "db" in self["roles"]: - self._db_driver = db_driver_for_service.get(self) + if "db" in data["roles"]: + self._db_driver = db_driver_for_service.get(data, self) if self._db_driver: # Collect additional ports for the db driver - my_original_ports = self["additional_ports"] if "additional_ports" in self else {} + my_original_ports = data["additional_ports"] if "additional_ports" in data else {} db_ports = self._db_driver.collect_additional_ports() - self["additional_ports"] = db_ports.copy() - self["additional_ports"].update(my_original_ports) + data["additional_ports"] = db_ports.copy() + data["additional_ports"].update(my_original_ports) # Load the absolute path of the config documents specified in config[]["from"] if self.absolute_paths: @@ -424,8 +416,8 @@ def _initialize_data_after_merge(self): # Fallback: Assume cwd folders_to_search = [os.getcwd()] - if "config" in self and isinstance(self["config"], dict): - for config in self["config"].values(): + if "config" in data and isinstance(data["config"], dict): + for config in data["config"].values(): # sanity check if from and to are in this config entry, if not it's invalid. # the validation will catch this later if "from" not in config or "to" not in config: @@ -450,17 +442,19 @@ def _initialize_data_after_merge(self): f"entries. Based on how the configuration was merged, the following places were searched: " f"{str(folders_to_search)}" ) + return data - def _initialize_data_after_variables(self): + def _initialize_data_after_variables(self, data): """ Normalizes all host-paths to only use the system-type directory separator. """ - if "additional_volumes" in self: - for obj in self.doc["additional_volumes"].values(): + if "additional_volumes" in data: + for obj in data["additional_volumes"].values(): obj["host"] = cppath.normalize(obj["host"]) - if "config" in self: - for obj in self.doc["config"].values(): + if "config" in data: + for obj in data["config"].values(): obj["$source"] = cppath.normalize(obj["$source"]) + return data def validate(self) -> bool: """ Validates the Schema and if a database driver is defined, validates that the driver is installed. """ @@ -468,14 +462,15 @@ def validate(self) -> bool: return False # Db Driver constraints. If role db is set, a "driver" has to be set and code has to exist for it. - if "roles" in self and "db" in self["roles"]: - if "driver" not in self or self._db_driver is None: + if self.internal_contains("roles") and "db" in self.internal_get("roles"): + if not self.internal_contains("driver") or self._db_driver is None: raise ConfigcrunchError( - f"Service {self['$name']} validation: " + f"Service {self.internal_get('$name')} validation: " f"If a service has the role 'db' it has to have a valid " f"'driver' entry with a driver that is available." ) - self._db_driver.validate_service() + with self.internal_access(): + self._db_driver.validate_service() return True def before_start(self): @@ -618,7 +613,7 @@ def collect_ports(self) -> dict: return self._loaded_port_mappings def error_str(self) -> str: - return f"{self.__class__.__name__}<{(self['$name'] if '$name' in self else '???')}>" + return f"{self.__class__.__name__}<{(self.internal_get('$name') if self.internal_contains('$name') else '???')}>" @variable_helper def parent(self) -> 'App': @@ -655,7 +650,7 @@ def volume_path(self) -> str: host: '/home/peter/my_projects/project1/_riptide/data/service_name/cache' container: '/foo/bar/cache' """ - path = os.path.join(get_project_meta_folder(self.get_project().folder()), 'data', self["$name"]) + path = os.path.join(get_project_meta_folder(self.get_project().folder()), 'data', self.internal_get("$name")) return path @variable_helper @@ -673,12 +668,12 @@ def get_working_directory(self) -> str: something: '/src/working_dir' """ - workdir = None if "src" not in self["roles"] else CONTAINER_SRC_PATH - if "working_directory" in self: - if PurePosixPath(self["working_directory"]).is_absolute(): - return self["working_directory"] + workdir = None if "src" not in self.internal_get("roles") else CONTAINER_SRC_PATH + if self.internal_contains("working_directory"): + if PurePosixPath(self.internal_get("working_directory")).is_absolute(): + return self.internal_get("working_directory") elif workdir is not None: - return str(PurePosixPath(workdir).joinpath(self["working_directory"])) + return str(PurePosixPath(workdir).joinpath(self.internal_get("working_directory"))) return workdir @variable_helper @@ -695,6 +690,6 @@ def domain(self) -> str: something: 'https://project--service.riptide.local' """ - if "main" in self["roles"]: - return self.get_project()["name"] + "." + self.parent_doc.parent_doc.parent_doc["proxy"]["url"] - return self.get_project()["name"] + DOMAIN_PROJECT_SERVICE_SEP + self["$name"] + "." + self.parent_doc.parent_doc.parent_doc["proxy"]["url"] + if "main" in self.internal_get("roles"): + return self.get_project().internal_get("name") + "." + self.parent_doc.parent_doc.parent_doc.internal_get("proxy")["url"] + return self.get_project().internal_get("name") + DOMAIN_PROJECT_SERVICE_SEP + self.internal_get("$name") + "." + self.parent_doc.parent_doc.parent_doc.internal_get("proxy")["url"] diff --git a/riptide/config/loader.py b/riptide/config/loader.py index 07ca2bc..ad48124 100644 --- a/riptide/config/loader.py +++ b/riptide/config/loader.py @@ -6,7 +6,7 @@ from collections import OrderedDict from typing import TYPE_CHECKING -from configcrunch.advanced_loader import load_multiple_yml +from configcrunch import load_multiple_yml from riptide.config import repositories from riptide.config.document.config import Config from riptide.config.document.project import Project @@ -60,8 +60,8 @@ def load_config(project_file=None, skip_project_load=False, enable_local_project system_config = Config.from_yaml(config_path) # The user is not allowed to add a project entry to their main config file - if "project" in system_config: - del system_config["project"] + if system_config.internal_contains("project"): + system_config.internal_delete("project") system_config.upgrade() system_config.validate() @@ -76,11 +76,11 @@ def load_config(project_file=None, skip_project_load=False, enable_local_project project_config = load_multiple_yml(Project, project_path, local_project_path) else: project_config = load_multiple_yml(Project, project_path) - project_config["$path"] = project_path + project_config.internal_set("$path", project_path) project_config.resolve_and_merge_references(repos) - system_config["project"] = project_config + system_config.internal_set("project", project_config) project_config.parent_doc = system_config except FileNotFoundError: pass @@ -88,6 +88,7 @@ def load_config(project_file=None, skip_project_load=False, enable_local_project system_config.process_vars() system_config.validate() + system_config.freeze() for plugin in load_plugins().values(): plugin.after_reload_config(system_config) @@ -139,9 +140,9 @@ def write_project(project: 'Project', rename=False): """ # Check reserved names - if project['name'] in RESERVED_NAMES: + if project.internal_get('name') in RESERVED_NAMES: raise FileExistsError( - f'The project name {project["name"]} is reserved by Riptide. ' + f'The project name {project.internal_get("name")} is reserved by Riptide. ' f'Please use a different name for your project.' ) @@ -152,25 +153,25 @@ def write_project(project: 'Project', rename=False): # need to do anything. If not and rename is not passed, thrown an error, if # rename is passed or if the path for the project didn't exist yet: Write it to the file. changed = True - if project['name'] in projects: + if project.internal_get('name') in projects: changed = False - if projects[project['name']] != project['$path']: + if projects[project.internal_get('name')] != project.internal_get('$path'): changed = True if not rename: raise FileExistsError( - f'The Riptide project named {project["name"]} is already located at ' - f'{projects[project["name"]]} but your current project file is at {project["$path"]}.\n' - f'Each project name can only be mapped to one path. If you want to "rename" {project["name"]} to use ' + f'The Riptide project named {project.internal_get("name")} is already located at ' + f'{projects[project.internal_get("name")]} but your current project file is at {project.internal_get("$path")}.\n' + f'Each project name can only be mapped to one path. If you want to "rename" {project.internal_get("name")} to use ' f'this new path, pass the --rename flag, otherwise rename the project in the riptide.yml file.\n' f'If you want to edit these mappings manually, have a look at the file {riptide_projects_file()}.' ) if changed: - projects[project['name']] = project['$path'] + projects[project.internal_get("name")] = project.internal_get("$path") with open(riptide_projects_file(), mode='w') as file: json.dump(projects, file) if rename: print("Project reference renamed.") - print(f"{project['name']} -> {projects[project['name']]}") + print(f"{project.internal_get('name')} -> {projects[project.internal_get('name')]}") exit(0) diff --git a/riptide/config/repositories.py b/riptide/config/repositories.py index 8653fbf..38383ee 100644 --- a/riptide/config/repositories.py +++ b/riptide/config/repositories.py @@ -67,7 +67,7 @@ def collect(system_config): # Get all repos that are downloaded repos = set(next(os.walk(base_dir))[1]) # Get all expected repositories, clean up the names to match the directory names - repos_in_system_config = [remove_all_special_chars(repo) for repo in system_config["repos"]] + repos_in_system_config = [remove_all_special_chars(repo) for repo in system_config.internal_get("repos")] # Get all repos that are downloaded, but not in the system config to_remove = repos - set(repos_in_system_config) diff --git a/riptide/config/service/config_files.py b/riptide/config/service/config_files.py index 6440a0d..154d173 100644 --- a/riptide/config/service/config_files.py +++ b/riptide/config/service/config_files.py @@ -53,36 +53,34 @@ def process_config( is_in_source_path = bind_path.startswith('/src/') and 'roles' in service and 'src' in service['roles'] target_file = get_config_file_path(config_name, service, is_in_source_path, bind_path) - if not regenerate and os.path.exists(target_file): - return - - # Additional helper functions - read_file_partial = partial(read_file, config["$source"]) - read_file_partial.__name__ = read_file.__name__ - - with open(config["$source"], 'r') as stream: - processed_file = service.process_vars_for(stream.read(), [ - read_file_partial - ]) - - if is_in_source_path: - notice_file = target_file + '.riptide_info.txt' - if not os.path.isfile(notice_file): - with open(notice_file, 'w') as f: - f.writelines([ - f'The file {os.path.basename(target_file)} was created by Riptide. Do not modify it.\n' - f'It will automatically be re-generated if you restart the project.\n' - f'Please add this file and {os.path.basename(target_file)} to the ignore file of your VCS.\n\n' - f'The {os.path.basename(target_file)} is based on a template file, which you can find here:\n' - f' {config["$source"]}\n\n' - f'Please have a look at the documentation if you want to use a different template file (Schema for ' - f'Services, entry "config").' - ]) - - os.makedirs(os.path.dirname(target_file), exist_ok=True) - - with open(target_file, 'w') as f: - f.write(processed_file) + if regenerate or not os.path.exists(target_file): + # Additional helper functions + read_file_partial = partial(read_file, config["$source"]) + read_file_partial.__name__ = read_file.__name__ + + with open(config["$source"], 'r') as stream: + processed_file = service.process_vars_for(stream.read(), [ + read_file_partial + ]) + + if is_in_source_path: + notice_file = target_file + '.riptide_info.txt' + if not os.path.isfile(notice_file): + with open(notice_file, 'w') as f: + f.writelines([ + f'The file {os.path.basename(target_file)} was created by Riptide. Do not modify it.\n' + f'It will automatically be re-generated if you restart the project.\n' + f'Please add this file and {os.path.basename(target_file)} to the ignore file of your VCS.\n\n' + f'The {os.path.basename(target_file)} is based on a template file, which you can find here:\n' + f' {config["$source"]}\n\n' + f'Please have a look at the documentation if you want to use a different template file (Schema for ' + f'Services, entry "config").' + ]) + + os.makedirs(os.path.dirname(target_file), exist_ok=True) + + with open(target_file, 'w') as f: + f.write(processed_file) # Only add a volume if needed if not is_in_source_path: diff --git a/riptide/db/driver/db_driver_for_service.py b/riptide/db/driver/db_driver_for_service.py index 69f0416..c800ee6 100644 --- a/riptide/db/driver/db_driver_for_service.py +++ b/riptide/db/driver/db_driver_for_service.py @@ -1,15 +1,17 @@ """Module to resolve database drivers for services""" -from typing import Union +from typing import Union, TYPE_CHECKING import pkg_resources from riptide.db.driver.abstract import AbstractDbDriver +if TYPE_CHECKING: + from riptide.config.document.service import Service DB_DRIVER_ENTRYPOINT_KEY = 'riptide.db_driver' -def get(service: 'Service') -> Union[AbstractDbDriver, None]: +def get(service_data: dict, service: 'Service') -> Union[AbstractDbDriver, None]: """Returns the db driver instance for this service, if a driver is defined.""" # Look up package entrypoints for db drivers drivers = { @@ -17,6 +19,6 @@ def get(service: 'Service') -> Union[AbstractDbDriver, None]: entry_point.load() for entry_point in pkg_resources.iter_entry_points(DB_DRIVER_ENTRYPOINT_KEY) } - if service["driver"]["name"] in drivers: - return drivers[service["driver"]["name"]](service) + if service_data["driver"]["name"] in drivers: + return drivers[service_data["driver"]["name"]](service) return None diff --git a/run_tests.sh b/run_tests.sh index 396bcf3..989460e 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -3,7 +3,7 @@ ## RUN TESTS (Linux only) # # This will run all tests in this project, including the engine integration tests with the Docker engine backend. -# The tests are run on Python versions 3.6 - 3.7. +# The tests are run on Python versions 3.6 - 3.x. # See tox.ini for details :) # # For these tests to run, you need to have Docker installed. The tests will use a Docker image found in @@ -20,10 +20,10 @@ # # 0. Build the integration test image... -docker build -t riptide_integration_test riptide/tests/docker_image +docker buildx build -t riptide_integration_test riptide/tests/docker_image # 1. Build the runner image... -docker build -t riptide_docker_tox test_assets/riptide-docker-tox +docker buildx build -t riptide_docker_tox test_assets/riptide-docker-tox # 2. Run the image... docker run \ @@ -37,4 +37,4 @@ docker run \ --network host \ --workdir $(pwd) \ riptide_docker_tox \ - "tox" \ No newline at end of file + "tox" diff --git a/setup.py b/setup.py index 7bd7d8e..7bcffe4 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ long_description_content_type='text/x-rst', url='https://github.com/Parakoopa/riptide-lib/', install_requires=[ - 'configcrunch >= 0.3.7', + 'configcrunch >= 1.0.0', 'schema >= 0.6', 'pyyaml >= 5.1', 'appdirs >= 1.4', diff --git a/test_assets/riptide-docker-tox/Dockerfile b/test_assets/riptide-docker-tox/Dockerfile index f573285..5e03523 100644 --- a/test_assets/riptide-docker-tox/Dockerfile +++ b/test_assets/riptide-docker-tox/Dockerfile @@ -3,7 +3,6 @@ # To be tagged as: riptide_docker_tox FROM ubuntu:18.04 -MAINTAINER devteam@level12.io RUN apt-get clean && apt-get update && apt-get install -y gnupg2 locales && locale-gen en_US.UTF-8 ENV LANG en_US.UTF-8 @@ -19,15 +18,15 @@ RUN export DEBIAN_FRONTEND=noninteractive \ && apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 \ && apt-get update -q \ && apt-get install -y curl git mercurial \ - python3.6 python3.6-dev libpython3.6-dev libpython3.6-stdlib \ python3.7 python3.7-dev libpython3.7-dev libpython3.7-stdlib \ python3.8 python3.8-dev libpython3.8-dev libpython3.8-stdlib python3.8-distutils \ python3.9 python3.9-dev libpython3.9-dev libpython3.9-stdlib python3.9-distutils \ + python3.10 python3.10-dev libpython3.10-dev libpython3.10-stdlib python3.10-distutils \ && curl -fSL "https://bootstrap.pypa.io/get-pip.py" -o get-pip.py \ - && python3.6 get-pip.py \ && python3.7 get-pip.py \ && python3.8 get-pip.py \ && python3.9 get-pip.py \ + && python3.10 get-pip.py \ && rm get-pip.py \ && rm -rf /var/lib/apt/lists/* From b0f1e84552da33a1769a4a9b3af915fdc02a3c45 Mon Sep 17 00:00:00 2001 From: Parakoopa Date: Wed, 15 Dec 2021 22:10:50 +0100 Subject: [PATCH 13/17] Remove Jinja2 dependency --- riptide/config/service/config_files.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/riptide/config/service/config_files.py b/riptide/config/service/config_files.py index 154d173..aa6e33b 100644 --- a/riptide/config/service/config_files.py +++ b/riptide/config/service/config_files.py @@ -5,8 +5,6 @@ from functools import partial from typing import TYPE_CHECKING, Dict -from jinja2 import Environment - from riptide.config.files import get_project_meta_folder, remove_all_special_chars from riptide.config.service.config_files_helper_functions import read_file @@ -14,7 +12,6 @@ from riptide.config.document.service import Service FOLDER_FOR_PROCESSED_CONFIG = 'processed_config' -jinja2env = Environment() def process_config( From 2e6580b1331ffbb71cbcb2b948b2d81c14dfbc01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Thu, 16 Dec 2021 17:40:04 +0100 Subject: [PATCH 14/17] Setup.py dependency updates --- setup.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/setup.py b/setup.py index 7bcffe4..102eb48 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,4 @@ -__version__ = '0.6.1' +__version__ = '0.7.0' from setuptools import setup, find_packages # README read-in @@ -19,12 +19,12 @@ url='https://github.com/Parakoopa/riptide-lib/', install_requires=[ 'configcrunch >= 1.0.0', - 'schema >= 0.6', - 'pyyaml >= 5.1', + 'schema >= 0.7', + 'pyyaml >= 5.4', 'appdirs >= 1.4', - 'janus >= 0.7.0', - 'psutil >= 5.6', - 'GitPython >= 3.0', + 'janus >= 0.7', + 'psutil >= 5.8', + 'GitPython >= 3.1', 'pywinpty >= 0.5.5; sys_platform == "win32"', 'python-hosts >= 0.4', 'python-dotenv >= 0.19.0' From 42e8c5965aaeac164a4a290825ce109852c0b2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Tue, 28 Dec 2021 09:30:14 +0100 Subject: [PATCH 15/17] #14: Support multiple possible start commands for services --- riptide/config/document/service.py | 41 +++++++++++++++++++++++++++--- riptide/engine/abstract.py | 8 ++++-- riptide/engine/results.py | 2 +- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/riptide/config/document/service.py b/riptide/config/document/service.py index 9256934..d7d8e5d 100644 --- a/riptide/config/document/service.py +++ b/riptide/config/document/service.py @@ -78,10 +78,27 @@ def schema(cls) -> Schema: image: str Docker Image to use - [command]: str - Command to run inside of the container. Default's to command defined in image. + [command]: str or map + If this is not set: + The default command in the image is used and considered in the "default" command group (see below). - .. warning:: Avoid quotes (", ') inside of the command, as those may lead to strange side effects. + If it is a string: + Command to run inside of the container. Default's to command defined in image. This command will be in + the "default" command group (see below). + + If it is a map: + A list of commands that this service supports. Keys are the "command group", values the commands to run. + Each service must have a command defined for the "default" command group. You can speficy a command group + to use when using `riptide start`. Default is the "default" command group, this one is also used by the + Riptide Proxy autostart feature. For more information on this see the `--cmd` flag of `riptide start`. + + Example:: + + comamnd: + default: "npm run default" + debug: "npm run debug" + + .. warning:: Avoid quotes (", ') inside of commands, as those may lead to strange side effects. [port]: int HTTP port that the web service is accessible under. This port will be used by the proxy server to redirect @@ -301,7 +318,12 @@ def schema(cls) -> Schema: Optional('$name'): str, # Added by system during processing parent app. Optional('roles'): [str], 'image': str, - Optional('command'): str, + Optional('command'): Or( + str, { + "default": str, + str: str + } + ), Optional('port'): int, Optional('logging'): { Optional('stdout'): bool, @@ -492,6 +514,17 @@ def before_start(self): self["working_directory"] ), exist_ok=True) + def get_command(self, group: str = "default"): + """Returns the command to use for the given group. 'command' must be set in self""" + if "command" not in self: + raise ValueError("No command defined.") + if isinstance(self["command"], dict): + if group in self["command"]: + return self["command"][group] + return self["command"]["default"] + else: + return self["command"] + def get_project(self) -> 'Project': """ Returns the project or raises an error if this is not assigned to a project diff --git a/riptide/engine/abstract.py b/riptide/engine/abstract.py index 1d3af01..33fc9f6 100644 --- a/riptide/engine/abstract.py +++ b/riptide/engine/abstract.py @@ -25,7 +25,8 @@ class AbstractEngine(ABC): def start_project(self, project: 'Project', services: List[str], - quick=False) -> MultiResultQueue[StartStopResultStep]: + quick=False, + command_group: str = "default") -> MultiResultQueue[StartStopResultStep]: """ Starts all services in the project. @@ -43,6 +44,7 @@ def start_project(self, :type project: 'Project' :param services: Names of the services to start :param quick: If True: Skip pre_start and post_start commands. + :param command_group: The command group of all services that should be used for the started containers. :return: MultiResultQueue[StartResult] """ @@ -157,7 +159,8 @@ def cmd_in_service(self, def service_fg(self, project: 'Project', service_name: str, - arguments: List[str] + arguments: List[str], + command_group: str = "default" ) -> None: """ Execute a service and attach output to stdout/stdin/stderr. @@ -177,6 +180,7 @@ def service_fg(self, :param project: 'Project' :param service_name: str :param arguments: List of arguments + :param command_group: The command group of all services that should be used for the fg service container :return: """ diff --git a/riptide/engine/results.py b/riptide/engine/results.py index 6a4e2ad..1ffa168 100644 --- a/riptide/engine/results.py +++ b/riptide/engine/results.py @@ -29,7 +29,7 @@ def __init__(self, message: str, details: str = None, cause: Exception = None, * self.cause = cause self.traceback_string = "Unknown reason." if cause: - self.traceback_string = "".join(traceback.format_exception(etype=type(cause), value=cause, tb=cause.__traceback__)) + self.traceback_string = "".join(traceback.format_exception(type(cause), value=cause, tb=cause.__traceback__)) super().__init__(*args, **kwargs) def __str__(self): From 612d6514b1d9d4f2436e44b60a2060d8ea3ce4fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Tue, 28 Dec 2021 10:10:25 +0100 Subject: [PATCH 16/17] #13: Support for specifying which .env files to use --- riptide/config/document/command.py | 9 ++++++--- riptide/config/document/project.py | 12 +++++++++++- riptide/config/document/service.py | 6 ++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/riptide/config/document/command.py b/riptide/config/document/command.py index 7097ba8..e547153 100644 --- a/riptide/config/document/command.py +++ b/riptide/config/document/command.py @@ -93,7 +93,8 @@ def schema_normal(cls): into the command container. [read_env_file]: bool - If enabled, read the environment variables in the ``.env`` file. Default: True + If enabled, read the environment variables in the env-files defined in the project (``env_files``). + Default: True **Example Document:** @@ -156,7 +157,8 @@ def schema_in_service(cls): Key is the name of the variable, value is the value. [read_env_file]: bool - If enabled, read the environment variables in the ``.env`` file. Default: True + If enabled, read the environment variables in the env-files defined in the project (``env_files``). + Default: True **Example Document:** @@ -305,7 +307,8 @@ def collect_environment(self) -> dict: env[key] = value if "read_env_file" not in self or self["read_env_file"]: - env.update(dotenv_values(os.path.join(self.get_project().folder(), '.env'))) + for env_file_path in self.get_project()['env_files']: + env.update(dotenv_values(os.path.join(self.get_project().folder(), env_file_path))) try: cols, lines = os.get_terminal_size() diff --git a/riptide/config/document/project.py b/riptide/config/document/project.py index 15e06b0..c86171a 100644 --- a/riptide/config/document/project.py +++ b/riptide/config/document/project.py @@ -51,6 +51,13 @@ def schema(cls) -> Schema: List of services to start when running `riptide start`. If not set, all services are started. You can also control which services to start using flags. See `riptide start --help` for more information. + [env_files]: List[str] + A list of paths to env-files, relative to the project path, that should be read-in by services and command + when starting. See the ``read_env_file`` flag at :class:`~riptide.config.document.service.Service` and + :class:`~riptide.config.document.command.Command` for more information. + + Defaults to ["./.env"]. + **Example Document:** .. code-block:: yaml @@ -70,7 +77,8 @@ def schema(cls) -> Schema: 'src': str, 'app': DocReference(App), Optional('links'): [str], - Optional('default_services'): [str] + Optional('default_services'): [str], + Optional('env_files'): [str] } ) @@ -89,6 +97,8 @@ def validate(self) -> bool: def _initialize_data_after_merge(self, data): if 'links' not in data: data['links'] = [] + if 'env_files' not in data: + data['env_files'] = ["./.env"] return data def folder(self): diff --git a/riptide/config/document/service.py b/riptide/config/document/service.py index 9256934..6cba72c 100644 --- a/riptide/config/document/service.py +++ b/riptide/config/document/service.py @@ -251,7 +251,8 @@ def schema(cls) -> Schema: Default: False [read_env_file]: bool - If enabled, read the environment variables in the ``.env`` file. Default: True + If enabled, read the environment variables in the env-files defined in the project (``env_files``). + Default: True **Example Document:** @@ -588,7 +589,8 @@ def collect_environment(self) -> dict: env[name] = value if "read_env_file" not in self or self["read_env_file"]: - env.update(dotenv_values(os.path.join(self.get_project().folder(), '.env'))) + for env_file_path in self.get_project()['env_files']: + env.update(dotenv_values(os.path.join(self.get_project().folder(), env_file_path))) # db driver if self._db_driver: From d33dfe48bb542f94b89febf11b988a2722dd1bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20K=C3=B6pcke?= Date: Tue, 28 Dec 2021 12:54:01 +0100 Subject: [PATCH 17/17] Github username update --- README.rst | 34 ++++++++++++------------- requirements_extra_riptide_from_git.txt | 8 +++--- riptide/assets/blank_user_config.yml | 2 +- riptide/config/document/config.py | 2 +- setup.py | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/README.rst b/README.rst index 5a8e0b0..5f0ab81 100644 --- a/README.rst +++ b/README.rst @@ -16,32 +16,32 @@ *More:* docs_ repo_ docker_images_ ====================== =================== =================== =================== -.. _lib: https://github.com/Parakoopa/riptide-lib -.. _cli: https://github.com/Parakoopa/riptide-cli -.. _proxy: https://github.com/Parakoopa/riptide-proxy -.. _configcrunch: https://github.com/Parakoopa/configcrunch -.. _engine_docker: https://github.com/Parakoopa/riptide-engine-docker -.. _db_mysql: https://github.com/Parakoopa/riptide-db-mysql -.. _db_mongo: https://github.com/Parakoopa/riptide-db-mongo -.. _docs: https://github.com/Parakoopa/riptide-docs -.. _repo: https://github.com/Parakoopa/riptide-repo -.. _docker_images: https://github.com/Parakoopa/riptide-docker-images -.. _php_xdebug: https://github.com/Parakoopa/riptide-plugin-php-xdebug -.. _k8s_client: https://github.com/Parakoopa/riptide-k8s-client -.. _k8s_controller: https://github.com/Parakoopa/riptide-k8s-controller +.. _lib: https://github.com/theCapypara/riptide-lib +.. _cli: https://github.com/theCapypara/riptide-cli +.. _proxy: https://github.com/theCapypara/riptide-proxy +.. _configcrunch: https://github.com/theCapypara/configcrunch +.. _engine_docker: https://github.com/theCapypara/riptide-engine-docker +.. _db_mysql: https://github.com/theCapypara/riptide-db-mysql +.. _db_mongo: https://github.com/theCapypara/riptide-db-mongo +.. _docs: https://github.com/theCapypara/riptide-docs +.. _repo: https://github.com/theCapypara/riptide-repo +.. _docker_images: https://github.com/theCapypara/riptide-docker-images +.. _php_xdebug: https://github.com/theCapypara/riptide-plugin-php-xdebug +.. _k8s_client: https://github.com/theCapypara/riptide-k8s-client +.. _k8s_controller: https://github.com/theCapypara/riptide-k8s-controller |build| |docs| |pypi-version| |pypi-downloads| |pypi-license| |pypi-pyversions| |slack| -.. |build| image:: https://img.shields.io/github/workflow/status/Parakoopa/riptide-lib/Build,%20test%20and%20publish - :target: https://github.com/Parakoopa/riptide-lib/actions +.. |build| image:: https://img.shields.io/github/workflow/status/theCapypara/riptide-lib/Build,%20test%20and%20publish + :target: https://github.com/theCapypara/riptide-lib/actions :alt: Build Status .. |docs| image:: https://readthedocs.org/projects/riptide-docs/badge/?version=latest :target: https://riptide-docs.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status -.. |slack| image:: https://slack.riptide.parakoopa.de/badge.svg - :target: https://slack.riptide.parakoopa.de +.. |slack| image:: https://slack.riptide.theCapypara.de/badge.svg + :target: https://slack.riptide.theCapypara.de :alt: Join our Slack workspace .. |pypi-version| image:: https://img.shields.io/pypi/v/riptide-lib diff --git a/requirements_extra_riptide_from_git.txt b/requirements_extra_riptide_from_git.txt index 2b7c8ce..f48a5e4 100644 --- a/requirements_extra_riptide_from_git.txt +++ b/requirements_extra_riptide_from_git.txt @@ -1,4 +1,4 @@ --e git+https://github.com/Parakoopa/riptide-cli.git#egg=riptide-cli --e git+https://github.com/Parakoopa/riptide-proxy.git#egg=riptide-proxy --e git+https://github.com/Parakoopa/riptide-engine-docker.git#egg=riptide-engine_docker --e git+https://github.com/Parakoopa/riptide-db-mysql.git#egg=riptide-db_mysql +-e git+https://github.com/theCapypara/riptide-cli.git#egg=riptide-cli +-e git+https://github.com/theCapypara/riptide-proxy.git#egg=riptide-proxy +-e git+https://github.com/theCapypara/riptide-engine-docker.git#egg=riptide-engine_docker +-e git+https://github.com/theCapypara/riptide-db-mysql.git#egg=riptide-db_mysql diff --git a/riptide/assets/blank_user_config.yml b/riptide/assets/blank_user_config.yml index e2a37d9..83843da 100644 --- a/riptide/assets/blank_user_config.yml +++ b/riptide/assets/blank_user_config.yml @@ -16,7 +16,7 @@ riptide: # List of Riptide repositories. repos: # Public Riptide repo by the community - - https://github.com/Parakoopa/riptide-repo.git + - https://github.com/theCapypara/riptide-repo.git # Add all project hostnames to the /etc/hosts file automatically, see documentation. update_hosts_file: true diff --git a/riptide/config/document/config.py b/riptide/config/document/config.py index 6cc5c2c..94d04d0 100644 --- a/riptide/config/document/config.py +++ b/riptide/config/document/config.py @@ -123,7 +123,7 @@ def schema(cls) -> Schema: - 127.0.0.1/32 engine: docker repos: - - https://github.com/Parakoopa/riptide-repo.git + - https://github.com/theCapypara/riptide-repo.git update_hosts_file: true performance: dont_sync_named_volumes_with_host: auto diff --git a/setup.py b/setup.py index 102eb48..52b7a9a 100644 --- a/setup.py +++ b/setup.py @@ -16,7 +16,7 @@ description='Tool to manage development environments for web applications using containers - Library Package', long_description=long_description, long_description_content_type='text/x-rst', - url='https://github.com/Parakoopa/riptide-lib/', + url='https://github.com/theCapypara/riptide-lib/', install_requires=[ 'configcrunch >= 1.0.0', 'schema >= 0.7',