From 6a3fbcce0e0e115769cc15664b03eee94ac4b7f0 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 10:21:06 -0500 Subject: [PATCH 01/25] Version Bump --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index e1ec16d..63b09b7 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='netbox_config_backup', - version='2.1.0', + version='2.1.1-beta1', description='NetBox Configuration Backup', long_description='Plugin to backup device configuration', url='https://github.com/dansheps/netbox-config-backup/', From 31621e4606d7edd42543edecb2a115cb151500b7 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 11:02:54 -0500 Subject: [PATCH 02/25] Update CI workflow --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 98c3db8..3d28f51 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -51,7 +51,7 @@ jobs: - name: Install dependencies & set up configuration run: | python -m pip install --upgrade pip - pip install -r netbox\requirements.txt + pip install -r netbox\\requirements.txt pip install pycodestyle coverage tblib pip install -e netbox-config-backup From 82ddbfdd3b1ea07e5c54a6996e92cefb1f0267dd Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 11:04:52 -0500 Subject: [PATCH 03/25] Update CI workflow --- .github/workflows/ci-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 3d28f51..5428fda 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -11,7 +11,7 @@ jobs: NETBOX_CONFIGURATION: netbox.configuration_testing strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] + python-version: ['3.10', '3.11', '3.12'] services: redis: image: redis @@ -51,7 +51,7 @@ jobs: - name: Install dependencies & set up configuration run: | python -m pip install --upgrade pip - pip install -r netbox\\requirements.txt + pip install -r netbox/requirements.txt pip install pycodestyle coverage tblib pip install -e netbox-config-backup From b77488b2e1580a09f1d6659acf6413e08daf2c85 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 11:13:46 -0500 Subject: [PATCH 04/25] Update CI test --- .github/workflows/ci-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5428fda..536521b 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -54,6 +54,7 @@ jobs: pip install -r netbox/requirements.txt pip install pycodestyle coverage tblib pip install -e netbox-config-backup + cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ - name: Run tests - run: coverage run --source="netbox-config-backup/" netbox/manage.py test netbox-config-backup/ --parallel \ No newline at end of file + run: coverage run --source="netbox" netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file From 88e4eb0c52f9ad183cff2fa80d3c04502dba1a64 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 11:18:24 -0500 Subject: [PATCH 05/25] Update CI --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 536521b..8df5d81 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -57,4 +57,4 @@ jobs: cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ - name: Run tests - run: coverage run --source="netbox" netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file + run: coverage run --source="netbox-config-backup" ../netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file From 910ad66de36ee8c7478cb880d6025e3d7dc060c5 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 11:25:03 -0500 Subject: [PATCH 06/25] Update CI --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 8df5d81..80b8011 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -57,4 +57,4 @@ jobs: cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ - name: Run tests - run: coverage run --source="netbox-config-backup" ../netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file + run: coverage run --source="netbox-config-backup" ../netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file From f9e76a3b668976a0ac6ccb9c18ab9e067a0cfc46 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 5 Sep 2024 11:46:33 -0500 Subject: [PATCH 07/25] Update CI --- .github/workflows/ci-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 80b8011..5132e33 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -55,6 +55,7 @@ jobs: pip install pycodestyle coverage tblib pip install -e netbox-config-backup cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ + ls -la - name: Run tests - run: coverage run --source="netbox-config-backup" ../netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file + run: coverage run --source="netbox-config-backup" netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file From e85b6a072ffcd859eafac4e192160a4c7dd005ce Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Mon, 9 Sep 2024 00:24:01 -0500 Subject: [PATCH 08/25] Update CI Tests --- .github/workflows/ci-tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5132e33..de206f0 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -55,7 +55,9 @@ jobs: pip install pycodestyle coverage tblib pip install -e netbox-config-backup cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ - ls -la - name: Run tests - run: coverage run --source="netbox-config-backup" netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel \ No newline at end of file + run: coverage run --source="netbox-config-backup/netbox_config_backup" netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel + + - name: Show coverage report + run: coverage report --skip-covered --omit '*/migrations/*,*/tests/*' \ No newline at end of file From 487c4f52f29cd489dabd13ff8f19453b38dc806d Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Mon, 9 Sep 2024 08:52:12 -0500 Subject: [PATCH 09/25] Update CI tests --- .github/configuration.testing.py | 12 +++++++++++- .github/workflows/ci-tests.yml | 3 ++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/configuration.testing.py b/.github/configuration.testing.py index fcbfee2..1c90c54 100644 --- a/.github/configuration.testing.py +++ b/.github/configuration.testing.py @@ -15,9 +15,19 @@ } PLUGINS = [ - 'netbox_secretstore', + 'netbox_config_backup', ] +PLUGINS_MODULES = { + 'netbox_config_backup': { + 'repository': 'c:\\Development\\backuprepotest\\', + 'repository': '/tmp/repository/', + 'committer': 'Test Committer ', + 'author': 'Test Committer ', + 'frequency': 3600, + } +} + REDIS = { 'tasks': { 'HOST': 'localhost', diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index de206f0..4449ec8 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -51,10 +51,11 @@ jobs: - name: Install dependencies & set up configuration run: | python -m pip install --upgrade pip - pip install -r netbox/requirements.txt + pip install -r netbox/ requirements.txt pip install pycodestyle coverage tblib pip install -e netbox-config-backup cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ + ls -la netbox/netbox/netbox/ - name: Run tests run: coverage run --source="netbox-config-backup/netbox_config_backup" netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel From 9f8336ffa7cbdce718502aa4fe467e1b9e5cbb9c Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Mon, 9 Sep 2024 08:53:35 -0500 Subject: [PATCH 10/25] Update CI --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 4449ec8..a87464c 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -51,7 +51,7 @@ jobs: - name: Install dependencies & set up configuration run: | python -m pip install --upgrade pip - pip install -r netbox/ requirements.txt + pip install -r netbox/requirements.txt pip install pycodestyle coverage tblib pip install -e netbox-config-backup cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ From 5453907d7c451345b5a0976eca9651e6fd14cfcf Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Mon, 9 Sep 2024 08:57:10 -0500 Subject: [PATCH 11/25] Update tests --- .github/workflows/ci-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index a87464c..d7f94a2 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -54,7 +54,7 @@ jobs: pip install -r netbox/requirements.txt pip install pycodestyle coverage tblib pip install -e netbox-config-backup - cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/ + cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/configuration_testing.py ls -la netbox/netbox/netbox/ - name: Run tests From 32ed15b325dd0fd336933da509574dddf40a461b Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Mon, 9 Sep 2024 09:04:56 -0500 Subject: [PATCH 12/25] Update CI --- .github/configuration.testing.py | 3 +-- .github/workflows/ci-tests.yml | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/configuration.testing.py b/.github/configuration.testing.py index 1c90c54..e51653d 100644 --- a/.github/configuration.testing.py +++ b/.github/configuration.testing.py @@ -20,12 +20,11 @@ PLUGINS_MODULES = { 'netbox_config_backup': { - 'repository': 'c:\\Development\\backuprepotest\\', 'repository': '/tmp/repository/', 'committer': 'Test Committer ', 'author': 'Test Committer ', 'frequency': 3600, - } + }, } REDIS = { diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index d7f94a2..88c3f59 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -55,7 +55,8 @@ jobs: pip install pycodestyle coverage tblib pip install -e netbox-config-backup cp -f netbox-config-backup/.github/configuration.testing.py netbox/netbox/netbox/configuration_testing.py - ls -la netbox/netbox/netbox/ + mkdir /tmp/repository + git init /tmp/repository - name: Run tests run: coverage run --source="netbox-config-backup/netbox_config_backup" netbox/netbox/manage.py test netbox-config-backup/netbox_config_backup --parallel From 02b6d6580ef9ea68b350ba8769c8d12efdfdf51b Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Mon, 9 Sep 2024 10:45:46 -0500 Subject: [PATCH 13/25] Fix test issue with config --- .github/configuration.testing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/configuration.testing.py b/.github/configuration.testing.py index e51653d..595e4a5 100644 --- a/.github/configuration.testing.py +++ b/.github/configuration.testing.py @@ -18,7 +18,7 @@ 'netbox_config_backup', ] -PLUGINS_MODULES = { +PLUGINS_CONFIG = { 'netbox_config_backup': { 'repository': '/tmp/repository/', 'committer': 'Test Committer ', From ca97abc1312cabb83ebedda1522a743b5ab17a5e Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 12 Sep 2024 08:58:13 -0500 Subject: [PATCH 14/25] Upgrade model to PrimaryModel, add GraphQL support --- netbox_config_backup/__init__.py | 1 + netbox_config_backup/forms.py | 26 ++++++++++-- netbox_config_backup/graphql/__init__.py | 0 netbox_config_backup/graphql/filters.py | 14 +++++++ netbox_config_backup/graphql/schema.py | 17 ++++++++ netbox_config_backup/graphql/types.py | 25 ++++++++++++ netbox_config_backup/models/backups.py | 9 ++++- netbox_config_backup/tables.py | 3 ++ netbox_config_backup/tasks.py | 40 ++++++++++--------- .../netbox_config_backup/backup.html | 14 ++++++- netbox_config_backup/urls.py | 6 +++ netbox_config_backup/views.py | 20 +++++++++- 12 files changed, 147 insertions(+), 28 deletions(-) create mode 100644 netbox_config_backup/graphql/__init__.py create mode 100644 netbox_config_backup/graphql/filters.py create mode 100644 netbox_config_backup/graphql/schema.py create mode 100644 netbox_config_backup/graphql/types.py diff --git a/netbox_config_backup/__init__.py b/netbox_config_backup/__init__.py index b50e844..c1e216d 100644 --- a/netbox_config_backup/__init__.py +++ b/netbox_config_backup/__init__.py @@ -27,6 +27,7 @@ class NetboxConfigBackup(PluginConfig): queues = [ 'jobs' ] + graphql_schema = 'graphql.schema.schema' config = NetboxConfigBackup diff --git a/netbox_config_backup/forms.py b/netbox_config_backup/forms.py index 2e86eb4..e8d04cd 100644 --- a/netbox_config_backup/forms.py +++ b/netbox_config_backup/forms.py @@ -6,16 +6,20 @@ from dcim.choices import DeviceStatusChoices from dcim.models import Device from ipam.models import IPAddress +from netbox.forms import NetBoxModelForm, NetBoxModelBulkEditForm from netbox_config_backup.models import Backup -from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField +from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField, CommentField __all__ = ( 'BackupForm', 'BackupFilterSetForm', + 'BackupBulkEditForm', ) +from utilities.forms.rendering import FieldSet -class BackupForm(forms.ModelForm): + +class BackupForm(NetBoxModelForm): device = DynamicModelChoiceField( label='Device', required=False, @@ -36,9 +40,11 @@ class BackupForm(forms.ModelForm): 'assigned_to_interface': True }, ) + comments = CommentField() + class Meta: model = Backup - fields = ('name', 'device', 'ip', 'status') + fields = ('name', 'device', 'ip', 'status', 'description', 'comments', 'config_status') def clean(self): super().clean() @@ -71,7 +77,7 @@ class BackupFilterSetForm(forms.Form): 'status': [DeviceStatusChoices.STATUS_ACTIVE], 'platform__napalm__ne': None, 'has_primary_ip': True, - } + }, ) ip = forms.CharField( required=False, @@ -84,3 +90,15 @@ class BackupFilterSetForm(forms.Form): ) +class BackupBulkEditForm(NetBoxModelBulkEditForm): + + description = forms.CharField( + label=_('Description'), + max_length=200, + required=False + ) + comments = CommentField() + + model = Backup + fieldsets = () + nullable_fields = () diff --git a/netbox_config_backup/graphql/__init__.py b/netbox_config_backup/graphql/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/netbox_config_backup/graphql/filters.py b/netbox_config_backup/graphql/filters.py new file mode 100644 index 0000000..c5e476c --- /dev/null +++ b/netbox_config_backup/graphql/filters.py @@ -0,0 +1,14 @@ +import strawberry_django +from netbox_config_backup import filtersets, models + +from netbox.graphql.filter_mixins import autotype_decorator, BaseFilterMixin + +__all__ = ( + 'BackupFilter', +) + + +@strawberry_django.filter(models.Backup, lookups=True) +@autotype_decorator(filtersets.BackupFilterSet) +class BackupFilter(BaseFilterMixin): + pass diff --git a/netbox_config_backup/graphql/schema.py b/netbox_config_backup/graphql/schema.py new file mode 100644 index 0000000..4d74922 --- /dev/null +++ b/netbox_config_backup/graphql/schema.py @@ -0,0 +1,17 @@ +from typing import List + +import strawberry +import strawberry_django + +from .types import * + + +@strawberry.type(name="Query") +class BackupQuery: + backup: BackupType = strawberry_django.field() + backup_list: List[BackupType] = strawberry_django.field() + + +schema = [ + BackupQuery, +] diff --git a/netbox_config_backup/graphql/types.py b/netbox_config_backup/graphql/types.py new file mode 100644 index 0000000..0a7db99 --- /dev/null +++ b/netbox_config_backup/graphql/types.py @@ -0,0 +1,25 @@ +from typing import Annotated + +import strawberry +import strawberry_django + +from netbox.graphql.types import NetBoxObjectType +from .filters import * + +from netbox_config_backup import models + +__all__ = ( + 'BackupType', +) + + +@strawberry_django.type( + models.Backup, + fields='__all__', + filters=BackupFilter +) +class BackupType(NetBoxObjectType): + + name: str + device: Annotated["DeviceType", strawberry.lazy('dcim.graphql.types')] | None + ip: Annotated["IPAddressType", strawberry.lazy('ipam.graphql.types')] | None diff --git a/netbox_config_backup/models/backups.py b/netbox_config_backup/models/backups.py index 9e66295..313c760 100644 --- a/netbox_config_backup/models/backups.py +++ b/netbox_config_backup/models/backups.py @@ -10,7 +10,7 @@ from dcim.models import Device from core.choices import JobStatusChoices -from netbox.models import NetBoxModel +from netbox.models import PrimaryModel from netbox_config_backup.choices import StatusChoices from netbox_config_backup.helpers import get_repository_dir @@ -22,7 +22,7 @@ logger = logging.getLogger(f"netbox_config_backup") -class Backup(NetBoxModel): +class Backup(PrimaryModel): name = models.CharField(max_length=255, unique=True) uuid = models.UUIDField(default=uuid.uuid4, editable=False) status = models.CharField( @@ -42,6 +42,10 @@ class Backup(NetBoxModel): blank=True, null=True ) + config_status = models.BooleanField( + blank=True, + null=True + ) objects = BackupQuerySet.as_manager() @@ -95,6 +99,7 @@ def set_config(self, configs, files=('running', 'startup'), pk=None): stored = stored_configs.get(file) if stored_configs.get(file) is not None else '' #logger.debug(f'[{pk}] Getting new config') current = configs.get(file) if configs.get(file) is not None else '' + #logger.debug(f'[{pk}] Starting diff for {file}') if Differ(stored, current).is_diff(): changes = True diff --git a/netbox_config_backup/tables.py b/netbox_config_backup/tables.py index 1d49725..bc2fb8c 100644 --- a/netbox_config_backup/tables.py +++ b/netbox_config_backup/tables.py @@ -47,6 +47,9 @@ class BackupTable(BaseTable): backup_count = tables.Column( accessor='changes' ) + config_status = tables.BooleanColumn( + verbose_name='Config Saved' + ) class Meta(BaseTable.Meta): model = Backup diff --git a/netbox_config_backup/tasks.py b/netbox_config_backup/tasks.py index d1da741..58532fa 100644 --- a/netbox_config_backup/tasks.py +++ b/netbox_config_backup/tasks.py @@ -1,32 +1,18 @@ -import logging -import sys import traceback from datetime import timedelta from django.utils import timezone -from django_rq import job from netmiko import NetmikoAuthenticationException, NetmikoTimeoutException from core.choices import JobStatusChoices from netbox import settings from netbox.api.exceptions import ServiceUnavailable -from netbox.config import get_config -from netbox_config_backup.models import Backup, BackupJob, BackupCommit +from netbox_config_backup.models import BackupJob +from netbox_config_backup.utils.configs import check_config_save_status +from netbox_config_backup.utils.logger import get_logger from netbox_config_backup.utils.rq import can_backup -def get_logger(): - # Setup logging to Stdout - formatter = logging.Formatter(f'[%(asctime)s][%(levelname)s] - %(message)s') - stdouthandler = logging.StreamHandler(sys.stdout) - stdouthandler.setLevel(logging.DEBUG) - stdouthandler.setFormatter(formatter) - logger = logging.getLogger(f"netbox_config_backup") - logger.addHandler(stdouthandler) - - return logger - - def napalm_init(device, ip=None, extra_args={}): from netbox import settings username = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_USERNAME', None) @@ -99,7 +85,25 @@ def backup_config(backup, pk=None): logger.info(f'{backup}: Backup started') #logger.debug(f'[{pk}] Connecting') d = napalm_init(backup.device, ip) - #logger.debug(f'[{pk}] Finished Connection') + #logger.debug(f'[{pk} + + try: + status = check_config_save_status(d) + if status is not None: + if status and not backup.config_status: + backup.config_status = status + backup.save() + elif not status and backup.config_status: + backup.config_status = status + backup.save() + elif not status and backup.config_status is None: + backup.config_status = status + backup.save() + elif status and backup.config_status is None: + backup.config_status = status + backup.save() + except Exception as e: + logger.error(f'{backup}: had error setting backup status: {e}') #logger.debug(f'[{pk}] Getting config') configs = d.get_config() diff --git a/netbox_config_backup/templates/netbox_config_backup/backup.html b/netbox_config_backup/templates/netbox_config_backup/backup.html index bb80566..532ddfd 100644 --- a/netbox_config_backup/templates/netbox_config_backup/backup.html +++ b/netbox_config_backup/templates/netbox_config_backup/backup.html @@ -38,15 +38,27 @@
{{ object.ip|placeholder }} {% endif %} + + Description + {{ object.description|placeholder }} + + {% include 'inc/panels/comments.html' %} + + +
Status
+ + + + @@ -66,8 +78,6 @@
Config Saved{{ object.config_status | placeholder }}
Scheduled {{ status.scheduled | placeholder }}{% if status.scheduled %} ({{status.next_attempt}}){% endif %}
-
-
{% endblock %} \ No newline at end of file diff --git a/netbox_config_backup/urls.py b/netbox_config_backup/urls.py index b09180e..9394638 100644 --- a/netbox_config_backup/urls.py +++ b/netbox_config_backup/urls.py @@ -1,11 +1,15 @@ from django.urls import path + +from netbox.views.generic import ObjectChangeLogView from . import views +from .models import Backup urlpatterns = [ path('unassigned/', views.UnassignedBackupListView.as_view(), name='unassignedbackup_list'), path('devices/', views.BackupListView.as_view(), name='backup_list'), path('devices/add/', views.BackupEditView.as_view(), name='backup_add'), path('devices//', views.BackupView.as_view(), name='backup'), + path('devices//changelog', ObjectChangeLogView.as_view(), name="backup_changelog", kwargs={'model': Backup}), path('devices//edit/', views.BackupEditView.as_view(), name='backup_edit'), path('devices//delete/', views.BackupDeleteView.as_view(), name='backup_delete'), path('devices//backups/', views.BackupBackupsView.as_view(), name='backup_backups'), @@ -13,5 +17,7 @@ path('devices//config//', views.ConfigView.as_view(), name='backup_config'), path('devices//diff/', views.DiffView.as_view(), name='backup_diff'), path('devices//diff//', views.DiffView.as_view(), name='backup_diff'), + path('devices/edit/', views.BackupBulkEditView.as_view(), name='backup_bulk_edit'), + path('devices/delete/', views.BackupBulkDeleteView.as_view(), name='backup_bulk_delete'), path('devices//diff///', views.DiffView.as_view(), name='backup_diff'), ] diff --git a/netbox_config_backup/views.py b/netbox_config_backup/views.py index 43e89a5..6109d51 100644 --- a/netbox_config_backup/views.py +++ b/netbox_config_backup/views.py @@ -7,10 +7,11 @@ from django.views import View from core.choices import JobStatusChoices -from netbox.views.generic import ObjectDeleteView, ObjectEditView, ObjectView, ObjectListView, ObjectChildrenView +from netbox.views.generic import ObjectDeleteView, ObjectEditView, ObjectView, ObjectListView, ObjectChildrenView, \ + BulkEditView, BulkDeleteView from netbox_config_backup.filtersets import BackupFilterSet, BackupsFilterSet -from netbox_config_backup.forms import BackupForm, BackupFilterSetForm +from netbox_config_backup.forms import BackupForm, BackupFilterSetForm, BackupBulkEditForm from netbox_config_backup.git import GitBackup from netbox_config_backup.models import Backup, BackupJob, BackupCommitTreeChange, BackupCommit, BackupObject from netbox_config_backup.tables import BackupTable, BackupsTable @@ -141,6 +142,21 @@ def get_return_url(self, request, obj=None): return reverse('home') +@register_model_view(Backup, 'bulk_edit') +class BackupBulkEditView(BulkEditView): + queryset = Backup.objects.all() + form = BackupBulkEditForm + filterset = BackupFilterSet + table = BackupTable + + +@register_model_view(Backup, 'bulk_delete') +class BackupBulkDeleteView(BulkDeleteView): + queryset = Backup.objects.all() + filterset = BackupFilterSet + table = BackupTable + + @register_model_view(Backup, 'config') class ConfigView(ObjectView): queryset = Backup.objects.all() From 7bf2439210ae7741ec4852e94776336889c36c4c Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 12 Sep 2024 09:02:44 -0500 Subject: [PATCH 15/25] Add new config_status option --- .../migrations/0014_backup_config_status.py | 18 ++++ netbox_config_backup/utils/configs.py | 83 +++++++++++++++++++ netbox_config_backup/utils/logger.py | 14 ++++ 3 files changed, 115 insertions(+) create mode 100644 netbox_config_backup/migrations/0014_backup_config_status.py create mode 100644 netbox_config_backup/utils/configs.py create mode 100644 netbox_config_backup/utils/logger.py diff --git a/netbox_config_backup/migrations/0014_backup_config_status.py b/netbox_config_backup/migrations/0014_backup_config_status.py new file mode 100644 index 0000000..e037e6d --- /dev/null +++ b/netbox_config_backup/migrations/0014_backup_config_status.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.8 on 2024-09-06 02:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_config_backup', '0013_backup__to_netboxmodel'), + ] + + operations = [ + migrations.AddField( + model_name='backup', + name='config_status', + field=models.BooleanField(blank=True, null=True), + ), + ] diff --git a/netbox_config_backup/utils/configs.py b/netbox_config_backup/utils/configs.py new file mode 100644 index 0000000..4cda63a --- /dev/null +++ b/netbox_config_backup/utils/configs.py @@ -0,0 +1,83 @@ +from datetime import datetime +import re + +from netbox_config_backup.utils.logger import get_logger +logger = get_logger() + + +def check_config_save_status(d): + logger.info(f'Switch: {d.hostname}') + platform = { + 'ios': { + 'running': { + 'command': 'show running-config | inc ! Last configuration change', + 'regex': r'(?P\d+):(?P\d+):(?P\d+) \S+ \S+ (?P\S+) (?P\d+) (?P\S+)(?: by \S+)?' + }, + 'startup':{ + 'command': 'show startup-config | inc ! Last configuration change', + 'regex': r'(?P\d+):(?P\d+):(?P\d+) \S+ \S+ (?P\S+) (?P\d+) (?P\S+)(?: by \S+)?' + } + }, + 'nxos_ssh': { + 'running': { + 'command': 'show running-config | inc "!Running configuration last done at:"', + 'regex': r'(?P\S+)\s+(?P\d+) (?P\d+):(?P\d+):(?P\d+) (?P\d+)' + }, + 'startup': { + 'command': 'show startup-config | inc "!Startup config saved at:"', + 'regex': r'(?P\S+)\s+(?P\d+) (?P\d+):(?P\d+):(?P\d+) (?P\d+)' + } + }, + } + + try: + datetimes = { + 'running': None, + 'startup': None + } + dates = { + 'running': None, + 'startup': None + } + for file in ['running', 'startup']: + command = d.cli(commands=[platform.get(d.platform, {}).get(file, {}).get('command', '')]) + result = list(command.values()).pop() + regex = platform.get(d.platform, {}).get(file, {}).get('regex', '') + search = re.search(regex, result) + + if search is not None and search.groupdict(): + status = search.groupdict() + year = status.get('year') + month = status.get('month') + day = f"0{int(status.get('day'))}" if int(status.get('day')) < 10 else f"{int(status.get('day'))}" + hours = status.get('hours') + minutes = status.get('minutes') + seconds = status.get('seconds') + + date = f'{year}-{month}-{day} {hours}:{minutes}:{seconds}' + + dates[file] = date + datetimes[file] = datetime.strptime(date, '%Y-%b-%d %H:%M:%S') + + # logger.debug(f'\t{file}: {result} ({date}) ({datetimes[file]})') + else: + logger.debug(f'\tNo {file} time found, platform: {d.platform}') + # logger.debug(f'\t{file}: {result}') + + if datetimes['running'] is None and datetimes['startup'] is not None: + logger.debug(f'\tValid backup as booted from startup') + return True + elif datetimes['startup'] is None: + logger.debug(f'\tNo startup time') + # logger.info(f'{datetimes["running"]} ({dates["running"]}): {datetimes["startup"]} ({dates["startup"]})') + return + elif datetimes['running'] <= datetimes['startup']: + logger.debug(f'\tRunning config less then startup') + return True + elif datetimes['running'] > datetimes['startup']: + logger.debug(f'\tRunning config greater then startup') + return False + + except Exception as e: + + logger.error(f'Exception when trying to check config status: {e}') \ No newline at end of file diff --git a/netbox_config_backup/utils/logger.py b/netbox_config_backup/utils/logger.py new file mode 100644 index 0000000..915d5bd --- /dev/null +++ b/netbox_config_backup/utils/logger.py @@ -0,0 +1,14 @@ +import logging +import sys + + +def get_logger(): + # Setup logging to Stdout + formatter = logging.Formatter(f'[%(asctime)s][%(levelname)s] - %(message)s') + stdouthandler = logging.StreamHandler(sys.stdout) + stdouthandler.setLevel(logging.DEBUG) + stdouthandler.setFormatter(formatter) + logger = logging.getLogger(f"netbox_config_backup") + logger.addHandler(stdouthandler) + + return logger From 38efb1c233d49af98c224865fe3f9b36f7d11a0a Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 12 Sep 2024 09:03:04 -0500 Subject: [PATCH 16/25] Setup test suite --- netbox_config_backup/api/serializers.py | 9 +-- netbox_config_backup/api/views.py | 4 +- .../management/commands/fork.py | 18 +++++- netbox_config_backup/tests/test_api.py | 46 ++++++++++++++ netbox_config_backup/tests/test_filtersets.py | 58 +++++++++++++++++ netbox_config_backup/tests/test_forms.py | 29 +++++++++ netbox_config_backup/tests/test_models.py | 35 +++++++---- netbox_config_backup/tests/test_views.py | 63 +++++++++++++++++++ 8 files changed, 240 insertions(+), 22 deletions(-) create mode 100644 netbox_config_backup/tests/test_api.py create mode 100644 netbox_config_backup/tests/test_filtersets.py create mode 100644 netbox_config_backup/tests/test_forms.py create mode 100644 netbox_config_backup/tests/test_views.py diff --git a/netbox_config_backup/api/serializers.py b/netbox_config_backup/api/serializers.py index dbc4f68..c07cd2e 100644 --- a/netbox_config_backup/api/serializers.py +++ b/netbox_config_backup/api/serializers.py @@ -13,12 +13,13 @@ class BackupSerializer(NetBoxModelSerializer): - url = serializers.HyperlinkedIdentityField(view_name='plugins-api:netbox_config_backup-api:backup-detail') - device = DeviceSerializer(nested=True) - ip = IPAddressSerializer(nested=True) + device = DeviceSerializer(nested=True, required=False, allow_null=True), + ip = IPAddressSerializer(nested=True, required=False, allow_null=True) class Meta: model = Backup fields = [ - 'id', 'url', 'display', 'device', 'ip', 'name', 'uuid', 'status' + 'id', 'url', 'display_url', 'display', 'name', 'device', 'ip', + 'uuid', 'status', 'config_status', ] + brief_fields = ('display', 'id', 'name', 'url') diff --git a/netbox_config_backup/api/views.py b/netbox_config_backup/api/views.py index 334075b..ced9056 100644 --- a/netbox_config_backup/api/views.py +++ b/netbox_config_backup/api/views.py @@ -1,9 +1,9 @@ -from rest_framework.viewsets import ModelViewSet +from netbox.api.viewsets import NetBoxModelViewSet from netbox_config_backup.api import BackupSerializer from netbox_config_backup.models import Backup -class BackupViewSet(ModelViewSet): +class BackupViewSet(NetBoxModelViewSet): queryset = Backup.objects.all() serializer_class = BackupSerializer \ No newline at end of file diff --git a/netbox_config_backup/management/commands/fork.py b/netbox_config_backup/management/commands/fork.py index 0196b53..cd3881e 100644 --- a/netbox_config_backup/management/commands/fork.py +++ b/netbox_config_backup/management/commands/fork.py @@ -24,11 +24,23 @@ def test(i): time.sleep(10) self.stdout.write(f"Child {i} sleep complete") - processes = [] - for i in range(1, 2): + processes = {} + for i in range(1, 3): p = Process(target=test, args=(i,)) p.start() p.join(1) self.stdout.write(f"Child {i} running") - processes.append(p) + processes.update({p.pid: p}) + + while True: + if len(processes) == 0: + break + for pid in list(processes.keys()): + process = processes.get(pid, None) + if not process.is_alive(): + del processes[pid] + time.sleep(1) + + + diff --git a/netbox_config_backup/tests/test_api.py b/netbox_config_backup/tests/test_api.py new file mode 100644 index 0000000..ad64bc8 --- /dev/null +++ b/netbox_config_backup/tests/test_api.py @@ -0,0 +1,46 @@ +from django.urls import reverse +from rest_framework import status + +from utilities.testing import APIViewTestCases, APITestCase + +from netbox_config_backup.models import Backup + + + +class AppTest(APITestCase): + def test_root(self): + url = reverse("plugins-api:netbox_config_backup-api:api-root") + response = self.client.get(f"{url}?format=api", **self.header) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + + +class BackupTest(APIViewTestCases.APIViewTestCase): + model = Backup + view_namespace = "plugins-api:netbox_config_backup" + brief_fields = ['display', 'id', 'name', 'url'] + create_data = [ + { + 'name': 'Backup 4', + 'config_status': False, + }, + { + 'name': 'Backup 5', + 'config_status': False, + }, + { + 'name': 'Backup 6', + 'config_status': False, + }, + ] + + bulk_update_data = { + 'config_status': True + } + + @classmethod + def setUpTestData(cls): + + Backup.objects.create(name='Backup 1') + Backup.objects.create(name='Backup 2') + Backup.objects.create(name='Backup 3') diff --git a/netbox_config_backup/tests/test_filtersets.py b/netbox_config_backup/tests/test_filtersets.py new file mode 100644 index 0000000..041dfa8 --- /dev/null +++ b/netbox_config_backup/tests/test_filtersets.py @@ -0,0 +1,58 @@ +from django.test import TestCase + +from dcim.models import Site, Manufacturer, DeviceType, DeviceRole, Device +from ipam.models import IPAddress +from utilities.testing import ChangeLoggedFilterSetTests + +from netbox_config_backup.filtersets import BackupFilterSet +from netbox_config_backup.models import Backup + + + +class BackupTestCase(TestCase): + queryset = Backup.objects.all() + filterset = BackupFilterSet + + @classmethod + def setUpTestData(cls): + site = Site.objects.create(name='Site 1', slug='site-1') + manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='manufacturer-1') + device_type = DeviceType.objects.create( + manufacturer=manufacturer, model='Device Type 1', slug='device-type-1' + ) + role = DeviceRole.objects.create( + name='Device Role 1', slug='device-role-1' + ) + + ip = IPAddress.objects.create( + address='10.10.10.10/32' + ) + + devices = ( + Device(name='Device 1', device_type=device_type, role=role, site=site), + Device(name='Device 2', device_type=device_type, role=role, site=site), + ) + Device.objects.bulk_create(devices) + + backups = ( + Backup(name='Backup 1', device=devices[0]), + Backup(name='Backup 2', device=devices[1]), + Backup(name='Backup 3', device=devices[1], ip=ip), + ) + Backup.objects.bulk_create(backups) + + def test_q(self): + params = {'q': 'Backup 1'} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) + + def test_name(self): + params = {'name': ['Backup 1', 'Backup 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_device(self): + params = {'device': ['Device 2']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) + + def test_ip(self): + params = {'ip_address': ['10.10.10.10']} + self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) \ No newline at end of file diff --git a/netbox_config_backup/tests/test_forms.py b/netbox_config_backup/tests/test_forms.py new file mode 100644 index 0000000..2b9fde1 --- /dev/null +++ b/netbox_config_backup/tests/test_forms.py @@ -0,0 +1,29 @@ +from django.test import TestCase + +from dcim.models import Device, Platform +from utilities.testing import create_test_device + +from netbox_napalm_plugin.models import NapalmPlatformConfig + +from netbox_config_backup.forms import * +from netbox_config_backup.models import * + + + +class BackupTestCase(TestCase): + + @classmethod + def setUpTestData(cls): + platform = Platform.objects.create(name='Cisco IOS', slug='cisco-ios') + device = create_test_device(name='Device 1', platform=platform) + NapalmPlatformConfig.objects.create(platform=platform, napalm_driver='cisco_ios') + + def test_backup(self): + form = BackupForm(data={ + 'name': 'New Backup', + 'device': Device.objects.first().pk, + 'status': 'disabled' + }) + print(form.errors) + self.assertTrue(form.is_valid()) + self.assertTrue(form.save()) diff --git a/netbox_config_backup/tests/test_models.py b/netbox_config_backup/tests/test_models.py index c609c55..306bfe3 100644 --- a/netbox_config_backup/tests/test_models.py +++ b/netbox_config_backup/tests/test_models.py @@ -7,24 +7,33 @@ class TestBackup(TestCase): + @classmethod def setUpTestData(cls): + Site.objects.create(name='Site 1', slug='site-1') + manufacturer = Manufacturer.objects.create(name='Manufacturer 1', slug='manufacturer-1') + DeviceType.objects.create(model='Generic Type', slug='generic-type', manufacturer=manufacturer) + DeviceRole.objects.create(name='Generic Role', slug='generic-role') + + def test_create_backup(self): + configs = { + 'running': 'Test Backup', + 'startup': 'Test Backup' + } + + site = Site.objects.first() + role = DeviceRole.objects.first() + device_type = DeviceType.objects.first() - site = Site.objects.create(name='Site 1') - manufacturer = Manufacturer.objects.create(name='Manufacturer 1') - device_type = DeviceType.objects.create(model='Device Type 1', manufacturer=manufacturer) - role = DeviceRole.objects.create(name='Switch') device = Device.objects.create( - name='Device 1', - site=site, + name='Test Device', device_type=device_type, role=role, - status='active' + site=site ) - interface = Interface.objects.create(name='Interface 1', device=device, type='1000baset') - address = IPAddress.objects.create(assigned_object=interface, address='10.0.0.1/32') - device.primary_ip4 = address - device.save() + backup = Backup.objects.create(name='Backup 1', device=device) + backup.set_config(configs) + retrieved = backup.get_config() - def test_create_backup(self): - pass + self.assertEqual(configs['running'], retrieved['running']) + self.assertEqual(configs['startup'], retrieved['startup']) diff --git a/netbox_config_backup/tests/test_views.py b/netbox_config_backup/tests/test_views.py new file mode 100644 index 0000000..bef6c7d --- /dev/null +++ b/netbox_config_backup/tests/test_views.py @@ -0,0 +1,63 @@ +from dcim.models import Device +from extras.models import Tag +from utilities.testing import ViewTestCases, create_tags, create_test_device + +from netbox_config_backup.models import Backup + + +class BackupTestCase( + ViewTestCases.GetObjectViewTestCase, + ViewTestCases.GetObjectChangelogViewTestCase, + ViewTestCases.CreateObjectViewTestCase, + ViewTestCases.EditObjectViewTestCase, + ViewTestCases.DeleteObjectViewTestCase, + ViewTestCases.ListObjectsViewTestCase, + ViewTestCases.BulkEditObjectsViewTestCase, + ViewTestCases.BulkDeleteObjectsViewTestCase +): + # ViewTestCases.BulkImportObjectsViewTestCase, + model = Backup + + @classmethod + def setUpTestData(cls): + devices = ( + create_test_device(name="Device 1"), + create_test_device(name="Device 2"), + create_test_device(name="Device 3"), + create_test_device(name="Device 4"), + ) + + backups = ( + Backup(name="Backup 1", device=devices[0]), + Backup(name="Backup 2", device=devices[1]), + Backup(name="Backup 3", device=devices[2]), + ) + Backup.objects.bulk_create(backups) + + cls.form_data = { + 'name': 'Backup X', + 'status': 'disabled' + } + + cls.bulk_edit_data = { + 'description': 'A description', + } + + """ + cls.csv_data = ( + "name,slug,description", + "Region 4,region-4,Fourth region", + "Region 5,region-5,Fifth region", + "Region 6,region-6,Sixth region", + ) + + cls.csv_update_data = ( + "id,name,description", + f"{regions[0].pk},Region 7,Fourth region7", + f"{regions[1].pk},Region 8,Fifth region8", + f"{regions[2].pk},Region 0,Sixth region9", + ) + """ + + def _get_base_url(self): + return 'plugins:netbox_config_backup:backup_{}' From 3c193c4f5f5f58b526b66847adda48ee6c7477ff Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 12 Sep 2024 09:08:44 -0500 Subject: [PATCH 17/25] Remove display_url from serializer --- netbox_config_backup/api/serializers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/netbox_config_backup/api/serializers.py b/netbox_config_backup/api/serializers.py index c07cd2e..6f90417 100644 --- a/netbox_config_backup/api/serializers.py +++ b/netbox_config_backup/api/serializers.py @@ -19,7 +19,7 @@ class BackupSerializer(NetBoxModelSerializer): class Meta: model = Backup fields = [ - 'id', 'url', 'display_url', 'display', 'name', 'device', 'ip', + 'id', 'url', 'display', 'name', 'device', 'ip', 'uuid', 'status', 'config_status', ] brief_fields = ('display', 'id', 'name', 'url') From 96af3f27566933326ec629b6d04f9934b994933e Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 12 Sep 2024 09:21:08 -0500 Subject: [PATCH 18/25] Add migration for Primary model --- ...0015_backup_comments_backup_description.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 netbox_config_backup/migrations/0015_backup_comments_backup_description.py diff --git a/netbox_config_backup/migrations/0015_backup_comments_backup_description.py b/netbox_config_backup/migrations/0015_backup_comments_backup_description.py new file mode 100644 index 0000000..43afa32 --- /dev/null +++ b/netbox_config_backup/migrations/0015_backup_comments_backup_description.py @@ -0,0 +1,23 @@ +# Generated by Django 5.0.8 on 2024-09-12 13:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('netbox_config_backup', '0014_backup_config_status'), + ] + + operations = [ + migrations.AddField( + model_name='backup', + name='comments', + field=models.TextField(blank=True), + ), + migrations.AddField( + model_name='backup', + name='description', + field=models.CharField(blank=True, max_length=200), + ), + ] From d9530d568a68f6c0c9f9f096dae048040209a6cf Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Thu, 12 Sep 2024 18:39:43 -0500 Subject: [PATCH 19/25] Add napalm to the config for CI --- .github/configuration.testing.py | 1 + netbox_config_backup/models/abstract.py | 1 + netbox_config_backup/utils/configs.py | 4 ---- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/configuration.testing.py b/.github/configuration.testing.py index 595e4a5..7c5362f 100644 --- a/.github/configuration.testing.py +++ b/.github/configuration.testing.py @@ -15,6 +15,7 @@ } PLUGINS = [ + 'netbox_napalm_plugin', 'netbox_config_backup', ] diff --git a/netbox_config_backup/models/abstract.py b/netbox_config_backup/models/abstract.py index 3ede8d3..0b4cfb7 100644 --- a/netbox_config_backup/models/abstract.py +++ b/netbox_config_backup/models/abstract.py @@ -3,5 +3,6 @@ class BigIDModel(models.Model): id = models.BigAutoField(primary_key=True) + class Meta: abstract = True diff --git a/netbox_config_backup/utils/configs.py b/netbox_config_backup/utils/configs.py index 4cda63a..336a386 100644 --- a/netbox_config_backup/utils/configs.py +++ b/netbox_config_backup/utils/configs.py @@ -58,18 +58,14 @@ def check_config_save_status(d): dates[file] = date datetimes[file] = datetime.strptime(date, '%Y-%b-%d %H:%M:%S') - - # logger.debug(f'\t{file}: {result} ({date}) ({datetimes[file]})') else: logger.debug(f'\tNo {file} time found, platform: {d.platform}') - # logger.debug(f'\t{file}: {result}') if datetimes['running'] is None and datetimes['startup'] is not None: logger.debug(f'\tValid backup as booted from startup') return True elif datetimes['startup'] is None: logger.debug(f'\tNo startup time') - # logger.info(f'{datetimes["running"]} ({dates["running"]}): {datetimes["startup"]} ({dates["startup"]})') return elif datetimes['running'] <= datetimes['startup']: logger.debug(f'\tRunning config less then startup') From bb726632479d3b61ed767e27eac7ed23d1f8d543 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 13 Sep 2024 08:27:27 -0500 Subject: [PATCH 20/25] Fix configuration.testing.py --- .github/configuration.testing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/configuration.testing.py b/.github/configuration.testing.py index 7c5362f..0588711 100644 --- a/.github/configuration.testing.py +++ b/.github/configuration.testing.py @@ -26,6 +26,10 @@ 'author': 'Test Committer ', 'frequency': 3600, }, + 'netbox_napalm_plugin': { + 'NAPALM_USERNAME': 'xxx', + 'NAPALM_PASSWORD': 'yyy', + } } REDIS = { From a5cde3d01cd0f8d416fd0a74ec9b5be94d7f7bdc Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 13 Sep 2024 08:28:01 -0500 Subject: [PATCH 21/25] Upgrade to pyproject.toml and improve CI and build process --- .github/workflows/build-test.yml | 25 +++++++++++ .github/workflows/pypi.yml | 4 +- netbox_config_backup/tasks.py | 72 +++++++++++++++++++------------- pyproject.toml | 42 +++++++++++++++++++ 4 files changed, 113 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/build-test.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 0000000..e1f0076 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,25 @@ +name: Build Test +on: [push, pull_request] +jobs: + pypi-publish: + name: Test Build Process + runs-on: ubuntu-latest + environment: release + permissions: + id-token: write + strategy: + matrix: + python-version: [3.12] + steps: + - name: Checkout repo + uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install --upgrade setuptools wheel + - name: Build + run: python -m build diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml index f41d778..b3943ee 100644 --- a/.github/workflows/pypi.yml +++ b/.github/workflows/pypi.yml @@ -11,7 +11,7 @@ jobs: id-token: write strategy: matrix: - python-version: [3.9] + python-version: [3.12] steps: - name: Checkout repo uses: actions/checkout@v2 @@ -24,7 +24,7 @@ jobs: python -m pip install --upgrade pip pip install --upgrade setuptools wheel - name: Build - run: python setup.py sdist bdist_wheel + run: python -m build #- name: Publish package to TestPyPI # uses: pypa/gh-action-pypi-publish@release/v1 # with: diff --git a/netbox_config_backup/tasks.py b/netbox_config_backup/tasks.py index 58532fa..0643139 100644 --- a/netbox_config_backup/tasks.py +++ b/netbox_config_backup/tasks.py @@ -15,10 +15,21 @@ def napalm_init(device, ip=None, extra_args={}): from netbox import settings - username = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_USERNAME', None) - password = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_PASSWORD', None) - timeout = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_TIMEOUT', None) - optional_args = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get('NAPALM_ARGS', []).copy() + + username = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get( + 'NAPALM_USERNAME', None + ) + password = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get( + 'NAPALM_PASSWORD', None + ) + timeout = settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}).get( + 'NAPALM_TIMEOUT', None + ) + optional_args = ( + settings.PLUGINS_CONFIG.get('netbox_napalm_plugin', {}) + .get('NAPALM_ARGS', []) + .copy() + ) if device and device.platform and device.platform.napalm.napalm_args is not None: optional_args.update(device.platform.napalm.napalm_args) @@ -31,9 +42,7 @@ def napalm_init(device, ip=None, extra_args={}): elif device.primary_ip and device.primary_ip is not None: host = str(device.primary_ip.address.ip) else: - raise ServiceUnavailable( - "This device does not have a primary IP address" - ) + raise ServiceUnavailable("This device does not have a primary IP address") # Check that NAPALM is installed try: @@ -41,16 +50,20 @@ def napalm_init(device, ip=None, extra_args={}): from napalm.base.exceptions import ModuleImportError except ModuleNotFoundError as e: if getattr(e, 'name') == 'napalm': - raise ServiceUnavailable("NAPALM is not installed. Please see the documentation for instructions.") + raise ServiceUnavailable( + "NAPALM is not installed. Please see the documentation for instructions." + ) raise e # Validate the configured driver try: driver = napalm.get_network_driver(device.platform.napalm.napalm_driver) except ModuleImportError: - raise ServiceUnavailable("NAPALM driver for platform {} not found: {}.".format( - device.platform, device.platform.napalm.napalm_driver - )) + raise ServiceUnavailable( + "NAPALM driver for platform {} not found: {}.".format( + device.platform, device.platform.napalm.napalm_driver + ) + ) # Connect to the device d = driver( @@ -58,7 +71,7 @@ def napalm_init(device, ip=None, extra_args={}): username=username, password=password, timeout=timeout, - optional_args=optional_args + optional_args=optional_args, ) try: d.open() @@ -67,7 +80,9 @@ def napalm_init(device, ip=None, extra_args={}): logger.info('Authentication error') elif isinstance(e, NetmikoTimeoutException): logger.info('Connection error') - raise ServiceUnavailable("Error connecting to the device at {}: {}".format(host, e)) + raise ServiceUnavailable( + "Error connecting to the device at {}: {}".format(host, e) + ) return d @@ -83,9 +98,9 @@ def backup_config(backup, pk=None): if backup.device is not None and ip is not None: logger.info(f'{backup}: Backup started') - #logger.debug(f'[{pk}] Connecting') + # logger.debug(f'[{pk}] Connecting') d = napalm_init(backup.device, ip) - #logger.debug(f'[{pk} + # logger.debug(f'[{pk} try: status = check_config_save_status(d) @@ -105,13 +120,13 @@ def backup_config(backup, pk=None): except Exception as e: logger.error(f'{backup}: had error setting backup status: {e}') - #logger.debug(f'[{pk}] Getting config') + # logger.debug(f'[{pk}] Getting config') configs = d.get_config() - #logger.debug(f'[{pk}] Finished config get') + # logger.debug(f'[{pk}] Finished config get') - #logger.debug(f'[{pk}] Setting config') + # logger.debug(f'[{pk}] Setting config') commit = backup.set_config(configs, pk=pk) - #logger.debug(f'[{pk}] Finished config set') + # logger.debug(f'[{pk}] Finished config set') d.close() logger.info(f'{backup}: Backup complete') @@ -123,6 +138,7 @@ def backup_config(backup, pk=None): def backup_job(pk): import netmiko + try: job_result = BackupJob.objects.get(pk=pk) except BackupJob.DoesNotExist: @@ -133,29 +149,29 @@ def backup_job(pk): if not can_backup(backup): logger.warning(f'Cannot backup due to additional factors') return 1 - delay = timedelta(seconds=settings.PLUGINS_CONFIG.get('netbox_config_backup', {}).get('frequency')) + delay = timedelta( + seconds=settings.PLUGINS_CONFIG.get('netbox_config_backup', {}).get('frequency') + ) job_result.started = timezone.now() job_result.status = JobStatusChoices.STATUS_RUNNING job_result.save() try: - #logger.debug(f'[{pk}] Starting backup') + # logger.debug(f'[{pk}] Starting backup') commit = backup_config(backup, pk=pk) - #logger.debug(f'[{pk}] Finished backup') + # logger.debug(f'[{pk}] Finished backup') job_result.set_status(JobStatusChoices.STATUS_COMPLETED) job_result.data = {'commit': f'{commit}' if commit is not None else ''} job_result.set_status(JobStatusChoices.STATUS_COMPLETED) # Enqueue next job if one doesn't exist try: - #logger.debug(f'[{pk}] Starting Enqueue') - BackupJob.objects.filter( - backup=backup - ).exclude( + # logger.debug(f'[{pk}] Starting Enqueue') + BackupJob.objects.filter(backup=backup).exclude( status__in=JobStatusChoices.TERMINAL_STATE_CHOICES ).update(status=JobStatusChoices.STATUS_FAILED) BackupJob.enqueue_if_needed(backup, delay=delay, job_id=job_result.job_id) - #logger.debug(f'[{pk}] Finished Enqueue') + # logger.debug(f'[{pk}] Finished Enqueue') except Exception as e: logger.error(f'Job Enqueue after completion failed for job: {backup}') logger.error(f'\tException: {e}') @@ -172,7 +188,7 @@ def backup_job(pk): job_result.set_status(JobStatusChoices.STATUS_ERRORED) BackupJob.enqueue_if_needed(backup, delay=delay, job_id=job_result.job_id) - #logger.debug(f'[{pk}] Saving result') + # logger.debug(f'[{pk}] Saving result') job_result.save() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..e333500 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "netbox-config-backup" +authors = [ + {name = "Daniel Sheppard", email = "dans@dansheps.com"} +] +maintainers = [ + {name = "Daniel Sheppard", email = "dans@dansheps.com"}, +] +description = "Plugin to backup device configuration" +readme = "README.md" +requires-python = ">=3.10" +keywords = ["netbox-plugin", ] +version = "0.0.1" +license = {file = "LICENSE"} +classifiers = [ + "Programming Language :: Python :: 3", +] +dependencies = [ + 'netbox-napalm-plugin', + 'netmiko>=4.0.0', + 'napalm', + 'uuid', + 'dulwich', + 'pydriller', + 'deepdiff', +] + +[project.urls] +Documentation = "https://github.com/dansheps/netbox-config-backup/blob/main/README.md" +Source = "https://github.com/dansheps/netbox-config-backup" +Tracker = "https://github.com/dansheps/netbox-config-backup/issues" + +[tool.setuptools.packages.find] +include=["netbox_config_backup"] +exclude=["netbox_config_backup.tests"] + +[tool.black] +skip-string-normalization = 1 \ No newline at end of file From 0bb6c793b8df706055f995e35fd4dc3704049591 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 13 Sep 2024 08:29:51 -0500 Subject: [PATCH 22/25] Fix version statement and remove setup.py --- pyproject.toml | 2 +- setup.py | 33 --------------------------------- 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index e333500..3228e67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ description = "Plugin to backup device configuration" readme = "README.md" requires-python = ">=3.10" keywords = ["netbox-plugin", ] -version = "0.0.1" +version = "2.1.1-beta1" license = {file = "LICENSE"} classifiers = [ "Programming Language :: Python :: 3", diff --git a/setup.py b/setup.py deleted file mode 100644 index 63b09b7..0000000 --- a/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -from setuptools import find_packages, setup - -setup( - name='netbox_config_backup', - version='2.1.1-beta1', - description='NetBox Configuration Backup', - long_description='Plugin to backup device configuration', - url='https://github.com/dansheps/netbox-config-backup/', - download_url='https://github.com/dansheps/netbox-config-backup/', - author='Daniel Sheppard', - author_email='dans@dansheps.com', - maintainer='Daniel Sheppard', - maintainer_email='dans@dansheps.com', - install_requires=[ - 'netbox-napalm-plugin', - 'netmiko>=4.0.0', - 'napalm', - 'uuid', - 'dulwich', - 'pydriller', - 'deepdiff', - ], - packages=find_packages(), - include_package_data=True, - license='Proprietary', - zip_safe=False, - platform=[], - keywords=['netbox', 'netbox-plugin'], - classifiers=[ - 'Framework :: Django', - 'Programming Language :: Python :: 3', - ] -) From 3c5ed042885ece6a67166101fc9f792d93434f4d Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 13 Sep 2024 11:46:15 -0500 Subject: [PATCH 23/25] Update serializer to properly define URL --- netbox_config_backup/api/serializers.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/netbox_config_backup/api/serializers.py b/netbox_config_backup/api/serializers.py index 6f90417..4c08db2 100644 --- a/netbox_config_backup/api/serializers.py +++ b/netbox_config_backup/api/serializers.py @@ -13,6 +13,9 @@ class BackupSerializer(NetBoxModelSerializer): + url = serializers.HyperlinkedIdentityField( + view_name='plugins-api:netbox_config_backup-api:backup-detail' + ) device = DeviceSerializer(nested=True, required=False, allow_null=True), ip = IPAddressSerializer(nested=True, required=False, allow_null=True) From 2b7525c390ea919732547bea046188f6899bcbbf Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 13 Sep 2024 16:40:06 -0500 Subject: [PATCH 24/25] Fix filter testing --- netbox_config_backup/filtersets.py | 12 ++++++++---- netbox_config_backup/tests/test_filtersets.py | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/netbox_config_backup/filtersets.py b/netbox_config_backup/filtersets.py index 60a61d8..a3d273a 100644 --- a/netbox_config_backup/filtersets.py +++ b/netbox_config_backup/filtersets.py @@ -62,8 +62,14 @@ def search(self, queryset, name, value): def filter_address(self, queryset, name, value): try: - return queryset.filter(ip__address__net_host_contained=value) - except ValidationError: + if type(value) is list: + query = Q() + for val in value: + query |= Q(ip__address__net_host_contained=val) + return queryset.filter(query) + else: + return queryset.filter(ip__address__net_host_contained=value) + except ValidationError as e: return queryset.none() @@ -83,10 +89,8 @@ class Meta: fields = ['id', 'file'] def search(self, queryset, name, value): - print('Search') if not value.strip(): return queryset - print(value) qs_filter = ( Q(file__type=value) | Q(file__type__startswith=value) diff --git a/netbox_config_backup/tests/test_filtersets.py b/netbox_config_backup/tests/test_filtersets.py index 041dfa8..ecae713 100644 --- a/netbox_config_backup/tests/test_filtersets.py +++ b/netbox_config_backup/tests/test_filtersets.py @@ -25,7 +25,7 @@ def setUpTestData(cls): ) ip = IPAddress.objects.create( - address='10.10.10.10/32' + address='10.10.10.10/24' ) devices = ( @@ -54,5 +54,5 @@ def test_device(self): self.assertEqual(self.filterset(params, self.queryset).qs.count(), 2) def test_ip(self): - params = {'ip_address': ['10.10.10.10']} + params = {'ip': '10.10.10.10'} self.assertEqual(self.filterset(params, self.queryset).qs.count(), 1) \ No newline at end of file From 2b6b77c46062b337fb985914d71e486d4cc426e8 Mon Sep 17 00:00:00 2001 From: Daniel Sheppard Date: Fri, 13 Sep 2024 16:42:57 -0500 Subject: [PATCH 25/25] Update build test workflow --- .github/workflows/build-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index e1f0076..43a2a5f 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -21,5 +21,7 @@ jobs: run: | python -m pip install --upgrade pip pip install --upgrade setuptools wheel + - name: Install pypa/build + run: python3 -m pip install build --user - name: Build run: python -m build