Closes #9073: Remote data support for config contexts (#11692)

* WIP

* Add bulk sync view for config contexts

* Introduce 'sync' permission for synced data models

* Docs & cleanup

* Remove unused method

* Add a REST API endpoint to synchronize config context data

Author: jeremystretch

docs/models/extras/configcontext.md

@@ -18,6 +18,10 @@ A numeric value which influences the order in which context data is merged. Cont
 
 The context data expressed in JSON format.
 
+### Data File
+
+Config context data may optionally be sourced from a remote [data file](../core/datafile.md), which is synchronized from a remote data source. When designating a data file, there is no need to specify local data for the config context: It will be populated automatically from the data file.
+
 ### Is Active
 
 If not selected, this config context will be excluded from rendering. This can be convenient to temporarily disable a config context.

docs/release-notes/version-3.5.md

@@ -4,6 +4,7 @@
 
 ### Enhancements
 
+* [#9073](https://github.com/netbox-community/netbox/issues/9073) - Enable syncing config context data from remote sources
 * [#11254](https://github.com/netbox-community/netbox/issues/11254) - Introduce the `X-Request-ID` HTTP header to annotate the unique ID of each request for change logging
 * [#11440](https://github.com/netbox-community/netbox/issues/11440) - Add an `enabled` field for device type interfaces
 * [#11517](https://github.com/netbox-community/netbox/issues/11517) - Standardize the inclusion of related objects across the entire UI

netbox/core/models/data.py

@@ -1,5 +1,6 @@
 import logging
 import os
+import yaml
 from fnmatch import fnmatchcase
 from urllib.parse import urlparse
 
@@ -283,6 +284,13 @@ def data_as_string(self):
         except UnicodeDecodeError:
             return None
 
+    def get_data(self):
+        """
+        Attempt to read the file data as JSON/YAML and return a native Python object.
+        """
+        # TODO: Something more robust
+        return yaml.safe_load(self.data_as_string)
+
     def refresh_from_disk(self, source_root):
         """
         Update instance attributes from the file on disk. Returns True if any attribute

netbox/extras/api/serializers.py

@@ -4,6 +4,7 @@
 from drf_yasg.utils import swagger_serializer_method
 from rest_framework import serializers
 
+from core.api.nested_serializers import NestedDataSourceSerializer, NestedDataFileSerializer
 from dcim.api.nested_serializers import (
     NestedDeviceRoleSerializer, NestedDeviceTypeSerializer, NestedLocationSerializer, NestedPlatformSerializer,
     NestedRegionSerializer, NestedSiteSerializer, NestedSiteGroupSerializer,
@@ -358,13 +359,20 @@ class ConfigContextSerializer(ValidatedModelSerializer):
         required=False,
         many=True
     )
+    data_source = NestedDataSourceSerializer(
+        required=False
+    )
+    data_file = NestedDataFileSerializer(
+        read_only=True
+    )
 
     class Meta:
         model = ConfigContext
         fields = [
             'id', 'url', 'display', 'name', 'weight', 'description', 'is_active', 'regions', 'site_groups', 'sites',
             'locations', 'device_types', 'roles', 'platforms', 'cluster_types', 'cluster_groups', 'clusters',
-            'tenant_groups', 'tenants', 'tags', 'data', 'created', 'last_updated',
+            'tenant_groups', 'tenants', 'tags', 'data_source', 'data_path', 'data_file', 'data_synced', 'data',
+            'created', 'last_updated',
         ]
 
 

netbox/extras/api/views.py

@@ -17,6 +17,7 @@
 from extras.reports import get_report, get_reports, run_report
 from extras.scripts import get_script, get_scripts, run_script
 from netbox.api.authentication import IsAuthenticatedOrLoginNotRequired
+from netbox.api.features import SyncedDataMixin
 from netbox.api.metadata import ContentTypeMetadata
 from netbox.api.viewsets import NetBoxModelViewSet
 from utilities.exceptions import RQWorkerNotRunningException
@@ -147,9 +148,10 @@ class JournalEntryViewSet(NetBoxModelViewSet):
 # Config contexts
 #
 
-class ConfigContextViewSet(NetBoxModelViewSet):
+class ConfigContextViewSet(SyncedDataMixin, NetBoxModelViewSet):
     queryset = ConfigContext.objects.prefetch_related(
-        'regions', 'site_groups', 'sites', 'locations', 'roles', 'platforms', 'tenant_groups', 'tenants',
+        'regions', 'site_groups', 'sites', 'locations', 'roles', 'platforms', 'tenant_groups', 'tenants', 'data_source',
+        'data_file',
     )
     serializer_class = serializers.ConfigContextSerializer
     filterset_class = filtersets.ConfigContextFilterSet

netbox/extras/constants.py

@@ -8,6 +8,7 @@
     'export_templates',
     'job_results',
     'journaling',
+    'synced_data',
     'tags',
     'webhooks'
 ]

netbox/extras/filtersets.py

@@ -4,6 +4,7 @@
 from django.db.models import Q
 from django.utils.translation import gettext as _
 
+from core.models import DataFile, DataSource
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
 from tenancy.models import Tenant, TenantGroup
@@ -422,10 +423,18 @@ class ConfigContextFilterSet(ChangeLoggedModelFilterSet):
         to_field_name='slug',
         label=_('Tag (slug)'),
     )
+    data_source_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data source (ID)'),
+    )
+    data_file_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data file (ID)'),
+    )
 
     class Meta:
         model = ConfigContext
-        fields = ['id', 'name', 'is_active']
+        fields = ['id', 'name', 'is_active', 'data_synced']
 
     def search(self, queryset, name, value):
         if not value.strip():

netbox/extras/forms/filtersets.py

@@ -3,6 +3,7 @@
 from django.contrib.contenttypes.models import ContentType
 from django.utils.translation import gettext as _
 
+from core.models import DataFile, DataSource
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from extras.choices import *
 from extras.models import *
@@ -263,11 +264,25 @@ class TagFilterForm(SavedFiltersMixin, FilterForm):
 class ConfigContextFilterForm(SavedFiltersMixin, FilterForm):
     fieldsets = (
         (None, ('q', 'filter_id', 'tag_id')),
+        ('Data', ('data_source_id', 'data_file_id')),
         ('Location', ('region_id', 'site_group_id', 'site_id', 'location_id')),
         ('Device', ('device_type_id', 'platform_id', 'role_id')),
         ('Cluster', ('cluster_type_id', 'cluster_group_id', 'cluster_id')),
         ('Tenant', ('tenant_group_id', 'tenant_id'))
     )
+    data_source_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file_id = DynamicModelMultipleChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('Data file'),
+        query_params={
+            'source_id': '$data_source_id'
+        }
+    )
     region_id = DynamicModelMultipleChoiceField(
         queryset=Region.objects.all(),
         required=False,

netbox/extras/forms/mixins.py

@@ -2,13 +2,15 @@
 from django import forms
 from django.utils.translation import gettext as _
 
+from core.models import DataFile, DataSource
 from extras.models import *
 from extras.choices import CustomFieldVisibilityChoices
-from utilities.forms.fields import DynamicModelMultipleChoiceField
+from utilities.forms.fields import DynamicModelChoiceField, DynamicModelMultipleChoiceField
 
 __all__ = (
     'CustomFieldsMixin',
     'SavedFiltersMixin',
+    'SyncedDataMixin',
 )
 
 
@@ -72,3 +74,19 @@ class SavedFiltersMixin(forms.Form):
             'usable': True,
         }
     )
+
+
+class SyncedDataMixin(forms.Form):
+    data_source = DynamicModelChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file = DynamicModelChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('File'),
+        query_params={
+            'source_id': '$data_source',
+        }
+    )

netbox/extras/forms/model_forms.py

@@ -5,6 +5,7 @@
 
 from dcim.models import DeviceRole, DeviceType, Location, Platform, Region, Site, SiteGroup
 from extras.choices import *
+from extras.forms.mixins import SyncedDataMixin
 from extras.models import *
 from extras.utils import FeatureQuery
 from netbox.forms import NetBoxModelForm
@@ -183,7 +184,7 @@ class Meta:
         ]
 
 
-class ConfigContextForm(BootstrapMixin, forms.ModelForm):
+class ConfigContextForm(BootstrapMixin, SyncedDataMixin, forms.ModelForm):
     regions = DynamicModelMultipleChoiceField(
         queryset=Region.objects.all(),
         required=False
@@ -236,10 +237,13 @@ class ConfigContextForm(BootstrapMixin, forms.ModelForm):
         queryset=Tag.objects.all(),
         required=False
     )
-    data = JSONField()
+    data = JSONField(
+        required=False
+    )
 
     fieldsets = (
         ('Config Context', ('name', 'weight', 'description', 'data', 'is_active')),
+        ('Data Source', ('data_source', 'data_file')),
         ('Assignment', (
             'regions', 'site_groups', 'sites', 'locations', 'device_types', 'roles', 'platforms', 'cluster_types',
             'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'tags',
@@ -251,9 +255,17 @@ class Meta:
         fields = (
             'name', 'weight', 'description', 'data', 'is_active', 'regions', 'site_groups', 'sites', 'locations',
             'roles', 'device_types', 'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups',
-            'tenants', 'tags',
+            'tenants', 'tags', 'data_source', 'data_file',
         )
 
+    def clean(self):
+        super().clean()
+
+        if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_source'):
+            raise forms.ValidationError("Must specify either local data or a data source")
+
+        return self.cleaned_data
+
 
 class ImageAttachmentForm(BootstrapMixin, forms.ModelForm):
 

netbox/extras/migrations/0085_configcontext_synced_data.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.6 on 2023-02-06 15:34
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+        ('extras', '0084_staging'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_file',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile'),
+        ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_path',
+            field=models.CharField(blank=True, editable=False, max_length=1000),
+        ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'),
+        ),
+        migrations.AddField(
+            model_name='configcontext',
+            name='data_synced',
+            field=models.DateTimeField(blank=True, editable=False, null=True),
+        ),
+    ]

netbox/extras/models/configcontexts.py

@@ -2,10 +2,11 @@
 from django.core.validators import ValidationError
 from django.db import models
 from django.urls import reverse
+from django.utils import timezone
 
 from extras.querysets import ConfigContextQuerySet
 from netbox.models import ChangeLoggedModel
-from netbox.models.features import WebhooksMixin
+from netbox.models.features import SyncedDataMixin, WebhooksMixin
 from utilities.utils import deepmerge
 
 
@@ -19,7 +20,7 @@
 # Config contexts
 #
 
-class ConfigContext(WebhooksMixin, ChangeLoggedModel):
+class ConfigContext(SyncedDataMixin, WebhooksMixin, ChangeLoggedModel):
     """
     A ConfigContext represents a set of arbitrary data available to any Device or VirtualMachine matching its assigned
     qualifiers (region, site, etc.). For example, the data stored in a ConfigContext assigned to site A and tenant B
@@ -130,6 +131,13 @@ def clean(self):
                 {'data': 'JSON data must be in object form. Example: {"foo": 123}'}
             )
 
+    def sync_data(self):
+        """
+        Synchronize context data from the designated DataFile (if any).
+        """
+        self.data = self.data_file.get_data()
+        self.data_synced = timezone.now()
+
 
 class ConfigContextModel(models.Model):
     """

netbox/extras/tables/tables.py

@@ -188,21 +188,30 @@ class Meta(NetBoxTable.Meta):
 
 
 class ConfigContextTable(NetBoxTable):
+    data_source = tables.Column(
+        linkify=True
+    )
+    data_file = tables.Column(
+        linkify=True
+    )
     name = tables.Column(
         linkify=True
     )
     is_active = columns.BooleanColumn(
         verbose_name='Active'
     )
+    is_synced = columns.BooleanColumn(
+        verbose_name='Synced'
+    )
 
     class Meta(NetBoxTable.Meta):
         model = ConfigContext
         fields = (
-            'pk', 'id', 'name', 'weight', 'is_active', 'description', 'regions', 'sites', 'locations', 'roles',
-            'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups', 'tenants', 'created',
-            'last_updated',
+            'pk', 'id', 'name', 'weight', 'is_active', 'is_synced', 'description', 'regions', 'sites', 'locations',
+            'roles', 'platforms', 'cluster_types', 'cluster_groups', 'clusters', 'tenant_groups', 'tenants',
+            'data_source', 'data_file', 'data_synced', 'created', 'last_updated',
         )
-        default_columns = ('pk', 'name', 'weight', 'is_active', 'description')
+        default_columns = ('pk', 'name', 'weight', 'is_active', 'is_synced', 'description')
 
 
 class ObjectChangeTable(NetBoxTable):

netbox/extras/urls.py

@@ -60,6 +60,7 @@
     path('config-contexts/add/', views.ConfigContextEditView.as_view(), name='configcontext_add'),
     path('config-contexts/edit/', views.ConfigContextBulkEditView.as_view(), name='configcontext_bulk_edit'),
     path('config-contexts/delete/', views.ConfigContextBulkDeleteView.as_view(), name='configcontext_bulk_delete'),
+    path('config-contexts/sync/', views.ConfigContextBulkSyncDataView.as_view(), name='configcontext_bulk_sync'),
     path('config-contexts/<int:pk>/', include(get_model_urls('extras', 'configcontext'))),
 
     # Image attachments

netbox/extras/views.py

@@ -352,7 +352,8 @@ class ConfigContextListView(generic.ObjectListView):
     filterset = filtersets.ConfigContextFilterSet
     filterset_form = forms.ConfigContextFilterForm
     table = tables.ConfigContextTable
-    actions = ('add', 'bulk_edit', 'bulk_delete')
+    template_name = 'extras/configcontext_list.html'
+    actions = ('add', 'bulk_edit', 'bulk_delete', 'bulk_sync')
 
 
 @register_model_view(ConfigContext)
@@ -416,6 +417,10 @@ class ConfigContextBulkDeleteView(generic.BulkDeleteView):
     table = tables.ConfigContextTable
 
 
+class ConfigContextBulkSyncDataView(generic.BulkSyncDataView):
+    queryset = ConfigContext.objects.all()
+
+
 class ObjectConfigContextView(generic.ObjectView):
     base_template = None
     template_name = 'extras/object_configcontext.html'