Closes netbox-community#11693: Enable remote data synchronization for export templates

Author: jeremystretch

docs/models/extras/exporttemplate.md

@@ -12,6 +12,10 @@ The name of the export template. This will appear in the "export" dropdown list
 
 The type of NetBox object to which the export template applies.
 
+### Data File
+
+Template code may optionally be sourced from a remote [data file](../core/datafile.md), which is synchronized from a remote data source. When designating a data file, there is no need to specify local content for the template: It will be populated automatically from the data file.
+
 ### Template Code
 
 Jinja2 template code for rendering the exported data.

netbox/extras/api/serializers.py

@@ -142,12 +142,19 @@ class ExportTemplateSerializer(ValidatedModelSerializer):
         queryset=ContentType.objects.filter(FeatureQuery('export_templates').get_query()),
         many=True
     )
+    data_source = NestedDataSourceSerializer(
+        required=False
+    )
+    data_file = NestedDataFileSerializer(
+        read_only=True
+    )
 
     class Meta:
         model = ExportTemplate
         fields = [
             'id', 'url', 'display', 'content_types', 'name', 'description', 'template_code', 'mime_type',
-            'file_extension', 'as_attachment', 'created', 'last_updated',
+            'file_extension', 'as_attachment', 'data_source', 'data_path', 'data_file', 'data_synced', 'created',
+            'last_updated',
         ]
 
 

netbox/extras/api/views.py

@@ -92,9 +92,9 @@ class CustomLinkViewSet(NetBoxModelViewSet):
 # Export templates
 #
 
-class ExportTemplateViewSet(NetBoxModelViewSet):
+class ExportTemplateViewSet(SyncedDataMixin, NetBoxModelViewSet):
     metadata_class = ContentTypeMetadata
-    queryset = ExportTemplate.objects.all()
+    queryset = ExportTemplate.objects.prefetch_related('data_source', 'data_file')
     serializer_class = serializers.ExportTemplateSerializer
     filterset_class = filtersets.ExportTemplateFilterSet
 

netbox/extras/filtersets.py

@@ -127,10 +127,18 @@ class ExportTemplateFilterSet(BaseFilterSet):
         field_name='content_types__id'
     )
     content_types = ContentTypeFilter()
+    data_source_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data source (ID)'),
+    )
+    data_file_id = django_filters.ModelMultipleChoiceFilter(
+        queryset=DataSource.objects.all(),
+        label=_('Data file (ID)'),
+    )
 
     class Meta:
         model = ExportTemplate
-        fields = ['id', 'content_types', 'name', 'description']
+        fields = ['id', 'content_types', 'name', 'description', 'data_synced']
 
     def search(self, queryset, name, value):
         if not value.strip():

netbox/extras/forms/filtersets.py

@@ -21,9 +21,9 @@
 __all__ = (
     'ConfigContextFilterForm',
     'CustomFieldFilterForm',
-    'JobResultFilterForm',
     'CustomLinkFilterForm',
     'ExportTemplateFilterForm',
+    'JobResultFilterForm',
     'JournalEntryFilterForm',
     'LocalConfigContextFilterForm',
     'ObjectChangeFilterForm',
@@ -160,8 +160,22 @@ class CustomLinkFilterForm(SavedFiltersMixin, FilterForm):
 class ExportTemplateFilterForm(SavedFiltersMixin, FilterForm):
     fieldsets = (
         (None, ('q', 'filter_id')),
+        ('Data', ('data_source_id', 'data_file_id')),
         ('Attributes', ('content_types', 'mime_type', 'file_extension', 'as_attachment')),
     )
+    data_source_id = DynamicModelMultipleChoiceField(
+        queryset=DataSource.objects.all(),
+        required=False,
+        label=_('Data source')
+    )
+    data_file_id = DynamicModelMultipleChoiceField(
+        queryset=DataFile.objects.all(),
+        required=False,
+        label=_('Data file'),
+        query_params={
+            'source_id': '$data_source_id'
+        }
+    )
     content_types = ContentTypeMultipleChoiceField(
         queryset=ContentType.objects.all(),
         limit_choices_to=FeatureQuery('export_templates'),

netbox/extras/forms/model_forms.py

@@ -96,19 +96,28 @@ class ExportTemplateForm(BootstrapMixin, forms.ModelForm):
         queryset=ContentType.objects.all(),
         limit_choices_to=FeatureQuery('export_templates')
     )
+    template_code = forms.CharField(
+        required=False,
+        widget=forms.Textarea(attrs={'class': 'font-monospace'})
+    )
 
     fieldsets = (
         ('Export Template', ('name', 'content_types', 'description')),
-        ('Template', ('template_code',)),
+        ('Content', ('data_source', 'data_file', 'template_code',)),
         ('Rendering', ('mime_type', 'file_extension', 'as_attachment')),
     )
 
     class Meta:
         model = ExportTemplate
         fields = '__all__'
-        widgets = {
-            'template_code': forms.Textarea(attrs={'class': 'font-monospace'}),
-        }
+
+    def clean(self):
+        super().clean()
+
+        if not self.cleaned_data.get('template_code') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must specify either local content or a data file")
+
+        return self.cleaned_data
 
 
 class SavedFilterForm(BootstrapMixin, forms.ModelForm):
@@ -261,8 +270,8 @@ class Meta:
     def clean(self):
         super().clean()
 
-        if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_source'):
-            raise forms.ValidationError("Must specify either local data or a data source")
+        if not self.cleaned_data.get('data') and not self.cleaned_data.get('data_file'):
+            raise forms.ValidationError("Must specify either local data or a data file")
 
         return self.cleaned_data
 

netbox/extras/migrations/0086_exporttemplate_synced_data.py

@@ -0,0 +1,35 @@
+# Generated by Django 4.1.6 on 2023-02-08 22:16
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0001_initial'),
+        ('extras', '0085_configcontext_synced_data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_file',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='core.datafile'),
+        ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_path',
+            field=models.CharField(blank=True, editable=False, max_length=1000),
+        ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_source',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='+', to='core.datasource'),
+        ),
+        migrations.AddField(
+            model_name='exporttemplate',
+            name='data_synced',
+            field=models.DateTimeField(blank=True, editable=False, null=True),
+        ),
+    ]

netbox/extras/models/models.py

@@ -26,7 +26,8 @@
 from netbox.constants import RQ_QUEUE_DEFAULT
 from netbox.models import ChangeLoggedModel
 from netbox.models.features import (
-    CloningMixin, CustomFieldsMixin, CustomLinksMixin, ExportTemplatesMixin, JobResultsMixin, TagsMixin, WebhooksMixin,
+    CloningMixin, CustomFieldsMixin, CustomLinksMixin, ExportTemplatesMixin, JobResultsMixin, SyncedDataMixin,
+    TagsMixin, WebhooksMixin,
 )
 from utilities.querysets import RestrictedQuerySet
 from utilities.utils import render_jinja2
@@ -281,7 +282,7 @@ def render(self, context):
         }
 
 
-class ExportTemplate(ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel):
+class ExportTemplate(SyncedDataMixin, ExportTemplatesMixin, WebhooksMixin, ChangeLoggedModel):
     content_types = models.ManyToManyField(
         to=ContentType,
         related_name='export_templates',
@@ -335,6 +336,13 @@ def clean(self):
                 'name': f'"{self.name}" is a reserved name. Please choose a different name.'
             })
 
+    def sync_data(self):
+        """
+        Synchronize template content from the designated DataFile (if any).
+        """
+        self.template_code = self.data_file.data_as_string
+        self.data_synced = timezone.now()
+
     def render(self, queryset):
         """
         Render the contents of the template.

netbox/extras/tables/tables.py

@@ -90,15 +90,24 @@ class ExportTemplateTable(NetBoxTable):
     )
     content_types = columns.ContentTypesColumn()
     as_attachment = columns.BooleanColumn()
+    data_source = tables.Column(
+        linkify=True
+    )
+    data_file = tables.Column(
+        linkify=True
+    )
+    is_synced = columns.BooleanColumn(
+        verbose_name='Synced'
+    )
 
     class Meta(NetBoxTable.Meta):
         model = ExportTemplate
         fields = (
             'pk', 'id', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment',
-            'created', 'last_updated',
+            'data_source', 'data_file', 'data_synced', 'created', 'last_updated',
         )
         default_columns = (
-            'pk', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment',
+            'pk', 'name', 'content_types', 'description', 'mime_type', 'file_extension', 'as_attachment', 'is_synced',
         )
 
 

netbox/extras/urls.py

@@ -29,6 +29,7 @@
     path('export-templates/import/', views.ExportTemplateBulkImportView.as_view(), name='exporttemplate_import'),
     path('export-templates/edit/', views.ExportTemplateBulkEditView.as_view(), name='exporttemplate_bulk_edit'),
     path('export-templates/delete/', views.ExportTemplateBulkDeleteView.as_view(), name='exporttemplate_bulk_delete'),
+    path('export-templates/sync/', views.ExportTemplateBulkSyncDataView.as_view(), name='exporttemplate_bulk_sync'),
     path('export-templates/<int:pk>/', include(get_model_urls('extras', 'exporttemplate'))),
 
     # Saved filters

netbox/extras/views.py

@@ -121,6 +121,8 @@ class ExportTemplateListView(generic.ObjectListView):
     filterset = filtersets.ExportTemplateFilterSet
     filterset_form = forms.ExportTemplateFilterForm
     table = tables.ExportTemplateTable
+    template_name = 'extras/exporttemplate_list.html'
+    actions = ('add', 'import', 'export', 'bulk_edit', 'bulk_delete', 'bulk_sync')
 
 
 @register_model_view(ExportTemplate)
@@ -158,6 +160,10 @@ class ExportTemplateBulkDeleteView(generic.BulkDeleteView):
     table = tables.ExportTemplateTable
 
 
+class ExportTemplateBulkSyncDataView(generic.BulkSyncDataView):
+    queryset = ExportTemplate.objects.all()
+
+
 #
 # Saved filters
 #

netbox/templates/extras/configcontext.html

@@ -50,10 +50,10 @@ <h5 class="card-header">Config Context</h5>
                 {% endif %}
               </td>
             </tr>
-          <tr>
-            <th scope="row">Data Synced</th>
-            <td>{{ object.data_synced|placeholder }}</td>
-          </tr>
+            <tr>
+              <th scope="row">Data Synced</th>
+              <td>{{ object.data_synced|placeholder }}</td>
+            </tr>
           </table>
         </div>
       </div>
@@ -86,22 +86,8 @@ <h5>Data</h5>
           {% include 'extras/inc/configcontext_format.html' %}
         </div>
         <div class="card-body">
-          {% if object.data_file and object.data_file.last_updated > object.data_synced %}
-            <div class="alert alert-warning" role="alert">
-              <i class="mdi mdi-alert"></i> Data is out of sync with upstream file (<a href="{{ object.data_file.get_absolute_url }}">{{ object.data_file }}</a>).
-              {% if perms.extras.sync_configcontext %}
-                <div class="float-end">
-                  <form action="{% url 'extras:configcontext_sync' pk=object.pk %}" method="post">
-                    {% csrf_token %}
-                    <button type="submit" class="btn btn-primary btn-sm">
-                      <i class="mdi mdi-sync" aria-hidden="true"></i> Sync
-                    </button>
-                  </form>
-                </div>
-              {% endif %}
-            </div>
-          {% endif %}
-            {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %}
+          {% include 'inc/sync_warning.html' %}
+          {% include 'extras/inc/configcontext_data.html' with data=object.data format=format %}
         </div>
       </div>
     </div>