Initial work on #13381

Author: jeremystretch

docs/plugins/development/data-backends.md

@@ -0,0 +1,24 @@
+# Data Backends
+
+[Data sources](../../models/core/datasource.md) can be defined to reference data which exists on systems of record outside NetBox, such as a git repository or Amazon S3 bucket. Plugins can register their own backend classes to introduce support for additional resource types. This is done by subclassing NetBox's `DataBackend` class.
+
+```python
+# data_backends.py
+from netbox.data_backends import DataBackend
+
+class MyDataBackend(DataBackend):
+    name = 'mybackend'
+    label = 'My Backend'
+    ...
+```
+
+To register one or more data backends with NetBox, define a list named `backends` at the end of this file:
+
+```python
+backends = [MyDataBackend]
+```
+
+!!! tip
+    The path to the list of search indexes can be modified by setting `data_backends` in the PluginConfig instance.
+
+::: core.data_backends.DataBackend

docs/plugins/development/index.md

@@ -109,6 +109,7 @@ NetBox looks for the `config` variable within a plugin's `__init__.py` to load i
 | `middleware`          | A list of middleware classes to append after NetBox's build-in middleware                                                |
 | `queues`              | A list of custom background task queues to create                                                                        |
 | `search_extensions`   | The dotted path to the list of search index classes (default: `search.indexes`)                                          |
+| `data_backends`       | The dotted path to the list of data source backend classes (default: `data_backends.backends`)                           |
 | `template_extensions` | The dotted path to the list of template extension classes (default: `template_content.template_extensions`)              |
 | `menu_items`          | The dotted path to the list of menu items provided by the plugin (default: `navigation.menu_items`)                      |
 | `graphql_schema`      | The dotted path to the plugin's GraphQL schema class, if any (default: `graphql.schema`)                                 |

mkdocs.yml

@@ -136,6 +136,7 @@ nav:
             - Forms: 'plugins/development/forms.md'
             - Filters & Filter Sets: 'plugins/development/filtersets.md'
             - Search: 'plugins/development/search.md'
+            - Data Backends: 'plugins/development/data-backends.md'
             - REST API: 'plugins/development/rest-api.md'
             - GraphQL API: 'plugins/development/graphql-api.md'
             - Background Tasks: 'plugins/development/background-tasks.md'

netbox/core/api/serializers.py

@@ -1,7 +1,10 @@
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
 from rest_framework import serializers
 
 from core.choices import *
 from core.models import *
+from core.utils import get_data_backend_choices
 from netbox.api.fields import ChoiceField, ContentTypeField
 from netbox.api.serializers import BaseModelSerializer, NetBoxModelSerializer
 from users.api.nested_serializers import NestedUserSerializer
@@ -19,7 +22,7 @@ class DataSourceSerializer(NetBoxModelSerializer):
         view_name='core-api:datasource-detail'
     )
     type = ChoiceField(
-        choices=DataSourceTypeChoices
+        choices=get_data_backend_choices()
     )
     status = ChoiceField(
         choices=DataSourceStatusChoices,
@@ -38,6 +41,13 @@ class Meta:
             'parameters', 'ignore_rules', 'created', 'last_updated', 'file_count',
         ]
 
+    def clean(self):
+
+        if self.type and self.type not in get_data_backend_choices():
+            raise ValidationError({
+                'type': _("Unknown backend type: {type}".format(type=self.type))
+            })
+
 
 class DataFileSerializer(NetBoxModelSerializer):
     url = serializers.HyperlinkedIdentityField(

netbox/core/choices.py

@@ -7,18 +7,6 @@
 # Data sources
 #
 
-class DataSourceTypeChoices(ChoiceSet):
-    LOCAL = 'local'
-    GIT = 'git'
-    AMAZON_S3 = 'amazon-s3'
-
-    CHOICES = (
-        (LOCAL, _('Local'), 'gray'),
-        (GIT, 'Git', 'blue'),
-        (AMAZON_S3, 'Amazon S3', 'blue'),
-    )
-
-
 class DataSourceStatusChoices(ChoiceSet):
     NEW = 'new'
     QUEUED = 'queued'

netbox/core/data_backends.py

@@ -10,61 +10,24 @@
 from django.conf import settings
 from django.utils.translation import gettext as _
 
-from netbox.registry import registry
-from .choices import DataSourceTypeChoices
+from core.utils import register_data_backend
+from netbox.data_backends import DataBackend
 from .exceptions import SyncError
 
 __all__ = (
-    'LocalBackend',
     'GitBackend',
+    'LocalBackend',
     'S3Backend',
 )
 
 logger = logging.getLogger('netbox.data_backends')
 
 
-def register_backend(name):
-    """
-    Decorator for registering a DataBackend class.
-    """
-
-    def _wrapper(cls):
-        registry['data_backends'][name] = cls
-        return cls
-
-    return _wrapper
-
-
-class DataBackend:
-    parameters = {}
-    sensitive_parameters = []
-
-    # Prevent Django's template engine from calling the backend
-    # class when referenced via DataSource.backend_class
-    do_not_call_in_templates = True
-
-    def __init__(self, url, **kwargs):
-        self.url = url
-        self.params = kwargs
-        self.config = self.init_config()
-
-    def init_config(self):
-        """
-        Hook to initialize the instance's configuration.
-        """
-        return
-
-    @property
-    def url_scheme(self):
-        return urlparse(self.url).scheme.lower()
-
-    @contextmanager
-    def fetch(self):
-        raise NotImplemented()
-
-
-@register_backend(DataSourceTypeChoices.LOCAL)
+@register_data_backend()
 class LocalBackend(DataBackend):
+    name = 'local'
+    label = _('Local')
+    is_local = True
 
     @contextmanager
     def fetch(self):
@@ -74,8 +37,10 @@ def fetch(self):
         yield local_path
 
 
-@register_backend(DataSourceTypeChoices.GIT)
+@register_data_backend()
 class GitBackend(DataBackend):
+    name = 'git'
+    label = 'Git'
     parameters = {
         'username': forms.CharField(
             required=False,
@@ -144,8 +109,10 @@ def fetch(self):
         local_path.cleanup()
 
 
-@register_backend(DataSourceTypeChoices.AMAZON_S3)
+@register_data_backend()
 class S3Backend(DataBackend):
+    name = 'amazon-s3'
+    label = 'Amazon S3'
     parameters = {
         'aws_access_key_id': forms.CharField(
             label=_('AWS access key ID'),

netbox/core/filtersets.py

@@ -6,6 +6,7 @@
 from netbox.filtersets import BaseFilterSet, ChangeLoggedModelFilterSet, NetBoxModelFilterSet
 from .choices import *
 from .models import *
+from .utils import get_data_backend_choices
 
 __all__ = (
     'DataFileFilterSet',
@@ -16,7 +17,7 @@
 
 class DataSourceFilterSet(NetBoxModelFilterSet):
     type = django_filters.MultipleChoiceFilter(
-        choices=DataSourceTypeChoices,
+        choices=get_data_backend_choices,
         null_value=None
     )
     status = django_filters.MultipleChoiceFilter(

netbox/core/forms/bulk_edit.py

@@ -1,10 +1,9 @@
 from django import forms
 from django.utils.translation import gettext_lazy as _
 
-from core.choices import DataSourceTypeChoices
 from core.models import *
+from core.utils import get_data_backend_choices
 from netbox.forms import NetBoxModelBulkEditForm
-from utilities.forms import add_blank_choice
 from utilities.forms.fields import CommentField
 from utilities.forms.widgets import BulkEditNullBooleanSelect
 
@@ -16,7 +15,8 @@
 class DataSourceBulkEditForm(NetBoxModelBulkEditForm):
     type = forms.ChoiceField(
         label=_('Type'),
-        choices=add_blank_choice(DataSourceTypeChoices),
+        # TODO: Field value should be empty on init (needs add_blank_choice())
+        choices=get_data_backend_choices,
         required=False,
         initial=''
     )

netbox/core/forms/filtersets.py

@@ -5,6 +5,7 @@
 
 from core.choices import *
 from core.models import *
+from core.utils import get_data_backend_choices
 from extras.forms.mixins import SavedFiltersMixin
 from extras.utils import FeatureQuery
 from netbox.forms import NetBoxModelFilterSetForm
@@ -27,7 +28,7 @@ class DataSourceFilterForm(NetBoxModelFilterSetForm):
     )
     type = forms.MultipleChoiceField(
         label=_('Type'),
-        choices=DataSourceTypeChoices,
+        choices=get_data_backend_choices,
         required=False
     )
     status = forms.MultipleChoiceField(

netbox/core/forms/model_forms.py

@@ -5,6 +5,7 @@
 
 from core.forms.mixins import SyncedDataMixin
 from core.models import *
+from core.utils import get_data_backend_choices
 from netbox.forms import NetBoxModelForm
 from netbox.registry import registry
 from utilities.forms import get_field_value
@@ -18,6 +19,10 @@
 
 
 class DataSourceForm(NetBoxModelForm):
+    type = forms.ChoiceField(
+        choices=get_data_backend_choices,
+        widget=HTMXSelect()
+    )
     comments = CommentField()
 
     class Meta:
@@ -26,7 +31,6 @@ class Meta:
             'name', 'type', 'source_url', 'enabled', 'description', 'comments', 'ignore_rules', 'tags',
         ]
         widgets = {
-            'type': HTMXSelect(),
             'ignore_rules': forms.Textarea(
                 attrs={
                     'rows': 5,
@@ -56,12 +60,13 @@ def __init__(self, *args, **kwargs):
 
         # Add backend-specific form fields
         self.backend_fields = []
-        for name, form_field in backend.parameters.items():
-            field_name = f'backend_{name}'
-            self.backend_fields.append(field_name)
-            self.fields[field_name] = copy.copy(form_field)
-            if self.instance and self.instance.parameters:
-                self.fields[field_name].initial = self.instance.parameters.get(name)
+        if backend:
+            for name, form_field in backend.parameters.items():
+                field_name = f'backend_{name}'
+                self.backend_fields.append(field_name)
+                self.fields[field_name] = copy.copy(form_field)
+                if self.instance and self.instance.parameters:
+                    self.fields[field_name].initial = self.instance.parameters.get(name)
 
     def save(self, *args, **kwargs):
 

netbox/core/migrations/0006_datasource_type_remove_choices.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.6 on 2023-10-20 17:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('core', '0005_job_created_auto_now'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='datasource',
+            name='type',
+            field=models.CharField(max_length=50),
+        ),
+    ]

netbox/core/models/data.py

@@ -45,9 +45,7 @@ class DataSource(JobsMixin, PrimaryModel):
     )
     type = models.CharField(
         verbose_name=_('type'),
-        max_length=50,
-        choices=DataSourceTypeChoices,
-        default=DataSourceTypeChoices.LOCAL
+        max_length=50
     )
     source_url = models.CharField(
         max_length=200,
@@ -96,8 +94,9 @@ def get_absolute_url(self):
     def docs_url(self):
         return f'{settings.STATIC_URL}docs/models/{self._meta.app_label}/{self._meta.model_name}/'
 
-    def get_type_color(self):
-        return DataSourceTypeChoices.colors.get(self.type)
+    def get_type_display(self):
+        if backend := registry['data_backends'].get(self.type):
+            return backend.label
 
     def get_status_color(self):
         return DataSourceStatusChoices.colors.get(self.status)
@@ -110,10 +109,6 @@ def url_scheme(self):
     def backend_class(self):
         return registry['data_backends'].get(self.type)
 
-    @property
-    def is_local(self):
-        return self.type == DataSourceTypeChoices.LOCAL
-
     @property
     def ready_for_sync(self):
         return self.enabled and self.status not in (
@@ -124,7 +119,7 @@ def ready_for_sync(self):
     def clean(self):
 
         # Ensure URL scheme matches selected type
-        if self.type == DataSourceTypeChoices.LOCAL and self.url_scheme not in ('file', ''):
+        if self.backend_class.is_local and self.url_scheme not in ('file', ''):
             raise ValidationError({
                 'source_url': f"URLs for local sources must start with file:// (or specify no scheme)"
             })

netbox/core/tables/data.py

@@ -15,9 +15,6 @@ class DataSourceTable(NetBoxTable):
         verbose_name=_('Name'),
         linkify=True
     )
-    type = columns.ChoiceFieldColumn(
-        verbose_name=_('Type'),
-    )
     status = columns.ChoiceFieldColumn(
         verbose_name=_('Status'),
     )
@@ -34,8 +31,8 @@ class DataSourceTable(NetBoxTable):
     class Meta(NetBoxTable.Meta):
         model = DataSource
         fields = (
-            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters', 'created',
-            'last_updated', 'file_count',
+            'pk', 'id', 'name', 'type', 'status', 'enabled', 'source_url', 'description', 'comments', 'parameters',
+            'created', 'last_updated', 'file_count',
         )
         default_columns = ('pk', 'name', 'type', 'status', 'enabled', 'description', 'file_count')